Added values from NF Portal Dataset schema to MC2 DataCatalog valid v…

.github/workflows/build-jsonld.yml

@@ -16,7 +16,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        schematic-version: [ 24.10.2 ]
+        schematic-version: [ 25.8.1 ]
 
     steps:
       - name: Install System Dependencies
@@ -43,7 +43,8 @@ jobs:
           python3.10 -m venv .venv
           chmod 755 .venv/bin/activate
           source .venv/bin/activate
-          pip3.10 install schematicpy==${{ matrix.schematic-version }} ipython==8.18.1
+          pip3.10 install --upgrade pip
+          pip3.10 install -r requirements.txt
           make CSV=$schema_filename
 
       - name: Commit files

Makefile

@@ -1,11 +1,33 @@
 CSV := mc2.model.csv
+QC := ./qc_model/mc2_qc.model.csv
+DATA := DataDSP Study FileView PublicationView GrantView ToolView EducationalResource DatasetView DataCatalog
 
-all: collate convert
+all: collate fix generate-json convert
+
+qc: collate fix qc_fix qc_convert
 
 collate:
 	@echo "Collating module components..."
+	python update_valid_values.py
 	head -1 modules/consortium/annotationProperty.csv > ${CSV}
 	tail -n +2 -q modules/*/annotationProperty.csv >> ${CSV}
 
+fix:
+	@echo "Modifying columnType to remove string_list"
+	awk -F ',' '{ gsub(/string_list/, ""); gsub(/^integer/, "number", $$10); gsub(/^integer/, "number", $$11); gsub(/^int/, "num", $$10); gsub(/^int/, "num", $$11); gsub(/^bool$$/, "boolean", $$10); gsub(/^bool$$/, "boolean", $$11); print }' OFS=',' ${CSV} > tmp_${CSV}
+	rm ${CSV}
+	mv tmp_${CSV} ${CSV}
+
+qc_fix:
+	awk -F ',' '{ gsub(/matchAtLeastOne.*/, "list like"); print }' OFS=',' ${CSV} > tmp.csv
+	mv tmp.csv ${QC}
+
 convert:
-	schematic schema convert ${CSV}
+	schematic schema convert ${CSV}
+
+qc_convert:
+	schematic schema convert ${QC}
+
+generate-json:
+	$(foreach d,$(DATA), schematic schema generate-jsonschema -dms ${CSV} -od . -dt $(d);)
+	rm *.schema.json

all_valid_values.csv

@@ -8912,6 +8912,7 @@ duo,GS,
 duo,DS,
 duo,GRU,
 duo,PS,
+duo,Pending Annotation,
 dataTier,Anonymous,
 dataTier,Open,
 dataTier,Controlled,
@@ -10200,6 +10201,7 @@ institution_alias,Sage,
 institution_alias,Salk,
 institution_alias,StJude,
 institution_alias,Stanford,
+institution_alias,TUM,
 institution_alias,U Miami,
 institution_alias,UA,
 institution_alias,UARK,
@@ -10233,6 +10235,7 @@ institution_alias,UWM,
 institution_alias,University of Illinois,
 institution_alias,University of Texas MD Anderson Cancer Center,
 institution_alias,Vanderbilt,
+institution_alias,VT,
 institution_alias,WUSTL,
 institution_alias,WCM,
 institution_alias,Wake Forest,
@@ -10289,6 +10292,7 @@ institution_name,St. Jude Children's Research Hospital,
 institution_name,Stanford University,
 institution_name,Stony Brook University,
 institution_name,The University of Texas Health Science Center at San Antonio,
+institution_name,Tufts University Medford,
 institution_name,University of Alabama at Birmingham,
 institution_name,University of Arizona,
 institution_name,University of Arkansas at Fayetteville,
@@ -10322,6 +10326,7 @@ institution_name,University of Virginia,
 institution_name,University of Washington,
 institution_name,University of Wisconsin-Madison,
 institution_name,Vanderbilt University,
+institution_name,Virginia Polytechnic Institute,
 institution_name,Washington University in St. Louis,
 institution_name,Wake Forest,
 institution_name,Weill Cornell Medicine,