Merge branch 'prod' of https://github.com/Sage-Bionetworks/data_curat…

…or_config into prod
Sage-Bionetworks · Oct 31, 2024 · f1f6cf5 · f1f6cf5
2 parents e328dbd + b0b18b4
commit f1f6cf5
Show file tree

Hide file tree

Showing 92 changed files with 69 additions and 46 deletions.
diff --git a/.github/config_schema.py b/.github/config_schema.py
@@ -11,7 +11,7 @@ def get_args():
     """Set up command-line interface and get arguments."""
     parser = argparse.ArgumentParser()
     parser.add_argument('-c', '--config_path', metavar="",
-                        required=True, help='path to schematic config ymal file')
+                        required=True, help='path to schematic config yaml file')
     parser.add_argument('-sr', '--service_repo', metavar="",
                         help='repo path to service')
     parser.add_argument('-o', '--out_dir', default='www', metavar="",

diff --git a/ADKP/dca_config.json b/ADKP/dca_config.json
@@ -7,8 +7,8 @@
     "template_menu_config_file": "https://raw.githubusercontent.com/adknowledgeportal/data-models/main/dca-template-config.json",
     "logo_location": "https://raw.githubusercontent.com/Sage-Bionetworks/data_curator_config/prod/ADKP/ADKnowledgePortal.png",
     "logo_link": "https://adknowledgeportal.synapse.org",
-    "dcc_help_link": "",
-    "portal_help_link": ""
+    "dcc_help_link": "https://dca-docs.scrollhelp.site/DCA/Working-version/Project-Agnostic/",
+    "portal_help_link": "https://help.adknowledgeportal.org/apd/"
   },
   "dca": {
     "use_compliance_dashboard": false,
@@ -25,7 +25,8 @@
       "use_annotations": true
     },
     "model_validate": {
-      "restrict_rules": false
+      "restrict_rules": false,
+      "enable_cross_manifest_validation": true
     },
     "model_submit": {
       "table_column_names": "display_label",

diff --git a/EL/dca_config.json b/EL/dca_config.json
@@ -27,7 +27,7 @@
     "model_submit": {
       "use_schema_labels": true,
       "table_manipulation": "replace",
-      "manifest_record_type": "table_and_file",
+      "manifest_record_type": "file_only",
       "hide_blanks": false
     }
   }

diff --git a/HTAN/dca_config.json b/HTAN/dca_config.json
@@ -1,14 +1,14 @@
 {
   "dcc": {
-    "name": "HTAN All Projects",
+    "name": "HTAN Phase 1 All Projects",
     "synapse_asset_view": "syn20446927",
-    "data_model_url": "https://raw.githubusercontent.com/ncihtan/data-models/v24.5.2/HTAN.model.jsonld",
+    "data_model_url": "https://raw.githubusercontent.com/ncihtan/data-models/v24.9.1/HTAN.model.jsonld",
     "data_model_info": "",
-    "template_menu_config_file": "https://raw.githubusercontent.com/ncihtan/data-models/v24.5.2/dca-template-config.json",
+    "template_menu_config_file": "https://raw.githubusercontent.com/ncihtan/data-models/v24.9.1/dca-template-config.json",
     "logo_location": "https://raw.githubusercontent.com/Sage-Bionetworks/data_curator_config/prod/HTAN/HTAN_text_logo.png",
     "logo_link": "https://humantumoratlas.org",
-    "dcc_help_link": "",
-    "portal_help_link": ""
+    "dcc_help_link": "https://dca-docs.scrollhelp.site/DCA/Working-version/HTAN/",
+    "portal_help_link": "https://docs.humantumoratlas.org/"
   },
   "dca": {
     "use_compliance_dashboard": false,

diff --git a/HTAN/templates/10xVisiumSpatialTranscriptomics-AuxiliaryFiles.xlsx b/HTAN/templates/10xVisiumSpatialTranscriptomics-AuxiliaryFiles.xlsx
diff --git a/HTAN/templates/10xVisiumSpatialTranscriptomics-RNA-seqLevel1.xlsx b/HTAN/templates/10xVisiumSpatialTranscriptomics-RNA-seqLevel1.xlsx
diff --git a/HTAN/templates/10xVisiumSpatialTranscriptomics-RNA-seqLevel2.xlsx b/HTAN/templates/10xVisiumSpatialTranscriptomics-RNA-seqLevel2.xlsx
diff --git a/HTAN/templates/10xVisiumSpatialTranscriptomics-RNA-seqLevel3.xlsx b/HTAN/templates/10xVisiumSpatialTranscriptomics-RNA-seqLevel3.xlsx
diff --git a/HTAN/templates/AcuteLymphoblasticLeukemiaTier3.xlsx b/HTAN/templates/AcuteLymphoblasticLeukemiaTier3.xlsx
diff --git a/HTAN/templates/Biospecimen.xlsx b/HTAN/templates/Biospecimen.xlsx
diff --git a/HTAN/templates/BreastCancerTier3.xlsx b/HTAN/templates/BreastCancerTier3.xlsx
diff --git a/HTAN/templates/BulkMethylation-seqLevel1.xlsx b/HTAN/templates/BulkMethylation-seqLevel1.xlsx
diff --git a/HTAN/templates/BulkMethylation-seqLevel2.xlsx b/HTAN/templates/BulkMethylation-seqLevel2.xlsx
diff --git a/HTAN/templates/BulkMethylation-seqLevel3.xlsx b/HTAN/templates/BulkMethylation-seqLevel3.xlsx
diff --git a/HTAN/templates/BulkRNA-seqLevel1.xlsx b/HTAN/templates/BulkRNA-seqLevel1.xlsx
diff --git a/HTAN/templates/BulkRNA-seqLevel2.xlsx b/HTAN/templates/BulkRNA-seqLevel2.xlsx
diff --git a/HTAN/templates/BulkRNA-seqLevel3.xlsx b/HTAN/templates/BulkRNA-seqLevel3.xlsx
diff --git a/HTAN/templates/BulkWESLevel1.xlsx b/HTAN/templates/BulkWESLevel1.xlsx
diff --git a/HTAN/templates/BulkWESLevel2.xlsx b/HTAN/templates/BulkWESLevel2.xlsx
diff --git a/HTAN/templates/BulkWESLevel3.xlsx b/HTAN/templates/BulkWESLevel3.xlsx
diff --git a/HTAN/templates/ClinicalDataTier2.xlsx b/HTAN/templates/ClinicalDataTier2.xlsx
diff --git a/HTAN/templates/ColorectalCancerTier3.xlsx b/HTAN/templates/ColorectalCancerTier3.xlsx
diff --git a/HTAN/templates/Demographics.xlsx b/HTAN/templates/Demographics.xlsx
diff --git a/HTAN/templates/Diagnosis.xlsx b/HTAN/templates/Diagnosis.xlsx
diff --git a/HTAN/templates/Exposure.xlsx b/HTAN/templates/Exposure.xlsx
diff --git a/HTAN/templates/FamilyHistory.xlsx b/HTAN/templates/FamilyHistory.xlsx
diff --git a/HTAN/templates/FollowUp.xlsx b/HTAN/templates/FollowUp.xlsx
diff --git a/HTAN/templates/HI-C-seqLevel1.xlsx b/HTAN/templates/HI-C-seqLevel1.xlsx
diff --git a/HTAN/templates/HI-C-seqLevel2.xlsx b/HTAN/templates/HI-C-seqLevel2.xlsx
diff --git a/HTAN/templates/HI-C-seqLevel3.xlsx b/HTAN/templates/HI-C-seqLevel3.xlsx
diff --git a/HTAN/templates/HTANRPPAAntibodyTable.xlsx b/HTAN/templates/HTANRPPAAntibodyTable.xlsx
diff --git a/HTAN/templates/ImagingLevel1.xlsx b/HTAN/templates/ImagingLevel1.xlsx
diff --git a/HTAN/templates/ImagingLevel2.xlsx b/HTAN/templates/ImagingLevel2.xlsx
diff --git a/HTAN/templates/ImagingLevel3Channels.xlsx b/HTAN/templates/ImagingLevel3Channels.xlsx
diff --git a/HTAN/templates/ImagingLevel3Image.xlsx b/HTAN/templates/ImagingLevel3Image.xlsx
diff --git a/HTAN/templates/ImagingLevel3Segmentation.xlsx b/HTAN/templates/ImagingLevel3Segmentation.xlsx
diff --git a/HTAN/templates/ImagingLevel4.xlsx b/HTAN/templates/ImagingLevel4.xlsx
diff --git a/HTAN/templates/LungCancerTier3.xlsx b/HTAN/templates/LungCancerTier3.xlsx
diff --git a/HTAN/templates/MassSpectrometryAuxiliaryFile.xlsx b/HTAN/templates/MassSpectrometryAuxiliaryFile.xlsx
diff --git a/HTAN/templates/MassSpectrometryLevel1.xlsx b/HTAN/templates/MassSpectrometryLevel1.xlsx
diff --git a/HTAN/templates/MassSpectrometryLevel2.xlsx b/HTAN/templates/MassSpectrometryLevel2.xlsx
diff --git a/HTAN/templates/MassSpectrometryLevel3.xlsx b/HTAN/templates/MassSpectrometryLevel3.xlsx
diff --git a/HTAN/templates/MassSpectrometryLevel4.xlsx b/HTAN/templates/MassSpectrometryLevel4.xlsx
diff --git a/HTAN/templates/MelanomaTier3.xlsx b/HTAN/templates/MelanomaTier3.xlsx
diff --git a/HTAN/templates/MolecularTest.xlsx b/HTAN/templates/MolecularTest.xlsx
diff --git a/HTAN/templates/NeuroblastomaandGliomaTier3.xlsx b/HTAN/templates/NeuroblastomaandGliomaTier3.xlsx
diff --git a/HTAN/templates/OtherAssay.xlsx b/HTAN/templates/OtherAssay.xlsx
diff --git a/HTAN/templates/OvarianCancerTier3.xlsx b/HTAN/templates/OvarianCancerTier3.xlsx
diff --git a/HTAN/templates/PancreaticCancerTier3.xlsx b/HTAN/templates/PancreaticCancerTier3.xlsx
diff --git a/HTAN/templates/Patient.xlsx b/HTAN/templates/Patient.xlsx
diff --git a/HTAN/templates/ProstateCancerTier3.xlsx b/HTAN/templates/ProstateCancerTier3.xlsx
diff --git a/HTAN/templates/PublicationManifest.xlsx b/HTAN/templates/PublicationManifest.xlsx
diff --git a/HTAN/templates/RPPALevel2.xlsx b/HTAN/templates/RPPALevel2.xlsx
diff --git a/HTAN/templates/RPPALevel3.xlsx b/HTAN/templates/RPPALevel3.xlsx
diff --git a/HTAN/templates/RPPALevel4.xlsx b/HTAN/templates/RPPALevel4.xlsx
diff --git a/HTAN/templates/SRRSBiospecimen.xlsx b/HTAN/templates/SRRSBiospecimen.xlsx
diff --git a/HTAN/templates/SRRSClinicalDataTier2.xlsx b/HTAN/templates/SRRSClinicalDataTier2.xlsx
diff --git a/HTAN/templates/SRRSImagingLevel2.xlsx b/HTAN/templates/SRRSImagingLevel2.xlsx
diff --git a/HTAN/templates/SarcomaTier3.xlsx b/HTAN/templates/SarcomaTier3.xlsx
diff --git a/HTAN/templates/ScATAC-seqLevel1.xlsx b/HTAN/templates/ScATAC-seqLevel1.xlsx
diff --git a/HTAN/templates/ScATAC-seqLevel2.xlsx b/HTAN/templates/ScATAC-seqLevel2.xlsx
diff --git a/HTAN/templates/ScATAC-seqLevel3.xlsx b/HTAN/templates/ScATAC-seqLevel3.xlsx
diff --git a/HTAN/templates/ScATAC-seqLevel4.xlsx b/HTAN/templates/ScATAC-seqLevel4.xlsx
diff --git a/HTAN/templates/ScDNA-seqLevel1.xlsx b/HTAN/templates/ScDNA-seqLevel1.xlsx
diff --git a/HTAN/templates/ScDNA-seqLevel2.xlsx b/HTAN/templates/ScDNA-seqLevel2.xlsx
diff --git a/HTAN/templates/ScRNA-seqLevel1.xlsx b/HTAN/templates/ScRNA-seqLevel1.xlsx
diff --git a/HTAN/templates/ScRNA-seqLevel2.xlsx b/HTAN/templates/ScRNA-seqLevel2.xlsx
diff --git a/HTAN/templates/ScRNA-seqLevel3.xlsx b/HTAN/templates/ScRNA-seqLevel3.xlsx
diff --git a/HTAN/templates/ScRNA-seqLevel4.xlsx b/HTAN/templates/ScRNA-seqLevel4.xlsx
diff --git a/HTAN/templates/ScmC-seqLevel1.xlsx b/HTAN/templates/ScmC-seqLevel1.xlsx
diff --git a/HTAN/templates/ScmC-seqLevel2.xlsx b/HTAN/templates/ScmC-seqLevel2.xlsx
diff --git a/HTAN/templates/Therapy.xlsx b/HTAN/templates/Therapy.xlsx
diff --git a/HTAN2/HTAN_text_logo.png b/HTAN2/HTAN_text_logo.png
diff --git a/HTAN2/dca_config.json b/HTAN2/dca_config.json
@@ -0,0 +1,34 @@
+{
+  "dcc": {
+    "name": "HTAN Phase 2 All Projects",
+    "synapse_asset_view": "syn63714258",
+    "data_model_url": "https://raw.githubusercontent.com/ncihtan/data-models/v24.9.1/HTAN.model.jsonld",
+    "data_model_info": "",
+    "template_menu_config_file": "https://raw.githubusercontent.com/ncihtan/data-models/v24.9.1/dca-template-config.json",
+    "logo_location": "https://raw.githubusercontent.com/Sage-Bionetworks/data_curator_config/prod/HTAN/HTAN_text_logo.png",
+    "logo_link": "https://humantumoratlas.org",
+    "dcc_help_link": "https://dca-docs.scrollhelp.site/DCA/Working-version/HTAN/",
+    "portal_help_link": "https://docs.humantumoratlas.org/"
+  },
+  "dca": {
+    "use_compliance_dashboard": false,
+    "primary_col": "#605ca8",
+    "secondary_col": "#5F008C",
+    "sidebar_col": "#191919"
+  },
+  "schematic": {
+    "manifest_generate": {
+      "output_format": "google_sheet",
+      "use_annotations": false
+    },
+    "model_validate": {
+      "restrict_rules": false
+    },
+    "model_submit": {
+      "use_schema_labels": true,
+      "table_manipulation": "replace",
+      "manifest_record_type": "file_only",
+      "hide_blanks": false
+    }
+  }
+}
diff --git a/MC2/dca_config.json b/MC2/dca_config.json
@@ -2,9 +2,9 @@
   "dcc": {
     "name": "Cancer Complexity Knowledge Portal - Database",
     "synapse_asset_view": "syn27210848",
-    "data_model_url": "https://raw.githubusercontent.com/mc2-center/data-models/v5.1.2/mc2.model.jsonld",
+    "data_model_url": "https://raw.githubusercontent.com/mc2-center/data-models/v7.1.0/mc2.model.jsonld",
     "data_model_info": "https://mc2-center.github.io/data-models/",
-    "template_menu_config_file": "https://raw.githubusercontent.com/mc2-center/data-models/v5.1.2/dca_config/dca-template-config.json",
+    "template_menu_config_file": "https://raw.githubusercontent.com/mc2-center/data-models/v7.1.0/dca_config/dca-template-config.json",
     "logo_location": "https://raw.githubusercontent.com/Sage-Bionetworks/data_curator_config/prod/MC2/cckp_logo.png",
     "logo_link": "https://cancercomplexity.synapse.org",
     "dcc_help_link": "",

diff --git a/MC2/templates/.DS_Store b/MC2/templates/.DS_Store
diff --git a/MC2/templates/Biospecimen.xlsx b/MC2/templates/Biospecimen.xlsx
diff --git a/MC2/templates/DatasetView.xlsx b/MC2/templates/DatasetView.xlsx
diff --git a/MC2/templates/EducationalResource.xlsx b/MC2/templates/EducationalResource.xlsx
diff --git a/MC2/templates/NanoStringGeoMXROISegmentAnnotation.xlsx b/MC2/templates/NanoStringGeoMXROISegmentAnnotation.xlsx
diff --git a/MC2/templates/NanoStringGeoMxDSPImagingLevel2.xlsx b/MC2/templates/NanoStringGeoMxDSPImagingLevel2.xlsx
diff --git a/MC2/templates/NanoStringGeoMxDSPLevel1.xlsx b/MC2/templates/NanoStringGeoMxDSPLevel1.xlsx
diff --git a/MC2/templates/NanoStringGeoMxDSPLevel2.xlsx b/MC2/templates/NanoStringGeoMxDSPLevel2.xlsx
diff --git a/MC2/templates/NanoStringGeoMxDSPLevel3.xlsx b/MC2/templates/NanoStringGeoMxDSPLevel3.xlsx
diff --git a/MC2/templates/PublicationView.xlsx b/MC2/templates/PublicationView.xlsx
diff --git a/MC2/templates/ToolView.xlsx b/MC2/templates/ToolView.xlsx
diff --git a/NF-OSI/dca_config.json b/NF-OSI/dca_config.json
@@ -2,9 +2,9 @@
   "dcc": {
     "name": "NF-OSI",
     "synapse_asset_view": "syn16858331",
-    "data_model_url": "https://raw.githubusercontent.com/nf-osi/nf-metadata-dictionary/v9.3.0/NF.jsonld",
+    "data_model_url": "https://raw.githubusercontent.com/nf-osi/nf-metadata-dictionary/v9.9.0/NF.jsonld",
     "data_model_info": "",
-    "template_menu_config_file": "https://raw.githubusercontent.com/nf-osi/nf-metadata-dictionary/v9.3.0/dca-template-config.json",
+    "template_menu_config_file": "https://raw.githubusercontent.com/nf-osi/nf-metadata-dictionary/v9.9.0/dca-template-config.json",
     "logo_location": "",
     "logo_link": "",
     "dcc_help_link": "",
@@ -17,6 +17,9 @@
     "sidebar_col": "#191919"
   },
   "schematic": {
+    "global": {
+      "data_model_labels": "class_label"
+    },
     "manifest_generate": {
       "output_format": "google_sheet",
       "use_annotations": true
@@ -25,7 +28,6 @@
       "restrict_rules": false
     },
     "model_submit": {
-      "use_schema_labels": true,
       "table_manipulation": "replace",
       "manifest_record_type": "table_and_file",
       "hide_blanks": true,

diff --git a/README.md b/README.md
@@ -43,6 +43,7 @@ Each DCC will have its own directory to store configuration files. At minimum it
 -   `model_validate`
 
     -   `restrict_rules`: `TRUE` or `FALSE`. [Schematic model validate and submit option](https://sage-schematic.readthedocs.io/en/develop/cli_reference.html#schematic-model-validate).
+    -   `enable_cross_manifest_validation`: `TRUE` or `FALSE`. Use cross manifest validation.
 
 -   `model_submit`
 

diff --git a/VEOIBD/dca-template-config.json b/VEOIBD/dca-template-config.json
@@ -9,31 +9,6 @@
       "display_name": "Biospecimen Metadata Template",
       "schema_name": "BiospecimenMetadataTemplate",
       "type": "record"
-    },
-    {
-      "display_name": "Bulk RNAseq Assay Template",
-      "schema_name": "BulkRNAseqAssayTemplate",
-      "type": "record"
-    },
-    {
-      "display_name": "scRNAseq Assay Template",
-      "schema_name": "ScRNAseqAssayTemplate",
-      "type": "record"
-    },
-    {
-      "display_name": "Bulk RNAseq Raw File Annotations",
-      "schema_name": "BulkRNAseqRawFileAnnotations",
-      "type": "file"
-    },
-    {
-      "display_name": "Bulk RNASeq Counts File Annotations",
-      "schema_name": "BulkRNASeqCountsFileAnnotations",
-      "type": "file"
-    },
-    {
-      "display_name": "Metadata File Annotations",
-      "schema_name": "MetadataFileAnnotations",
-      "type": "file"
     }
   ],
   "service_version": "v23.1.1",

diff --git a/VEOIBD/dca_config.json b/VEOIBD/dca_config.json
@@ -22,13 +22,13 @@
       "use_annotations": false
     },
     "model_validate": {
-      "restrict_rules": true
+      "restrict_rules": false
     },
     "model_submit": {
       "use_schema_labels": true,
-      "table_manipulation": "upsert",
+      "table_manipulation": "replace",
       "manifest_record_type": "file_only",
       "hide_blanks": false
     }
   }
-}
+}
diff --git a/demo/dca-template-config.json b/demo/dca-template-config.json
@@ -18,9 +18,14 @@
     {
       "display_name": "Other Assay",
       "schema_name": "OtherAssay",
+      "type": "file"
+    },
+    {
+      "display_name": "MockComponent",
+      "schema_name": "MockComponent",
       "type": "record"
     }
   ],
-  "service_version": "v23.1.1",
+  "service_version": "v24.7.2",
   "schema_version": ""
-}
+}
diff --git a/tenants.json b/tenants.json
@@ -26,7 +26,7 @@
       "config_location":"BTC/dca_config.json"
     },
     {
-      "name":"HTAN All Projects",
+      "name":"HTAN Phase 1 All Projects",
       "synapse_asset_view":"syn20446927",
       "config_location":"HTAN/dca_config.json"
     },
@@ -49,6 +49,11 @@
       "name":"DCA Demo Upsert",
       "synapse_asset_view":"syn51489635",
       "config_location":"demo_upsert/dca_config.json"
+    },
+    {
+      "name":"HTAN Phase 2 All Projects",
+      "synapse_asset_view":"syn63714258",
+      "config_location":"HTAN2/dca_config.json"
     }
   ]
 }