added demo config & fixed bug when used with mysql database & fixed b…

…ug when vcf file upload is enabled

.gitignore

@@ -172,6 +172,7 @@ fabric.properties
 .env_localtest
 .env_dev
 .env_prod
+.env_demo
 src/frontend_celery/logs
 /src/frontend_celery/webapp/static/files/
 classified_structural_variants

src/annotation_service/annotation_jobs/automatic_classification_job.py

@@ -391,8 +391,8 @@ def get_autoclass_json(self, variant_id, conn: Connection) -> str:
 
         result_json = json.dumps(result)
 
-        #with open("/mnt/storage2/users/ahdoebm1/HerediVar/src/annotation_service/test.json", "w") as f:
-        #    f.write(result_json)
+        with open("/var/www/html/heredivar-demo/logs/herediclassify_jsons/test.json", "w") as f:
+            f.write(result_json)
 
         return result_json
 

src/common/db_IO.py

@@ -3055,7 +3055,7 @@ def delete_unknown_heredicare_annotations(self, variant_id):
         heredicare_annotation_type_id = self.get_most_recent_annotation_type_id('heredicare')
         command = """
                 DELETE FROM variant_heredicare_annotation WHERE id IN (
-	                SELECT id FROM variant_heredicare_annotation WHERE annotation_type_id = %s AND variant_id = %s AND vid NOT IN (SELECT external_id FROM variant_ids WHERE annotation_type_id = %s AND variant_id = %s)
+	                SELECT id FROM (SELECT * FROM variant_heredicare_annotation) as useless WHERE annotation_type_id = %s AND variant_id = %s AND vid NOT IN (SELECT external_id FROM variant_ids WHERE annotation_type_id = %s AND variant_id = %s)
                 )
             """
         self.cursor.execute(command, (heredicare_annotation_type_id, variant_id, heredicare_vid_annotation_type_id, variant_id))

src/common/paths.py

@@ -229,5 +229,74 @@ def joinpaths(path, *paths):
     # IGV data
     igv_data_path = joinpaths(workdir, "src/frontend_celery/webapp/static/packages/igv/data")
 
+    # clinvar submission
+    clinvar_submission_schema = joinpaths(resources_dir, "clinvar_submission_schemas/clinvar_submission_schema_15_05_24.json")
+
+
+
+elif webapp_env == 'demo':
+    """ configuration for the demo environment """
+
+    # general paths
+    workdir = "/var/www/html/heredivar-demo"
+    datadir = joinpaths(workdir, "data/dbs")
+    hardcoded_datadir = joinpaths(workdir, "data/dbs_hardcoded")
+    toolsdir = joinpaths(workdir, "tools")
+    resources_dir = joinpaths(workdir, 'resources')
+    logs_dir = joinpaths(workdir, 'logs')
+    downloads_dir = joinpaths(workdir, 'downloads')
+    download_variant_list_dir = joinpaths(downloads_dir, 'variant_lists')
+
+    # webapp logs path
+    webapp_log_dir = joinpaths(logs_dir, "webapp")
+    webapp_log = joinpaths(webapp_log_dir, "webapp.log")
+
+    #tools
+    vep_path = joinpaths(toolsdir, "ensembl-vep")
+    vep_cache_dir = joinpaths(vep_path, "data/cache")
+    os.environ['PERL5LIB'] = vep_path + "/Bio/:" + vep_path + "/cpan/lib/perl5/:" + os.environ.get('PERL5LIB', '')
+    ngs_bits_path = joinpaths(toolsdir, "ngs-bits/bin")
+    htslib_path = joinpaths(toolsdir, "htslib-1.16")
+    samtools_path = joinpaths(toolsdir, "samtools/samtools")
+    automatic_classification_path = joinpaths(toolsdir, "herediclass")
+    automatic_classification_config_path = os.path.join(automatic_classification_path, "config_production.yaml")
+
+    # data
+    ref_genome_dir = joinpaths(workdir, "data/genomes")
+    ref_genome_path = joinpaths(ref_genome_dir, "GRCh38.fa")
+    ref_genome_path_grch37 = joinpaths(ref_genome_dir, "GRCh37.fa")
+    chainfile_path = joinpaths(ref_genome_dir, "hg19ToHg38.fixed.over.chain.gz")
+    ensembl_transcript_path = joinpaths(datadir, "ensembl/Homo_sapiens.GRCh38.110.gff3")
+    refseq_transcript_path = joinpaths(datadir, "RefSeq/refseq_transcripts_110.gff")
+    refseq_transcript_4_consequence_path = joinpaths(datadir, "RefSeq/refseq_transcripts_110.4consequence.gff")
+
+    #metadata
+    gnomad_path = joinpaths(datadir, "gnomAD/gnomAD_genome_GRCh38.vcf.gz")
+    gnomad_m_path  = joinpaths(datadir, "gnomAD/gnomAD_mito_GRCh38.vcf.gz")
+    phylop_file_path = joinpaths(datadir, "phyloP/hg38.phyloP100way.bw")
+    dbsnp_path = joinpaths(datadir, "dbSNP/dbSNP_v155.vcf.gz")
+    revel_path = joinpaths(datadir, "REVEL/revel_grch38_all_chromosomes.vcf.gz")
+    spliceai_snv_path = joinpaths(datadir, "SpliceAI/spliceai_scores.masked.snv.hg38.vcf.gz")
+    spliceai_indel_path = joinpaths(datadir, "SpliceAI/spliceai_scores.masked.indel.hg38.vcf.gz")
+    cadd_snvs_path = joinpaths(datadir, "CADD/CADD_SNVs_1.6_GRCh38.vcf.gz")
+    cadd_indels_path = joinpaths(datadir, "CADD/CADD_InDels_1.6_GRCh38.vcf.gz")
+    clinvar_path = joinpaths(datadir, "ClinVar/clinvar_converted_GRCh38.vcf.gz")
+    submission_summary_path = joinpaths(datadir, "ClinVar/submission_summary_preprocessed.txt.gz")
+    BRCA_exchange_path = joinpaths(datadir, "BRCA_exchange/BRCA_exchange_02-22-22.vcf.gz")
+    FLOSSIES_path = joinpaths(datadir, "FLOSSIES/FLOSSIES_25-03-2022.vcf.gz")
+    cancerhotspots_path = joinpaths(datadir, "cancerhotspots/cancerhotspots.v2.tsv")
+    #arup_brca_path = joinpaths(datadir, "ARUP/ARUP_BRCA_2022_04_01.vcf.gz")
+    tp53_db = joinpaths(datadir, "TP53_database/GermlineDownload_r20.normalized.vcf.gz")
+    hci_priors = joinpaths(datadir, "HCI_priors/priors.vcf.gz")
+    bayesdel = joinpaths(datadir, "BayesDEL/bayesdel_4.4.vcf.gz")
+    cosmic = joinpaths(datadir, "COSMIC/cosmic_cmc.vcf.gz")
+
+    # assays
+    cspec_brca_assays_splicing = joinpaths(datadir, "CSpec_BRCA_assays/splicing_assays.vcf.gz")
+    cspec_brca_assays_functional = joinpaths(datadir, "CSpec_BRCA_assays/functional_assays.vcf.gz")
+
+    # IGV data
+    igv_data_path = joinpaths(workdir, "src/frontend_celery/webapp/static/packages/igv/data")
+
     # clinvar submission
     clinvar_submission_schema = joinpaths(resources_dir, "clinvar_submission_schemas/clinvar_submission_schema_15_05_24.json")

src/frontend_celery/config.py

@@ -128,4 +128,6 @@ class LocaltestConfig(Config):
     PORT = os.environ.get('PORT')
     SERVER_NAME = HOST + ':' + PORT # add port to url_for()
 
-
+class DemoConfig(Config):
+    SESSION_COOKIE_SECURE = True
+    VCF_FILE_IMPORT_ACTIVE = True

src/frontend_celery/start_celery.sh

@@ -70,7 +70,7 @@ then
     celery -A celery_worker.celery worker --loglevel=info --concurrency=5 #--pool=solo
 fi
 
-if [ "${WEBAPP_ENV}" == "prod" ]
+if [ "${WEBAPP_ENV}" == "prod" ] || [ "${WEBAPP_ENV}" == "demo" ];
 then
     celery -A celery_worker.celery multi start single-worker --pidfile=$ROOT/run/celery/celery.pid --logfile=$ROOT/logs/celery/celery.log --loglevel=info -Ofair --concurrency=5 --without-heartbeat --without-gossip --without-mingle
 fi

src/frontend_celery/start_herediclass.sh

@@ -69,7 +69,7 @@ then
     python3 variant_classification/webservice.py --port $AUTOCLASS_PORT --host $AUTOCLASS_HOST
 fi
 
-if [ "${WEBAPP_ENV}" == "prod" ]
+if [ "${WEBAPP_ENV}" == "prod" ] || [ "${WEBAPP_ENV}" == "demo" ];
 then
     python3 variant_classification/webservice.py --port $AUTOCLASS_PORT --host $AUTOCLASS_HOST
 fi

src/frontend_celery/start_webapp.sh

@@ -58,7 +58,7 @@ then
    python3 main.py
 fi
 
-if [ "${WEBAPP_ENV}" == "prod" ]
+if [ "${WEBAPP_ENV}" == "prod"  ] || [ "${WEBAPP_ENV}" == "demo" ];
 then
    #export CURL_CA_BUNDLE=""
    set -o allexport

src/frontend_celery/webapp/variant/variant_functions.py

@@ -499,7 +499,7 @@ def create_variant_from_request(request_obj, user, conn):
                     result["flash_class"] = "alert-danger flash_id:variant_from_hgvs_error"
                     result["status"] = "error"
 
-    if create_variant_from == 'vcf_file' and current_app.config["vcf_file_import_active"]:
+    if create_variant_from == 'vcf_file' and current_app.config["VCF_FILE_IMPORT_ACTIVE"]:
         genome_build = request_obj.form.get('genome')
         if 'file' not in request_obj.files or genome_build is None:
             result["flash_message"] = 'You must specify the genome build and select a vcf file.'