bugfixes & testfixes

GC-HBOC · Oct 26, 2023 · 91f0909 · 91f0909
1 parent c39d682
commit 91f0909
Show file tree

Hide file tree

Showing 13 changed files with 196 additions and 290 deletions.
diff --git a/data/download_data.sh b/data/download_data.sh
@@ -456,7 +456,12 @@ rm $bayesdel_file/*.tbi
 
 python3 $tools/db_converter_bayesdel.py -i $bayesdel_file -o $bayesdel_file.vcf
 
-#$ngsbits/VcfCheck -lines 0 -in $bayesdel_file.vcf -ref $data/genomes/GRCh37.fa
+$ngsbits/VcfCheck -lines 0 -in $bayesdel_file.vcf -ref $data/genomes/GRCh37.fa > vcfcheck_errors.txt
+grep "^#" $bayesdel_file.vcf > vcfcheck_errors.vcf
+grep -v "^ERROR:" vcfcheck_errors.txt >> vcfcheck_errors.vcf
+
+python3 $tools/fix_bayesdel.py -i vcfcheck_errors.vcf -o vcfcheck_fixed.vcf -t $dbs/ensembl/Homo_sapiens.GRCh38.110.gff3
+$ngsbits/VcfCheck -lines 0 -in vcfcheck_fixed.vcf -ref $data/genomes/GRCh37.fa
 
 $ngsbits/VcfSort -in $bayesdel_file.vcf -out $bayesdel_file.vcf
 #$ngsbits/VcfLeftNormalize -in $bayesdel_file.vcf -stream -ref $data/genomes/GRCh37.fa -out $bayesdel_file.vcf.2

diff --git a/resources/backups/database_dumper/dump_database.sh b/resources/backups/database_dumper/dump_database.sh
@@ -15,11 +15,11 @@ DATE=$(date '+%F');
 path_to_structure=./structure/structure-$DATE.sql
 path_to_data=./data/data-$DATE.sql
 
-mysqldump --quick --column-statistics=0 HerediVar_ahdoebm1 -P 3306 -h SRV011.img.med.uni-tuebingen.de -u ahdoebm1 --no-tablespaces --no-data -r $path_to_structure -p
+mysqldump --quick --column-statistics=0 HerediVar_ahdoebm1 -P 3306 -h sql.img.med.uni-tuebingen.de -u ahdoebm1 --no-tablespaces --no-data -r $path_to_structure -p
 gzip $path_to_structure
 
 
-mysqldump --quick --column-statistics=0 HerediVar_ahdoebm1 -P 3306 -h SRV011.img.med.uni-tuebingen.de -u ahdoebm1 --no-tablespaces --no-create-info -r $path_to_data -p
+mysqldump --quick --column-statistics=0 HerediVar_ahdoebm1 -P 3306 -h sql.img.med.uni-tuebingen.de -u ahdoebm1 --no-tablespaces --no-create-info -r $path_to_data -p
 gzip $path_to_data
 
 

diff --git a/src/frontend_celery/tests/data/heredivar_test_data.sql b/src/frontend_celery/tests/data/heredivar_test_data.sql
diff --git a/src/frontend_celery/tests/data/variant_52.vcf b/src/frontend_celery/tests/data/variant_52.vcf
@@ -30,4 +30,4 @@
 ##INFO=<ID=user_classification,Number=1,Type=String,Description="The recent consensus classification by the VUS-task-force. Format: consensus_class|consensus_comment|submission_date|consensus_scheme|consensus_scheme_class|consensus_criteria_string. The consensus criteria string itself is a $ separated list with the Format: criterium_name+criterium_strength+criterium_evidence ">
 ##INFO=<ID=variation_id,Number=1,Type=Integer,Description="The ClinVar variation id of the variant.">
 #CHROM	POS	ID	REF	ALT	QUAL	FILTER	INFO
-chr13	32314943	52	A	G	.	.	rsid=7988901;phylop_100way=-0.021;cadd_scaled=7.930;spliceai_details=ZAR1L%7C0.00%7C0.00%7C0.02%7C0.00%7C31%7C-39%7C-44%7C32;spliceai_max_delta=0.02;gnomad_ac=5753;gnomad_af=0.0378129;gnomad_hom=353;gnomad_het=5047;gnomad_popmax=afr;gnomad_popmax_AF=0.131678;brca_exchange_clinical_significance=Benign_/_Little_Clinical_Significance;task_force_protein_domain=C-terminal_RAD51_binding_domain_(inkl._NLS1_und_BRC-)_;task_force_protein_domain_source=BWRL/ENIGMA;hexplorer=-5.12;hexplorer_mut=-4.84;hexplorer_wt=0.28;hexplorer_rev=4.56;hexplorer_rev_mut=-1.44;hexplorer_rev_wt=-6.00;user_classification=3|this_is_a_comment_with\tpecial_characters|2022-10-06_06:00:00|ACMG_standard|-|ps1+supporting_pathogenic+evidence1$ps2+strong_pathogenic+evidence2$pp1+strong_pathogenic+evidence3|;consequences=ENST00000533490|c.-390%2B372T>C|None|intron_variant|modifier|None|1|ZAR1L|None|1|1|0|1&ENST00000345108|None|None|upstream_gene_variant|modifier|None|None|ZAR1L|None|1|0|0|0&ENST00000544455|None|None|upstream_gene_variant|modifier|None|None|BRCA2|None|1|0|0|0&ENST00000530893|None|None|upstream_gene_variant|modifier|None|None|BRCA2|None|0|0|0|0&ENST00000380152|None|None|upstream_gene_variant|modifier|None|None|BRCA2|None|1|1|0|1&ENST00000680887|None|None|upstream_gene_variant|modifier|None|None|BRCA2|None|0|0|0|0&ENST00000614259|None|None|upstream_gene_variant|modifier|None|None|BRCA2|None|0|0|0|0&NM_001136571|c.-390%2B372T>C|None|intron_variant|modifier|None|1|ZAR1L|None|1|1|0|1&NM_000059|None|None|upstream_gene_variant|modifier|None|None|BRCA2|None|1|1|0|1&ENSR00000060894|None|None|regulatory_region_variant|modifier|None|None|None|None|None|None|None|None&ENSM00526233310|None|None|TF_binding_site_variant|modifier|None|None|None|None|None|None|None|None&ENSM00522614247|None|None|TF_binding_site_variant|modifier|None|None|None|None|None|None|None|None;assays=2022-08-08|functional|12345.0&2022-08-09|functional|6565.0&2022-08-09|splicing|3654.0;literature=9425226&15342711&16333312&25994375;consensus_classification=5|A_random_test_classification|2022-09-07_03:00:00|ACMG_standard|3|ps1+medium_pathogenic+ev1$ps2+strong_pathogenic+ev2$ps1+supporting_pathogenic+ev3|;variation_id=209597;clinvar_summary=reviewed_by_expert_panel:Benign;clinvar_submissions=Benign|2015-01-12|reviewed_by_expert_panel|C2675520:Breast-ovarian_cancer,_familial_2|Evidence-based_Network_for_the_Interpretation_of_Germline_Mutant_Alleles_(ENIGMA)|description:_Class_1_not_pathogenic_based_on_frequency_>1%_in_an_outbred_sampleset._Frequency_0.14_(African),_derived_from_1000_genomes_(2012-04-30).
+chr13	32314943	52	A	G	.	.	rsid=7988901;phylop_100way=-0.021;cadd_scaled=7.930;spliceai_details=ZAR1L%7C0.00%7C0.00%7C0.02%7C0.00%7C31%7C-39%7C-44%7C32;spliceai_max_delta=0.02;gnomad_ac=5753;gnomad_af=0.0378129;gnomad_hom=353;gnomad_het=5047;gnomad_popmax=afr;gnomad_popmax_AF=0.131678;brca_exchange_clinical_significance=Benign_/_Little_Clinical_Significance;task_force_protein_domain=C-terminal_RAD51_binding_domain_(inkl._NLS1_und_BRC-)_;task_force_protein_domain_source=BWRL/ENIGMA;hexplorer=-5.12;hexplorer_mut=-4.84;hexplorer_wt=0.28;hexplorer_rev=4.56;hexplorer_rev_mut=-1.44;hexplorer_rev_wt=-6.00;user_classification=3|this_is_a_comment_with\tpecial_characters|2022-10-06_06:00:00|ACMG_standard|-|PS1+very_strong_pathogenic+evidence1$PS2+strong_pathogenic+evidence2$PP1+very_strong_pathogenic+evidence3|;consequences=ENST00000533490|c.-390%2B372T>C|None|intron_variant|modifier|None|1|ZAR1L|None|1|1|0|1&ENST00000345108|None|None|upstream_gene_variant|modifier|None|None|ZAR1L|None|1|0|0|0&ENST00000544455|None|None|upstream_gene_variant|modifier|None|None|BRCA2|None|1|0|0|0&ENST00000530893|None|None|upstream_gene_variant|modifier|None|None|BRCA2|None|0|0|0|0&ENST00000380152|None|None|upstream_gene_variant|modifier|None|None|BRCA2|None|1|1|0|1&ENST00000680887|None|None|upstream_gene_variant|modifier|None|None|BRCA2|None|0|0|0|0&ENST00000614259|None|None|upstream_gene_variant|modifier|None|None|BRCA2|None|0|0|0|0&NM_001136571|c.-390%2B372T>C|None|intron_variant|modifier|None|1|ZAR1L|None|1|1|0|1&NM_000059|None|None|upstream_gene_variant|modifier|None|None|BRCA2|None|1|1|0|1&ENSR00000060894|None|None|regulatory_region_variant|modifier|None|None|None|None|None|None|None|None&ENSM00526233310|None|None|TF_binding_site_variant|modifier|None|None|None|None|None|None|None|None&ENSM00522614247|None|None|TF_binding_site_variant|modifier|None|None|None|None|None|None|None|None;assays=2022-08-08|functional|12345.0&2022-08-09|functional|6565.0&2022-08-09|splicing|3654.0;literature=9425226&15342711&16333312&25994375;consensus_classification=5|A_random_test_classification|2022-09-07_03:00:00|ACMG_standard|3|PS1+supporting_pathogenic+ev1$PS2+strong_pathogenic+ev2$PS1+very_strong_pathogenic+ev3|;variation_id=209597;clinvar_summary=reviewed_by_expert_panel:Benign;clinvar_submissions=Benign|2015-01-12|reviewed_by_expert_panel|C2675520:Breast-ovarian_cancer,_familial_2|Evidence-based_Network_for_the_Interpretation_of_Germline_Mutant_Alleles_(ENIGMA)|description:_Class_1_not_pathogenic_based_on_frequency_>1%_in_an_outbred_sampleset._Frequency_0.14_(African),_derived_from_1000_genomes_(2012-04-30).
diff --git a/src/frontend_celery/tests/functional/test_dev.py b/src/frontend_celery/tests/functional/test_dev.py
@@ -19,170 +19,30 @@ def get_all_links(html_data):
     return links
 
 
+def compare_vcf(reference_file, vcf_string):
+    for line in reference_file:
+        line = line.strip()
+        if line == '':
+            continue
+        if line.startswith('#'): 
+            if not line.startswith('##fileDate'): # skip the filedate header line because this one changes daily, but the test data is not updated daily
+                assert line in vcf_string # test that header line is there
+            continue
+
+        parts = line.split('\t')
+        info = parts[7]
+
+        assert info[0:7].join('\t') in vcf_string # test that variant is there
+
+        for info_entry in info.split(';'):
+            print(info_entry)
+            if 'consequences' in info_entry:
+                for consequence in info_entry.strip('consequences=').split('&'):
+                    assert consequence in vcf_string
+            else:
+                #if (info_entry not in vcf_string):
+                #    print(info_entry)
+                #    print(vcf_string)
+                assert info_entry.strip() in vcf_string # test that info is there
 
 
-def test_list_actions(test_client):
-    """
-    This first creates a new user list for the testuser and subsequentially adds variants, deletes, renames and searches them
-    """
-
-    ##### test that endpoint is reachable & no lists contained #####
-    response = test_client.get(url_for("user.my_lists"), follow_redirects=True)
-    data = html.unescape(response.data.decode('utf8'))
-
-    assert response.status_code == 200
-    assert data.count('name="user_list_row"') == 2
-    assert 'Please select a list to view its content!' in data
-
-
-    ##### create two new lists #####
-    response = test_client.post(
-        url_for("user.my_lists", type='create'), 
-        data={
-            "list_name": "list one"
-        },
-        follow_redirects=True
-    )
-    data = html.unescape(response.data.decode('utf8'))
-
-    assert response.status_code == 200
-    assert 'Successfully created new list' in data
-
-
-    conn = Connection(['super_user'])
-    user_lists = conn.get_lists_for_user(session['user']['user_id'])
-    assert len(user_lists) == 3
-    list_one_id = str(conn.get_latest_list_id())
-    conn.close()
-
-
-    response = test_client.post(
-        url_for("user.my_lists", type='create'), 
-        data={
-            "list_name": "list two"
-        },
-        follow_redirects=True
-    )
-    data = html.unescape(response.data.decode('utf8'))
-
-    assert response.status_code == 200
-    assert 'Successfully created new list' in data
-
-    conn = Connection(['super_user'])
-    user_lists = conn.get_lists_for_user(session['user']['user_id'])
-    assert len(user_lists) == 4
-    list_two_id = str(conn.get_latest_list_id())
-    conn.close()
-
-    ##### insert variants using bulk import #####
-    response = test_client.post(    
-        url_for("variant.search", selected_list_id = list_one_id, selected_variants = "", select_all_variants = "true", genes = "BARD1;BRCA2"), 
-        follow_redirects=True
-    )
-    data = html.unescape(response.data.decode('utf8'))
-    with open("/mnt/storage2/users/ahdoebm1/HerediVar/test.txt", "w") as testdoc:
-        testdoc.write(data)
-
-    assert response.status_code == 200
-    assert "Successfully inserted 2" in data
-
-    response = test_client.post(    
-        url_for("variant.search", selected_list_id = list_two_id, selected_variants = "", select_all_variants = "true", genes = "BRCA1;BRCA2"), 
-        follow_redirects=True
-    )
-    data = html.unescape(response.data.decode('utf8'))
-
-    assert response.status_code == 200
-    assert "Successfully inserted 2" in data
-
-    ##### duplicate a list #####
-    response = test_client.post(
-        url_for("user.my_lists", type='duplicate'), 
-        data={
-            "list_name": "duplicate of list one",
-            "public_read": "true",
-            "public_edit": "false",
-            "list_id": list_one_id
-        },
-        follow_redirects=True
-    )
-    data = html.unescape(response.data.decode('utf8'))
-
-    assert response.status_code == 200
-    assert "Successfully duplicated list" in data
-
-    # check that all variants have been duplicated properly
-    conn = Connection(['super_user'])
-    duplicated_list_id = conn.get_latest_list_id()
-    variants_in_duplicated_list = conn.get_variant_ids_from_list(duplicated_list_id)
-    list_one_variants = conn.get_variant_ids_from_list(list_one_id)
-    intersection = set(list_one_variants).intersection(set(variants_in_duplicated_list))
-    assert len(list_one_variants) == len(intersection) == len(variants_in_duplicated_list)
-    conn.close()
-
-    ##### intersect a list #####
-    response = test_client.post(
-        url_for("user.my_lists", type='intersect'), 
-        data={
-            "list_name": "intersection test",
-            "list_id": list_one_id,
-            "other_list_id": list_two_id,
-            "other_list_name": "",
-        },
-        follow_redirects=True
-    )
-    data = html.unescape(response.data.decode('utf8'))
-
-    assert response.status_code == 200
-
-    conn = Connection(['super_user'])
-    intersected_list_id = conn.get_latest_list_id()
-    variants_in_duplicated_list = conn.get_variant_ids_from_list(intersected_list_id)
-    assert variants_in_duplicated_list == ['52']
-    conn.close()
-
-    ##### subtract a list #####
-    response = test_client.post(
-        url_for("user.my_lists", type='subtract'), 
-        data={
-            "list_name": "subtraction test",
-            "list_id": list_one_id,
-            "other_list_id": list_two_id,
-            "other_list_name": "",
-        },
-        follow_redirects=True
-    )
-    data = html.unescape(response.data.decode('utf8'))
-
-    assert response.status_code == 200
-
-    conn = Connection(['super_user'])
-    subtracted_list_id = conn.get_latest_list_id()
-    variants_in_duplicated_list = conn.get_variant_ids_from_list(subtracted_list_id)
-    assert variants_in_duplicated_list == ['15']
-    conn.close()
-
-
-    ##### add a list #####
-    response = test_client.post(
-        url_for("user.my_lists", type='add'), 
-        data={
-            "list_name": "add test",
-            "list_id": list_one_id,
-            "other_list_id": list_two_id,
-            "other_list_name": "",
-        },
-        follow_redirects=True
-    )
-    data = html.unescape(response.data.decode('utf8'))
-
-    assert response.status_code == 200
-
-    conn = Connection(['super_user'])
-    subtracted_list_id = conn.get_latest_list_id()
-    variants_in_duplicated_list = conn.get_variant_ids_from_list(subtracted_list_id)
-    variant_ids_template = ['15', '52', '71']
-    for variant_id in variant_ids_template:
-        assert variant_id in variants_in_duplicated_list
-    assert len(variants_in_duplicated_list) == len(variant_ids_template)
-    conn.close()
diff --git a/src/frontend_celery/tests/functional/test_user_routs.py b/src/frontend_celery/tests/functional/test_user_routs.py
@@ -282,51 +282,6 @@ def test_user_lists(test_client):
     assert list_oi[4] == 1
     conn.close()
 
-def test_list_actions(test_client):
-    ##### create two new list #####
-    response = test_client.post(
-        url_for("user.my_lists", type='create'), 
-        data={
-            "list_name": "list one"
-        },
-        follow_redirects=True
-    )
-    data = html.unescape(response.data.decode('utf8'))
-
-    assert response.status_code == 200
-    assert 'Successfully created new list' in data
-
-    conn = Connection(['super_user'])
-    list_id_one = str(conn.get_latest_list_id())
-    conn.close()
-
-    response = test_client.post(
-        url_for("user.my_lists", type='create'), 
-        data={
-            "list_name": "list two"
-        },
-        follow_redirects=True
-    )
-    data = html.unescape(response.data.decode('utf8'))
-
-    assert response.status_code == 200
-    assert 'Successfully created new list' in data
-
-    conn = Connection(['super_user'])
-    list_id_two = str(conn.get_latest_list_id())
-    conn.close()
-
-    ##### add some variants to it using bulk insert #####
-    response = test_client.post(
-        url_for("varaint.search", type='create'), 
-        data={
-            "selected_list_id": list_id_one,
-            "genes": "BARD1"
-        },
-        follow_redirects=True
-    )
-    data = html.unescape(response.data.decode('utf8'))
-
 
 def test_list_actions(test_client):
     """
@@ -338,7 +293,7 @@ def test_list_actions(test_client):
     data = html.unescape(response.data.decode('utf8'))
 
     assert response.status_code == 200
-    assert data.count('name="user_list_row"') == 2
+    assert data.count('name="user_list_row"') == 3
     assert 'Please select a list to view its content!' in data
 
 
@@ -358,7 +313,7 @@ def test_list_actions(test_client):
 
     conn = Connection(['super_user'])
     user_lists = conn.get_lists_for_user(session['user']['user_id'])
-    assert len(user_lists) == 3
+    assert len(user_lists) == 4
     list_one_id = str(conn.get_latest_list_id())
     conn.close()
 
@@ -377,7 +332,7 @@ def test_list_actions(test_client):
 
     conn = Connection(['super_user'])
     user_lists = conn.get_lists_for_user(session['user']['user_id'])
-    assert len(user_lists) == 4
+    assert len(user_lists) == 5
     list_two_id = str(conn.get_latest_list_id())
     conn.close()
 

diff --git a/src/frontend_celery/tests/functional/test_variant_routes.py b/src/frontend_celery/tests/functional/test_variant_routes.py
@@ -276,7 +276,8 @@ def test_classify(test_client):
         follow_redirects=True
     )
     data = html.unescape(response.data.decode('utf8'))
-    #print(data)
+    #with open("/mnt/storage2/users/ahdoebm1/HerediVar/src/frontend_celery/tests/functional/test.html", 'w') as file:
+    #    file.write(data)
     assert response.status_code == 200
     assert "Successfully inserted new user classification" in data
 
@@ -298,7 +299,7 @@ def test_classify(test_client):
     )
     data = html.unescape(response.data.decode('utf8'))
     assert response.status_code == 200
-    assert "A mutually exclusive criterium to pp1 was selected." in data
+    assert "A mutually exclusive criterium to PP1 was selected." in data
 
     ##### test posting none as classification scheme #####
     response = test_client.post(
@@ -457,7 +458,7 @@ def test_create_variant(test_client):
     )
     data = html.unescape(response.data.decode('utf8'))
 
-    print(data)
+    #print(data)
 
     assert response.status_code == 200
     assert "Successfully inserted variant" in data
@@ -484,6 +485,7 @@ def test_create_variant(test_client):
         content_type='multipart/form-data'
     )
     data = html.unescape(response.data.decode('utf8'))
+    print(data)
     assert response.status_code == 200
     assert "Variant not imported: already in database!" in data
 
@@ -532,6 +534,8 @@ def test_create_variant(test_client):
 
 
 
+
+
 def test_create_variant_from_grch37(test_client):
     response = test_client.get(url_for("variant.create"), follow_redirects=True)
     assert response.status_code == 200

diff --git a/src/frontend_celery/tests/script/run_local_tests.sh b/src/frontend_celery/tests/script/run_local_tests.sh
@@ -16,18 +16,18 @@ MOST_RECENT_DUMP_DATE=$(cat $DATABASE_DUMPER_DIR/most_recent_dump.txt)
 source .venv/bin/activate
 
 
-zcat $DATABASE_DUMPER_DIR/structure/structure-$MOST_RECENT_DUMP_DATE.sql.gz | mysql --host SRV011.img.med.uni-tuebingen.de -uahdoebm1 -p20220303 HerediVar_ahdoebm1_test
-mysql --host SRV011.img.med.uni-tuebingen.de -uahdoebm1 -p20220303 HerediVar_ahdoebm1_test < $TEST_DATA_DIR/heredivar_test_data.sql
-#mysql --host SRV011.img.med.uni-tuebingen.de -uahdoebm1 -p20220303 HerediVar_ahdoebm1_test < $DATABASE_DUMPER_DIR/users/user_privileges_test.sql
-#mysql --host SRV011.img.med.uni-tuebingen.de -uahdoebm1 -p20220303 HerediVar_ahdoebm1_test -e "SHOW TABLES"
+zcat $DATABASE_DUMPER_DIR/structure/structure-$MOST_RECENT_DUMP_DATE.sql.gz | mysql --host sql.img.med.uni-tuebingen.de -uahdoebm1 -p20220303 HerediVar_ahdoebm1_test
+mysql --host sql.img.med.uni-tuebingen.de -uahdoebm1 -p20220303 HerediVar_ahdoebm1_test < $TEST_DATA_DIR/heredivar_test_data.sql
+#mysql --host sql.img.med.uni-tuebingen.de -uahdoebm1 -p20220303 HerediVar_ahdoebm1_test < $DATABASE_DUMPER_DIR/users/user_privileges_test.sql
+#mysql --host sql.img.med.uni-tuebingen.de -uahdoebm1 -p20220303 HerediVar_ahdoebm1_test -e "SHOW TABLES"
 
 
 # start keycloak
 $TEST_SCRIPT_DIR/start_keycloak_for_tests.sh
 
 # run tests
 cd src/frontend_celery
-python -m pytest -k 'test_dev'
+python -m pytest #-k 'test_dev'
 
 # stop keycloak
 pkill -s 0 -e java