bugfixes

resources/backups/database_dumper/update_database.sql

@@ -1,37 +1,37 @@
-DROP TABLE IF EXISTS `HerediVar`.`import_variant_queue`;
+-- DROP TABLE IF EXISTS `HerediVar`.`import_variant_queue`;
 
-DROP TABLE IF EXISTS `HerediVar`.`import_queue`;
-CREATE TABLE `HerediVar`.`import_queue` (
-  `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
-  `user_id` int(10) unsigned NOT NULL,
-  `requested_at` datetime NOT NULL DEFAULT current_timestamp(),
-  `status` enum('pending','progress','success','error','retry') NOT NULL DEFAULT 'pending',
-  `finished_at` datetime DEFAULT NULL,
-  `message` text DEFAULT '',
-  `celery_task_id` text DEFAULT NULL,
-  PRIMARY KEY (`id`),
-  KEY `FK_import_queue_user_id_idx` (`user_id`),
-  CONSTRAINT `FK_import_queue_user_id` FOREIGN KEY (`user_id`) REFERENCES `user` (`id`) ON DELETE NO ACTION ON UPDATE NO ACTION
-) ENGINE=InnoDB AUTO_INCREMENT=281 DEFAULT CHARSET=utf8 COLLATE=utf8_general_ci;
+-- DROP TABLE IF EXISTS `HerediVar`.`import_queue`;
+-- CREATE TABLE `HerediVar`.`import_queue` (
+--   `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
+--   `user_id` int(10) unsigned NOT NULL,
+--   `requested_at` datetime NOT NULL DEFAULT current_timestamp(),
+--   `status` enum('pending','progress','success','error','retry') NOT NULL DEFAULT 'pending',
+--   `finished_at` datetime DEFAULT NULL,
+--   `message` text DEFAULT '',
+--   `celery_task_id` text DEFAULT NULL,
+--   PRIMARY KEY (`id`),
+--   KEY `FK_import_queue_user_id_idx` (`user_id`),
+--   CONSTRAINT `FK_import_queue_user_id` FOREIGN KEY (`user_id`) REFERENCES `user` (`id`) ON DELETE NO ACTION ON UPDATE NO ACTION
+-- ) ENGINE=InnoDB AUTO_INCREMENT=281 DEFAULT CHARSET=utf8 COLLATE=utf8_general_ci;
 
 
 
 
-DROP TABLE IF EXISTS `HerediVar`.`import_variant_queue`;
-CREATE TABLE `HerediVar`.`import_variant_queue` (
-  `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
-  `import_queue_id` int(10) unsigned DEFAULT NULL,
-  `status` enum('pending','success','error','progress','deleted','update','retry') NOT NULL DEFAULT 'pending',
-  `requested_at` datetime NOT NULL DEFAULT current_timestamp(),
-  `finished_at` datetime DEFAULT NULL,
-  `message` text DEFAULT '',
-  `celery_task_id` varchar(45) DEFAULT NULL,
-  `vid` text NOT NULL,
-  PRIMARY KEY (`id`),
-  UNIQUE KEY `id_UNIQUE` (`id`),
-  KEY `fk_import_variant_queue_import_queue_idx` (`import_queue_id`),
-  CONSTRAINT `fk_import_variant_queue_import_queue` FOREIGN KEY (`import_queue_id`) REFERENCES `import_queue` (`id`) ON DELETE CASCADE ON UPDATE NO ACTION
-) ENGINE=InnoDB AUTO_INCREMENT=42036 DEFAULT CHARSET=utf8 COLLATE=utf8_general_ci;
+-- DROP TABLE IF EXISTS `HerediVar`.`import_variant_queue`;
+-- CREATE TABLE `HerediVar`.`import_variant_queue` (
+--   `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
+--   `import_queue_id` int(10) unsigned DEFAULT NULL,
+--   `status` enum('pending','success','error','progress','deleted','update','retry') NOT NULL DEFAULT 'pending',
+--   `requested_at` datetime NOT NULL DEFAULT current_timestamp(),
+--   `finished_at` datetime DEFAULT NULL,
+--   `message` text DEFAULT '',
+--   `celery_task_id` varchar(45) DEFAULT NULL,
+--   `vid` text NOT NULL,
+--   PRIMARY KEY (`id`),
+--   UNIQUE KEY `id_UNIQUE` (`id`),
+--   KEY `fk_import_variant_queue_import_queue_idx` (`import_queue_id`),
+--   CONSTRAINT `fk_import_variant_queue_import_queue` FOREIGN KEY (`import_queue_id`) REFERENCES `import_queue` (`id`) ON DELETE CASCADE ON UPDATE NO ACTION
+-- ) ENGINE=InnoDB AUTO_INCREMENT=42036 DEFAULT CHARSET=utf8 COLLATE=utf8_general_ci;
 -- 
 -- ALTER TABLE `HerediVar`.`classification_criterium` 
 -- CHANGE COLUMN `relevant_info` `relevant_info` TEXT NOT NULL DEFAULT '' ;
@@ -50,16 +50,20 @@ CREATE TABLE `HerediVar`.`import_variant_queue` (
 -- 
 -- 
 -- 
--- CREATE TABLE `HerediVar`.`variant_heredicare_annotation` (
---   `id` INT UNSIGNED NOT NULL AUTO_INCREMENT,
---   `variant_id` INT UNSIGNED NOT NULL,
---   `vid` VARCHAR(45) NOT NULL,
---   `n_fam` INT NOT NULL DEFAULT 0 COMMENT 'consensus class: 1:pathogen, 2: vus, 3: polymorphismus/neutral, 11: class1, 12: class2, 32: class3-, 13: class3, 34: class3+, 14: class4, 15: class5, 20: artefakt, 21: nicht klassifiziert, 4: unbekannt',
---   `n_pat` INT NOT NULL DEFAULT 0,
---   `consensus_class` ENUM('1', '2', '3', '11', '12', '32', '13', '34', '14', '15', '20', '21', '4') NOT NULL,
---   `comment` TEXT NOT NULL,
---   PRIMARY KEY (`id`),
---   UNIQUE INDEX `id_UNIQUE` (`id` ASC));
+DROP TABLE IF EXISTS `HerediVar`.`variant_heredicare_annotation`;
+CREATE TABLE `HerediVar`.`variant_heredicare_annotation` (
+  `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
+  `variant_id` int(10) unsigned NOT NULL,
+  `vid` varchar(45) NOT NULL,
+  `n_fam` int(11) NOT NULL DEFAULT 0 COMMENT 'consensus class: 1:pathogen, 2: vus, 3: polymorphismus/neutral, 11: class1, 12: class2, 32: class3-, 13: class3, 34: class3+, 14: class4, 15: class5, 20: artefakt, 21: nicht klassifiziert, 4: unbekannt',
+  `n_pat` int(11) NOT NULL DEFAULT 0,
+  `consensus_class` enum('1','2','3','11','12','32','13','34','14','15','20','21','4') DEFAULT NULL,
+  `comment` text DEFAULT NULL,
+  `date` date DEFAULT NULL,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `id_UNIQUE` (`id`)
+) ENGINE=InnoDB AUTO_INCREMENT=0 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
+
 -- 
 -- ALTER TABLE `HerediVar`.`variant_heredicare_annotation` 
 -- ADD COLUMN `date` DATE NULL AFTER `comment`;

src/annotation_service/annotation_jobs/spliceai_job.py

@@ -75,8 +75,10 @@ def annotate_missing_spliceai(self, input_vcf_path, output_vcf_path):
             #returncode, stderr, stdout = functions.execute_command(["ls", "-l", "/tmp"], "ls")
             #print(stdout)
             spliceai_code, spliceai_stderr, spliceai_stdout = self.annotate_spliceai_algorithm(input_vcf_path, output_vcf_path)
-            if 'ERROR' in spliceai_stderr:
+            if 'SpliceAI runtime ERROR:' in spliceai_stderr:
                 errors.append(spliceai_stderr)
+            if 'Skipping record' in spliceai_stderr:
+                errors.append("SpliceAI WARNING skipping: " + functions.find_between(spliceai_stderr, 'WARNING:', ': chr'))
 
         # need to insert some code here to merge the newly annotated variants and previously 
         # annotated ones from the db if there are files which contain more than one variant! 
@@ -93,11 +95,12 @@ def annotate_spliceai_algorithm(self, input_vcf_path, output_vcf_path):
         input_vcf_zipped_path = input_vcf_path + ".gz"
 
         # gbzip and index the input file as this is required for spliceai...
-        returncode, stderr, stdout = functions.execute_command([os.path.join(paths.htslib_path, 'bgzip'), '-f', input_vcf_path], 'bgzip')
+        returncode, stderr, stdout = functions.execute_command([os.path.join(paths.htslib_path, 'bgzip'), '-f', '-k', input_vcf_path], 'bgzip')
         if returncode != 0:
             return returncode, stderr, stdout
         returncode, stderr, stdout = functions.execute_command([os.path.join(paths.htslib_path, 'tabix'), "-f", "-p", "vcf", input_vcf_zipped_path], 'tabix')
         if returncode != 0:
+            functions.rm(input_vcf_zipped_path)
             return returncode, stderr, stdout
 
         # execute spliceai

src/annotation_service/main.py

@@ -10,6 +10,7 @@
 from .annotation_jobs import *
 from .annotation_jobs import litvar2_job
 import random
+from mysql.connector import Error, InternalError
 
 import os
 
@@ -217,6 +218,16 @@ def process_one_request(annotation_queue_id, job_config = get_default_job_config
         conn.close()
         #raise e #HTTPError(url = e.url, code = e.code, msg = "A HTTP error occured", hdrs = e.hdrs, fp = e.fp)
         return status, runtime_error
+    except InternalError as e:
+        # deadlock: code 1213
+        status = "retry"
+        conn.update_annotation_queue(row_id=annotation_queue_id, status=status, error_msg=str(e))
+        message = "Attempting retry because of database error: " + str(e)  + ' ' + traceback.format_exc()
+
+        if exists(vcf_path): 
+            os.remove(vcf_path)
+        conn.close()
+        return status, message
     except Exception as e:
         print("An exception occured: " + str(e))
         print(traceback.format_exc())

src/frontend_celery/start_celery.sh

@@ -8,16 +8,18 @@ set -o pipefail
 helpFunction()
 {
    echo ""
-   echo "Usage: $0 -w env"
+   echo "Usage: $0 -w env -h path"
    echo "This script starts the heredivar frontend gunicorn or development server"
    echo -e "\t-w Provide 'dev' for development server and 'prod' for production gunicorn server."
+   echo -e "\t-h The path to the users home directory."
    exit 1 # Exit script after printing help
 }
 
 while getopts "w:" opt
 do
    case "$opt" in
       w ) we="$OPTARG" ;;
+      h ) localhome="$OPTARG" ;;
       ? ) helpFunction ;; # Print helpFunction in case parameter is non-existent
    esac
 done
@@ -29,6 +31,12 @@ then
    helpFunction
 fi
 
+# set home for production environment. In systemd $HOME is not available but required for VEP
+if [ -n "$localhome"]
+then
+   export HOME=$localhome
+fi
+
 echo "preparing celery startup"
 
 

src/frontend_celery/webapp/tasks.py

@@ -448,7 +448,7 @@ def map_hg38(variant, user_id, conn:Connection, insert_variant = True, perform_a
                 #print(err_msg)
 
                 if 'unequal' in err_msg:
-                    if new_message not in message:
+                    if err_msg not in message:
                         message = functions.collect_info(message, "hgvs_msg=", err_msg, sep = " ~~ ")
                 elif err_msg != '':
                     preferred_transcripts = conn.get_preferred_transcripts(gene_id, return_all = True)