diff --git a/resources/backups/database_dumper/update_database.sql b/resources/backups/database_dumper/update_database.sql index 05aacf5d..f495fce9 100644 --- a/resources/backups/database_dumper/update_database.sql +++ b/resources/backups/database_dumper/update_database.sql @@ -1,37 +1,37 @@ -DROP TABLE IF EXISTS `HerediVar`.`import_variant_queue`; +-- DROP TABLE IF EXISTS `HerediVar`.`import_variant_queue`; -DROP TABLE IF EXISTS `HerediVar`.`import_queue`; -CREATE TABLE `HerediVar`.`import_queue` ( - `id` int(10) unsigned NOT NULL AUTO_INCREMENT, - `user_id` int(10) unsigned NOT NULL, - `requested_at` datetime NOT NULL DEFAULT current_timestamp(), - `status` enum('pending','progress','success','error','retry') NOT NULL DEFAULT 'pending', - `finished_at` datetime DEFAULT NULL, - `message` text DEFAULT '', - `celery_task_id` text DEFAULT NULL, - PRIMARY KEY (`id`), - KEY `FK_import_queue_user_id_idx` (`user_id`), - CONSTRAINT `FK_import_queue_user_id` FOREIGN KEY (`user_id`) REFERENCES `user` (`id`) ON DELETE NO ACTION ON UPDATE NO ACTION -) ENGINE=InnoDB AUTO_INCREMENT=281 DEFAULT CHARSET=utf8 COLLATE=utf8_general_ci; +-- DROP TABLE IF EXISTS `HerediVar`.`import_queue`; +-- CREATE TABLE `HerediVar`.`import_queue` ( +-- `id` int(10) unsigned NOT NULL AUTO_INCREMENT, +-- `user_id` int(10) unsigned NOT NULL, +-- `requested_at` datetime NOT NULL DEFAULT current_timestamp(), +-- `status` enum('pending','progress','success','error','retry') NOT NULL DEFAULT 'pending', +-- `finished_at` datetime DEFAULT NULL, +-- `message` text DEFAULT '', +-- `celery_task_id` text DEFAULT NULL, +-- PRIMARY KEY (`id`), +-- KEY `FK_import_queue_user_id_idx` (`user_id`), +-- CONSTRAINT `FK_import_queue_user_id` FOREIGN KEY (`user_id`) REFERENCES `user` (`id`) ON DELETE NO ACTION ON UPDATE NO ACTION +-- ) ENGINE=InnoDB AUTO_INCREMENT=281 DEFAULT CHARSET=utf8 COLLATE=utf8_general_ci; -DROP TABLE IF EXISTS `HerediVar`.`import_variant_queue`; -CREATE TABLE `HerediVar`.`import_variant_queue` ( - `id` int(10) unsigned NOT NULL AUTO_INCREMENT, - `import_queue_id` int(10) unsigned DEFAULT NULL, - `status` enum('pending','success','error','progress','deleted','update','retry') NOT NULL DEFAULT 'pending', - `requested_at` datetime NOT NULL DEFAULT current_timestamp(), - `finished_at` datetime DEFAULT NULL, - `message` text DEFAULT '', - `celery_task_id` varchar(45) DEFAULT NULL, - `vid` text NOT NULL, - PRIMARY KEY (`id`), - UNIQUE KEY `id_UNIQUE` (`id`), - KEY `fk_import_variant_queue_import_queue_idx` (`import_queue_id`), - CONSTRAINT `fk_import_variant_queue_import_queue` FOREIGN KEY (`import_queue_id`) REFERENCES `import_queue` (`id`) ON DELETE CASCADE ON UPDATE NO ACTION -) ENGINE=InnoDB AUTO_INCREMENT=42036 DEFAULT CHARSET=utf8 COLLATE=utf8_general_ci; +-- DROP TABLE IF EXISTS `HerediVar`.`import_variant_queue`; +-- CREATE TABLE `HerediVar`.`import_variant_queue` ( +-- `id` int(10) unsigned NOT NULL AUTO_INCREMENT, +-- `import_queue_id` int(10) unsigned DEFAULT NULL, +-- `status` enum('pending','success','error','progress','deleted','update','retry') NOT NULL DEFAULT 'pending', +-- `requested_at` datetime NOT NULL DEFAULT current_timestamp(), +-- `finished_at` datetime DEFAULT NULL, +-- `message` text DEFAULT '', +-- `celery_task_id` varchar(45) DEFAULT NULL, +-- `vid` text NOT NULL, +-- PRIMARY KEY (`id`), +-- UNIQUE KEY `id_UNIQUE` (`id`), +-- KEY `fk_import_variant_queue_import_queue_idx` (`import_queue_id`), +-- CONSTRAINT `fk_import_variant_queue_import_queue` FOREIGN KEY (`import_queue_id`) REFERENCES `import_queue` (`id`) ON DELETE CASCADE ON UPDATE NO ACTION +-- ) ENGINE=InnoDB AUTO_INCREMENT=42036 DEFAULT CHARSET=utf8 COLLATE=utf8_general_ci; -- -- ALTER TABLE `HerediVar`.`classification_criterium` -- CHANGE COLUMN `relevant_info` `relevant_info` TEXT NOT NULL DEFAULT '' ; @@ -50,16 +50,20 @@ CREATE TABLE `HerediVar`.`import_variant_queue` ( -- -- -- --- CREATE TABLE `HerediVar`.`variant_heredicare_annotation` ( --- `id` INT UNSIGNED NOT NULL AUTO_INCREMENT, --- `variant_id` INT UNSIGNED NOT NULL, --- `vid` VARCHAR(45) NOT NULL, --- `n_fam` INT NOT NULL DEFAULT 0 COMMENT 'consensus class: 1:pathogen, 2: vus, 3: polymorphismus/neutral, 11: class1, 12: class2, 32: class3-, 13: class3, 34: class3+, 14: class4, 15: class5, 20: artefakt, 21: nicht klassifiziert, 4: unbekannt', --- `n_pat` INT NOT NULL DEFAULT 0, --- `consensus_class` ENUM('1', '2', '3', '11', '12', '32', '13', '34', '14', '15', '20', '21', '4') NOT NULL, --- `comment` TEXT NOT NULL, --- PRIMARY KEY (`id`), --- UNIQUE INDEX `id_UNIQUE` (`id` ASC)); +DROP TABLE IF EXISTS `HerediVar`.`variant_heredicare_annotation`; +CREATE TABLE `HerediVar`.`variant_heredicare_annotation` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `variant_id` int(10) unsigned NOT NULL, + `vid` varchar(45) NOT NULL, + `n_fam` int(11) NOT NULL DEFAULT 0 COMMENT 'consensus class: 1:pathogen, 2: vus, 3: polymorphismus/neutral, 11: class1, 12: class2, 32: class3-, 13: class3, 34: class3+, 14: class4, 15: class5, 20: artefakt, 21: nicht klassifiziert, 4: unbekannt', + `n_pat` int(11) NOT NULL DEFAULT 0, + `consensus_class` enum('1','2','3','11','12','32','13','34','14','15','20','21','4') DEFAULT NULL, + `comment` text DEFAULT NULL, + `date` date DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `id_UNIQUE` (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=0 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci; + -- -- ALTER TABLE `HerediVar`.`variant_heredicare_annotation` -- ADD COLUMN `date` DATE NULL AFTER `comment`; diff --git a/src/annotation_service/annotation_jobs/spliceai_job.py b/src/annotation_service/annotation_jobs/spliceai_job.py index a0795c8a..d98ac8f3 100644 --- a/src/annotation_service/annotation_jobs/spliceai_job.py +++ b/src/annotation_service/annotation_jobs/spliceai_job.py @@ -75,8 +75,10 @@ def annotate_missing_spliceai(self, input_vcf_path, output_vcf_path): #returncode, stderr, stdout = functions.execute_command(["ls", "-l", "/tmp"], "ls") #print(stdout) spliceai_code, spliceai_stderr, spliceai_stdout = self.annotate_spliceai_algorithm(input_vcf_path, output_vcf_path) - if 'ERROR' in spliceai_stderr: + if 'SpliceAI runtime ERROR:' in spliceai_stderr: errors.append(spliceai_stderr) + if 'Skipping record' in spliceai_stderr: + errors.append("SpliceAI WARNING skipping: " + functions.find_between(spliceai_stderr, 'WARNING:', ': chr')) # need to insert some code here to merge the newly annotated variants and previously # annotated ones from the db if there are files which contain more than one variant! @@ -93,11 +95,12 @@ def annotate_spliceai_algorithm(self, input_vcf_path, output_vcf_path): input_vcf_zipped_path = input_vcf_path + ".gz" # gbzip and index the input file as this is required for spliceai... - returncode, stderr, stdout = functions.execute_command([os.path.join(paths.htslib_path, 'bgzip'), '-f', input_vcf_path], 'bgzip') + returncode, stderr, stdout = functions.execute_command([os.path.join(paths.htslib_path, 'bgzip'), '-f', '-k', input_vcf_path], 'bgzip') if returncode != 0: return returncode, stderr, stdout returncode, stderr, stdout = functions.execute_command([os.path.join(paths.htslib_path, 'tabix'), "-f", "-p", "vcf", input_vcf_zipped_path], 'tabix') if returncode != 0: + functions.rm(input_vcf_zipped_path) return returncode, stderr, stdout # execute spliceai diff --git a/src/annotation_service/main.py b/src/annotation_service/main.py index 22614948..29ce6e2e 100644 --- a/src/annotation_service/main.py +++ b/src/annotation_service/main.py @@ -10,6 +10,7 @@ from .annotation_jobs import * from .annotation_jobs import litvar2_job import random +from mysql.connector import Error, InternalError import os @@ -217,6 +218,16 @@ def process_one_request(annotation_queue_id, job_config = get_default_job_config conn.close() #raise e #HTTPError(url = e.url, code = e.code, msg = "A HTTP error occured", hdrs = e.hdrs, fp = e.fp) return status, runtime_error + except InternalError as e: + # deadlock: code 1213 + status = "retry" + conn.update_annotation_queue(row_id=annotation_queue_id, status=status, error_msg=str(e)) + message = "Attempting retry because of database error: " + str(e) + ' ' + traceback.format_exc() + + if exists(vcf_path): + os.remove(vcf_path) + conn.close() + return status, message except Exception as e: print("An exception occured: " + str(e)) print(traceback.format_exc()) diff --git a/src/frontend_celery/start_celery.sh b/src/frontend_celery/start_celery.sh index cd39ca58..364ca180 100755 --- a/src/frontend_celery/start_celery.sh +++ b/src/frontend_celery/start_celery.sh @@ -8,9 +8,10 @@ set -o pipefail helpFunction() { echo "" - echo "Usage: $0 -w env" + echo "Usage: $0 -w env -h path" echo "This script starts the heredivar frontend gunicorn or development server" echo -e "\t-w Provide 'dev' for development server and 'prod' for production gunicorn server." + echo -e "\t-h The path to the users home directory." exit 1 # Exit script after printing help } @@ -18,6 +19,7 @@ while getopts "w:" opt do case "$opt" in w ) we="$OPTARG" ;; + h ) localhome="$OPTARG" ;; ? ) helpFunction ;; # Print helpFunction in case parameter is non-existent esac done @@ -29,6 +31,12 @@ then helpFunction fi +# set home for production environment. In systemd $HOME is not available but required for VEP +if [ -n "$localhome"] +then + export HOME=$localhome +fi + echo "preparing celery startup" diff --git a/src/frontend_celery/webapp/tasks.py b/src/frontend_celery/webapp/tasks.py index edc106c8..e362aa25 100644 --- a/src/frontend_celery/webapp/tasks.py +++ b/src/frontend_celery/webapp/tasks.py @@ -448,7 +448,7 @@ def map_hg38(variant, user_id, conn:Connection, insert_variant = True, perform_a #print(err_msg) if 'unequal' in err_msg: - if new_message not in message: + if err_msg not in message: message = functions.collect_info(message, "hgvs_msg=", err_msg, sep = " ~~ ") elif err_msg != '': preferred_transcripts = conn.get_preferred_transcripts(gene_id, return_all = True)