diff --git a/resources/backups/database_dumper/update_database_2.sql b/resources/backups/database_dumper/update_database_2.sql index 7be0e24e..17b6b4c2 100644 --- a/resources/backups/database_dumper/update_database_2.sql +++ b/resources/backups/database_dumper/update_database_2.sql @@ -66,3 +66,41 @@ ADD CONSTRAINT `FK_variant_heredicare_annotation_variant` ALTER TABLE `HerediVar_ahdoebm1`.`annotation_queue` CHANGE COLUMN `error_message` `error_message` TEXT NULL DEFAULT '' ; + + +ALTER TABLE `HerediVar_ahdoebm1`.`variant_ids` +ADD COLUMN `annotation_type_id` INT UNSIGNED NOT NULL AFTER `id_source`; + + +INSERT INTO `HerediVar_ahdoebm1`.`annotation_type` (`title`, `display_title`, `description`, `value_type`, `version`, `version_date`, `group_name`, `is_transcript_specific`) VALUES ('heredicare_vid', 'HerediCare VID', 'The VID from HerediCare.The version_date is inaccurate. They are always up to date when reimporting from heredicare.', 'int', '-', '2023-01-01', 'ID', '0'); + +UPDATE variant_ids SET annotation_type_id = (SELECT id FROM annotation_type WHERE title = 'heredicare_vid') WHERE id_source = 'heredicare' + + +ALTER TABLE `HerediVar_ahdoebm1`.`variant_ids` +ADD INDEX `FK_variant_ids_annotation_type_idx` (`annotation_type_id` ASC); +; +ALTER TABLE `HerediVar_ahdoebm1`.`variant_ids` +ADD CONSTRAINT `FK_variant_ids_annotation_type` + FOREIGN KEY (`annotation_type_id`) + REFERENCES `HerediVar_ahdoebm1`.`annotation_type` (`id`) + ON DELETE NO ACTION + ON UPDATE NO ACTION; + + +ALTER TABLE `HerediVar_ahdoebm1`.`variant_ids` +DROP COLUMN `id_source`, +DROP INDEX `variant_id_external_id_id_source_key` , +ADD UNIQUE INDEX `variant_id_external_id_id_source_key` (`variant_id` ASC, `external_id` ASC); +; + +ALTER TABLE `HerediVar_ahdoebm1`.`variant_ids` +DROP INDEX `variant_id_external_id_id_source_key`; +ALTER TABLE `HerediVar_ahdoebm1`.`variant_ids` +ADD UNIQUE INDEX `unique_variant_ids` (`variant_id` ASC, `external_id` ASC, `annotation_type_id` ASC); +; +UPDATE annotation_type SET group_name = "ID" WHERE title = 'rsid' + + + +INSERT INTO `HerediVar_ahdoebm1`.`annotation_type` (`title`, `display_title`, `description`, `value_type`, `version`, `version_date`, `group_name`, `is_transcript_specific`) VALUES ('clinvar', 'ClinVar variation ID', 'The Variation ID from ClinVar', 'int', '-', '2023-02-26', 'None', '0'); diff --git a/src/annotation_service/annotation_jobs/_job.py b/src/annotation_service/annotation_jobs/_job.py index 9b62faed..baceead7 100644 --- a/src/annotation_service/annotation_jobs/_job.py +++ b/src/annotation_service/annotation_jobs/_job.py @@ -58,4 +58,15 @@ def insert_annotation(self, variant_id, info, info_name, annotation_type_id, con if value_modifier_function is not None: value = value_modifier_function(value) - conn.insert_variant_annotation(variant_id, annotation_type_id, value) \ No newline at end of file + conn.insert_variant_annotation(variant_id, annotation_type_id, value) + + def insert_external_id(self, variant_id, info, info_name, annotation_type_id, conn, value_modifier_function = None): + value = functions.find_between(info, info_name, '(;|$)') + + if value == '' or value is None: + return + + if value_modifier_function is not None: + value = value_modifier_function(value) + + conn.insert_external_variant_id(variant_id = variant_id, annotation_type_id = annotation_type_id, external_id = value) \ No newline at end of file diff --git a/src/annotation_service/annotation_jobs/annotate_from_vcf_job.py b/src/annotation_service/annotation_jobs/annotate_from_vcf_job.py index ddfe43b7..98aab40c 100644 --- a/src/annotation_service/annotation_jobs/annotate_from_vcf_job.py +++ b/src/annotation_service/annotation_jobs/annotate_from_vcf_job.py @@ -45,46 +45,47 @@ def execute(self, inpath, annotated_inpath, **kwargs): def save_to_db(self, info, variant_id, conn): - self.insert_annotation(variant_id, info, "dbSNP_RS=", 3, conn) + recent_annotation_ids = conn.get_recent_annotation_type_ids() + self.insert_external_id(variant_id, info, "dbSNP_RS=", recent_annotation_ids['rsid'], conn) - self.insert_annotation(variant_id, info, "REVEL=", 6, conn) + self.insert_annotation(variant_id, info, "REVEL=", recent_annotation_ids['revel'], conn) - self.insert_annotation(variant_id, info, "CADD=", 5, conn) + self.insert_annotation(variant_id, info, "CADD=", recent_annotation_ids['cadd_scaled'], conn) - self.insert_annotation(variant_id, info, "GnomAD_AC=", 11, conn) - self.insert_annotation(variant_id, info, "GnomAD_AF=", 12, conn) - self.insert_annotation(variant_id, info, "GnomAD_hom=", 13, conn) - self.insert_annotation(variant_id, info, "GnomAD_hemi=", 14, conn) - self.insert_annotation(variant_id, info, "GnomAD_het=", 15, conn) - self.insert_annotation(variant_id, info, "GnomAD_popmax=", 16, conn, value_modifier_function = lambda value : value.upper()) - self.insert_annotation(variant_id, info, "GnomAD_AF_popmax=", 51, conn) - self.insert_annotation(variant_id, info, "GnomADm_AC_hom=", 17, conn) + self.insert_annotation(variant_id, info, "GnomAD_AC=", recent_annotation_ids['gnomad_ac'], conn) + self.insert_annotation(variant_id, info, "GnomAD_AF=", recent_annotation_ids['gnomad_af'], conn) + self.insert_annotation(variant_id, info, "GnomAD_hom=", recent_annotation_ids['gnomad_hom'], conn) + self.insert_annotation(variant_id, info, "GnomAD_hemi=", recent_annotation_ids['gnomad_hemi'], conn) + self.insert_annotation(variant_id, info, "GnomAD_het=", recent_annotation_ids['gnomad_het'], conn) + self.insert_annotation(variant_id, info, "GnomAD_popmax=", recent_annotation_ids['gnomad_popmax'], conn, value_modifier_function = lambda value : value.upper()) + self.insert_annotation(variant_id, info, "GnomAD_AF_popmax=", recent_annotation_ids['gnomad_popmax_AF'], conn) + self.insert_annotation(variant_id, info, "GnomADm_AC_hom=", recent_annotation_ids['gnomadm_ac_hom'], conn) - self.insert_annotation(variant_id, info, "BRCA_exchange_clin_sig_short=", 18, conn, value_modifier_function = lambda value : value.replace('_', ' ').replace(',', ';')) + self.insert_annotation(variant_id, info, "BRCA_exchange_clin_sig_short=", recent_annotation_ids['brca_exchange_clinical_significance'], conn, value_modifier_function = lambda value : value.replace('_', ' ').replace(',', ';')) - self.insert_annotation(variant_id, info, "FLOSSIES_num_afr=", 19, conn) - self.insert_annotation(variant_id, info, "FLOSSIES_num_eur=", 20, conn) + self.insert_annotation(variant_id, info, "FLOSSIES_num_afr=", recent_annotation_ids['flossies_num_afr'], conn) + self.insert_annotation(variant_id, info, "FLOSSIES_num_eur=", recent_annotation_ids['flossies_num_eur'], conn) - self.insert_annotation(variant_id, info, "cancerhotspots_cancertypes=", 22, conn) - self.insert_annotation(variant_id, info, "cancerhotspots_AC=", 23, conn) - self.insert_annotation(variant_id, info, "cancerhotspots_AF=", 24, conn) + self.insert_annotation(variant_id, info, "cancerhotspots_cancertypes=", recent_annotation_ids['cancerhotspots_cancertypes'], conn) + self.insert_annotation(variant_id, info, "cancerhotspots_AC=", recent_annotation_ids['cancerhotspots_ac'], conn) + self.insert_annotation(variant_id, info, "cancerhotspots_AF=", recent_annotation_ids['cancerhotspots_af'], conn) - self.insert_annotation(variant_id, info, "ARUP_classification=", 21, conn) + self.insert_annotation(variant_id, info, "ARUP_classification=", recent_annotation_ids['arup_classification'], conn) - self.insert_annotation(variant_id, info, "HCI_prior=", 52, conn) + self.insert_annotation(variant_id, info, "HCI_prior=", recent_annotation_ids['hci_prior'], conn) - self.insert_annotation(variant_id, info, "BayesDEL_noAF=", 55, conn) + self.insert_annotation(variant_id, info, "BayesDEL_noAF=", recent_annotation_ids['bayesdel'], conn) # spliceai is saved to the database in the dedicated spliceai job (which must be called after this job anyway) #self.insert_annotation(variant_id, info, 'SpliceAI=', 7, conn, value_modifier_function= lambda value : ','.join(['|'.join(x.split('|')[1:]) for x in value.split(',')]) ) #self.insert_annotation(variant_id, info, 'SpliceAI=', 8, conn, value_modifier_function= lambda value : ','.join([str(max([float(x) for x in x.split('|')[2:6]])) for x in value.split(',')]) ) - self.insert_annotation(variant_id, info, "tp53db_class=", 27, conn) - self.insert_annotation(variant_id, info, "tp53db_bayes_del=", 30, conn) - self.insert_annotation(variant_id, info, "tp53db_DNE_LOF_class=", 29, conn) - self.insert_annotation(variant_id, info, "tp53db_DNE_class=", 31, conn) - self.insert_annotation(variant_id, info, "tp53db_domain_function=", 32, conn) - self.insert_annotation(variant_id, info, "tp53db_transactivation_class=", 33, conn) + self.insert_annotation(variant_id, info, "tp53db_class=", recent_annotation_ids['tp53db_class'], conn) + self.insert_annotation(variant_id, info, "tp53db_bayes_del=", recent_annotation_ids['tp53db_bayes_del'], conn) + self.insert_annotation(variant_id, info, "tp53db_DNE_LOF_class=", recent_annotation_ids['tp53db_DNE_LOF_class'], conn) + self.insert_annotation(variant_id, info, "tp53db_DNE_class=", recent_annotation_ids['tp53db_DNE_class'], conn) + self.insert_annotation(variant_id, info, "tp53db_domain_function=", recent_annotation_ids['tp53db_domain_function'], conn) + self.insert_annotation(variant_id, info, "tp53db_transactivation_class=", recent_annotation_ids['tp53db_transactivation_class'], conn) pmids = functions.find_between(info, 'tp53db_pubmed=', '(;|$)') if pmids is not None and pmids != '': if self.job_config['insert_literature']: @@ -101,6 +102,7 @@ def save_to_db(self, info, variant_id, conn): clinvar_submissions = clinvar_submissions.split('&') clv_revstat = functions.find_between(info, 'ClinVar_revstat=', '(;|$)') clv_varid = functions.find_between(info, 'ClinVar_varid=', '(;|$)') + self.insert_external_id(variant_id, info, "ClinVar_varid=", recent_annotation_ids['clinvar'], conn) clv_inpret = functions.find_between(info, 'ClinVar_inpret=', '(;|$)') if clv_revstat is not None and clv_inpret is not None and clv_varid is not None: diff --git a/src/annotation_service/annotation_jobs/heredicare_job.py b/src/annotation_service/annotation_jobs/heredicare_job.py index 5ad766d3..ba23d167 100644 --- a/src/annotation_service/annotation_jobs/heredicare_job.py +++ b/src/annotation_service/annotation_jobs/heredicare_job.py @@ -37,7 +37,8 @@ def save_to_db(self, info, variant_id, conn): conn.clear_heredicare_annotation(variant_id) - vids = conn.get_external_ids_from_variant_id(variant_id, id_source="heredicare") # the vids are imported from the import variants admin page + heredicare_vid_annotation_type_id = conn.get_most_recent_annotation_type_id('heredicare_vid') + vids = conn.get_external_ids_from_variant_id(variant_id, annotation_type_id=heredicare_vid_annotation_type_id) # the vids are imported from the import variants admin page #print(vids) diff --git a/src/common/db_IO.py b/src/common/db_IO.py index 8aeb4c92..0e62dc3c 100644 --- a/src/common/db_IO.py +++ b/src/common/db_IO.py @@ -249,25 +249,25 @@ def insert_variant(self, chr, pos, ref, alt, orig_chr, orig_pos, orig_ref, orig_ variant_id = self.get_variant_id(chr, pos, ref, alt) return variant_id # return the annotation_queue_id of the new variant - def insert_external_variant_id(self, variant_id, external_id, id_source): - command = "INSERT INTO variant_ids (variant_id, external_id, id_source) \ + def insert_external_variant_id(self, variant_id, external_id, annotation_type_id): + command = "INSERT INTO variant_ids (variant_id, external_id, annotation_type_id) \ SELECT %s, %s, %s FROM DUAL WHERE NOT EXISTS (SELECT * FROM variant_ids \ - WHERE `variant_id`=%s AND `external_id`=%s AND `id_source`=%s LIMIT 1)" - self.cursor.execute(command, (variant_id, external_id, id_source, variant_id, external_id, id_source)) + WHERE `variant_id`=%s AND `external_id`=%s AND `annotation_type_id`=%s LIMIT 1)" + self.cursor.execute(command, (variant_id, external_id, annotation_type_id, variant_id, external_id, annotation_type_id)) self.conn.commit() - def update_external_variant_id(self, variant_id, external_id, id_source): - command = "UPDATE variant_ids SET external_id = %s WHERE variant_id = %s AND id_source = %s" - self.cursor.execute(command, (external_id, variant_id, id_source)) - self.conn.commit() - - def insert_update_external_variant_id(self, variant_id, external_id, id_source): - previous_external_variant_id = self.get_external_ids_from_variant_id(variant_id, id_source=id_source) - #print(previous_external_variant_id) - if (len(previous_external_variant_id) == 1): # do update - self.update_external_variant_id(variant_id, external_id, id_source) - else: # save new - self.insert_external_variant_id(variant_id, external_id, id_source) + #def update_external_variant_id(self, variant_id, external_id, annotation_type_id): + # command = "UPDATE variant_ids SET external_id = %s WHERE variant_id = %s AND annotation_type_id = %s" + # self.cursor.execute(command, (external_id, variant_id, annotation_type_id)) + # self.conn.commit() +# + #def insert_update_external_variant_id(self, variant_id, external_id, annotation_type_id): + # previous_external_variant_id = self.get_external_ids_from_variant_id(variant_id, annotation_type_id=annotation_type_id) + # #print(previous_external_variant_id) + # if (len(previous_external_variant_id) == 1): # do update + # self.update_external_variant_id(variant_id, external_id, annotation_type_id) + # else: # save new + # self.insert_external_variant_id(variant_id, external_id, annotation_type_id) def insert_annotation_request(self, variant_id, user_id): # this inserts only if there is not an annotation request for this variant which is still pending #command = "INSERT INTO annotation_queue (variant_id, status, user_id) VALUES (%s, %s, %s)" @@ -614,7 +614,7 @@ def sort_consequences(self, a, b): return 0 - def get_variants_page_merged(self, page, page_size, sort_by, include_hidden, user_id, ranges = None, genes = None, consensus = None, user = None, hgvs = None, variant_ids_oi = None, include_heredicare_consensus = False): + def get_variants_page_merged(self, page, page_size, sort_by, include_hidden, user_id, ranges = None, genes = None, consensus = None, user = None, hgvs = None, variant_ids_oi = None, external_ids = None, include_heredicare_consensus = False): # get one page of variants determined by offset & pagesize prefix = "SELECT id, chr, pos, ref, alt FROM variant" @@ -662,7 +662,6 @@ def get_variants_page_merged(self, page, page_size, sort_by, include_hidden, use new_constraints = "variant.id IN (" + new_constraints_inner + ")" #postfix = self.add_constraints_to_command(postfix, new_constraints) constraints_complete = new_constraints - if include_heredicare_consensus and len(consensus_without_dash) > 0: heredicare_consensus = [] for c in consensus_without_dash: @@ -673,9 +672,7 @@ def get_variants_page_merged(self, page, page_size, sort_by, include_hidden, use actual_information += tuple(heredicare_consensus) actual_information += tuple(consensus_without_dash) constraints_complete = functions.enbrace(constraints_complete + " OR " + new_constraints) - postfix = self.add_constraints_to_command(postfix, constraints_complete) - if user is not None and len(user) > 0: new_constraints_inner = '' user_without_dash = [value for value in user if value != '-'] @@ -720,6 +717,46 @@ def get_variants_page_merged(self, page, page_size, sort_by, include_hidden, use new_constraints = "id IN " + placeholders actual_information += tuple(all_variants) postfix = self.add_constraints_to_command(postfix, new_constraints) + if external_ids is not None and len(external_ids) > 0: + ids_unknown_source = [] + ids_known_source = {} + for external_id in external_ids: + # handle rsids + if external_id.startswith('rs') or external_id[-5:].lower() == ':rsid': + external_id = external_id.strip('rs').strip(':rsid') + functions.extend_dict(ids_known_source, 'rsid', external_id) + continue + if external_id.startswith('COSV') or external_id[-7:].lower() == ':cosmic': + external_id = external_id.strip(':cosmic') + functions.extend_dict(ids_known_source, 'cosmic', external_id) + continue + if external_id[-8:].lower() == ':clinvar': + external_id = external_id.strip(':clinvar') + functions.extend_dict(ids_known_source, 'clinvar', external_id) + continue + if external_id[-11:] == ':heredicare': + external_id = external_id.strip(':heredicare') + functions.extend_dict(ids_known_source, 'heredicare_vid', external_id) + continue + ids_unknown_source.append(external_id) + new_constraints = [] + + for id_source in ids_known_source: + annotation_type_id = self.get_most_recent_annotation_type_id(id_source) + current_external_ids = ids_known_source[id_source] + placeholders = self.get_placeholders(len(current_external_ids)) + new_constraints.append("id IN (SELECT variant_id FROM variant_ids WHERE external_id IN " + placeholders + " AND annotation_type_id = %s )") + actual_information += tuple(current_external_ids) + actual_information += (annotation_type_id, ) + + if len(ids_unknown_source) > 0: + placeholders = self.get_placeholders(len(ids_unknown_source)) + new_constraints.append("id IN (SELECT variant_id FROM variant_ids WHERE external_id IN " + placeholders + ")") + actual_information += tuple(ids_unknown_source) + new_constraints = ' OR '.join(new_constraints) + postfix = self.add_constraints_to_command(postfix, new_constraints) + + if variant_ids_oi is not None and len(variant_ids_oi) > 0: placeholders = ["%s"] * len(variant_ids_oi) placeholders = ', '.join(placeholders) @@ -993,19 +1030,15 @@ def check_consensus_classification(self, variant_id, consensus_classification, c else: return True - def get_variant_id_from_external_id(self, id, id_source): #!! assumed that the external_id column contains unique entries for id, id_source pairs! - command = "SELECT variant_id FROM variant_ids WHERE external_id = %s AND id_source = %s" - self.cursor.execute(command, (id, id_source)) + def get_variant_id_from_external_id(self, external_id, annotation_type_id): #!! assumed that the external_id column contains unique entries for id, id_source pairs! + command = "SELECT variant_id FROM variant_ids WHERE external_id = %s AND annotation_type_id = %s" + self.cursor.execute(command, (external_id, annotation_type_id)) result = self.cursor.fetchone() if result is not None: return result[0] return result - def get_all_external_ids(self, id_source): - command = "SELECT external_id FROM variant_ids WHERE id_source = %s" - self.cursor.execute(command, (id_source, )) - result = self.cursor.fetchall() - return [x[0] for x in result] + def get_consensus_classification(self, variant_id, most_recent = False, sql_modifier=None): # it is possible to have multiple consensus classifications command = "SELECT id,user_id,variant_id,classification,comment,date,is_recent,classification_scheme_id,scheme_class FROM consensus_classification WHERE variant_id = %s" @@ -1369,33 +1402,43 @@ def convert_raw_import_variant_request(self, import_variant_request_raw): return result - # returns a list of external ids if an id source is given - # returns a list of tuples with (external_id, source) if no id source is given -> exports all external ids - def get_external_ids_from_variant_id(self, variant_id, id_source=''): - allowed_columns = ["external_id", "id_source"] - - columns_oi = ["external_id"] - if id_source == '' or id_source is None: - columns_oi = columns_oi + ["id_source"] - if any([c not in allowed_columns for c in columns_oi]): - return [] - command = "SELECT " + ", ".join(columns_oi) + " FROM variant_ids WHERE variant_id = %s" - information = (variant_id,) - if id_source != '': - command = command + " AND id_source = %s" - information = information + (id_source, ) + #returns a list of tuples with all information for each external id + def get_all_external_ids(self, variant_id): + command = """ + SELECT variant_ids.id, title, description, version, version_date, variant_id, external_id, group_name, display_title, value_type FROM variant_ids INNER JOIN ( + SELECT * + FROM annotation_type WHERE (title, version_date) IN ( + select title, MAX(version_date) version_date from annotation_type INNER JOIN ( + select variant_id, annotation_type_id, external_id from variant_ids where variant_id=%s + ) x + ON annotation_type.id = x.annotation_type_id + GROUP BY title + ) + ) y + ON y.id = variant_ids.annotation_type_id + WHERE variant_id=%s AND group_name = 'ID' + """ + self.cursor.execute(command, (variant_id, variant_id)) + result = self.cursor.fetchall() + return result + + def get_all_external_ids_from_annotation_type(self, annotation_type_id): + command = "SELECT external_id FROM variant_ids WHERE annotation_type_id = %s" + self.cursor.execute(command, (annotation_type_id, )) + result = self.cursor.fetchall() + return [x[0] for x in result] + + # returns a list of external ids of a specific type + def get_external_ids_from_variant_id(self, variant_id, annotation_type_id): + command = "SELECT external_id FROM variant_ids WHERE variant_id = %s AND annotation_type_id = %s" + information = (variant_id, annotation_type_id) self.cursor.execute(command, information) result = self.cursor.fetchall() - if result is None: - return [] - if id_source != '': - return [x[0] for x in result] - else: - return result + return [x[0] for x in result] - def delete_external_id(self, external_id, id_source, variant_id = None): - command = "DELETE FROM variant_ids WHERE external_id = %s AND id_source = %s" - actual_information = (external_id, id_source) + def delete_external_id(self, external_id, annotation_type_id, variant_id = None): + command = "DELETE FROM variant_ids WHERE external_id = %s AND annotation_type_id = %s" + actual_information = (external_id, annotation_type_id) if variant_id is not None: command += " AND variant_id = %s" actual_information += (variant_id, ) @@ -1644,7 +1687,7 @@ def get_recent_annotations(self, variant_id): # ! the ordering of the columns in ) \ ) y \ ON y.id = variant_annotation.annotation_type_id \ - WHERE variant_id=%s" + WHERE variant_id=%s AND group_name != 'ID'" self.cursor.execute(command, (variant_id, variant_id)) result = self.cursor.fetchall() return result @@ -1707,7 +1750,8 @@ def get_variant(self, variant_id, include_clinvar = True, include_consequences = True, include_assays = True, - include_literature = True + include_literature = True, + include_external_ids = True ) -> models.Variant: variant_raw = self.get_one_variant(variant_id) if variant_raw is None: @@ -1749,6 +1793,26 @@ def get_variant(self, variant_id, annotations.flag_linked_annotations() + # add external ids + external_ids = None + if include_external_ids: + external_ids = [] + external_ids_raw = self.get_all_external_ids(variant_id) + #variant_ids.id, title, description, version, version_date, variant_id, external_id, group_name, display_title, value_type + for external_id_raw in external_ids_raw: + new_external_id = models.Annotation(id = external_id_raw[0], + value = external_id_raw[6], + title = external_id_raw[1], + display_title = external_id_raw[8], + description = external_id_raw[2], + version = external_id_raw[3], + version_date = external_id_raw[4], + value_type = external_id_raw[9], + group_name = external_id_raw[7] + ) + external_ids.append(new_external_id) + + # add all consensus classifications consensus_classifications = None if include_consensus: @@ -1976,7 +2040,8 @@ def get_variant(self, variant_id, clinvar = clinvar, consequences = consequences, assays = assays, - literature = literature + literature = literature, + external_ids = external_ids ) return variant diff --git a/src/common/functions.py b/src/common/functions.py index 62e2aad9..c1b5e4c8 100644 --- a/src/common/functions.py +++ b/src/common/functions.py @@ -736,4 +736,12 @@ def num2heredicare(classification): "-": ["20", "21", "4", "-1"], "M": ["M"] } - return mapping[str(classification)] \ No newline at end of file + return mapping[str(classification)] + + +def extend_dict(dictionary, key, new_value): + if key in dictionary: + dictionary[key].append(new_value) + else: + dictionary[key] = [new_value] + return dictionary \ No newline at end of file diff --git a/src/common/models.py b/src/common/models.py index 72bd9d1d..e3278d38 100644 --- a/src/common/models.py +++ b/src/common/models.py @@ -666,6 +666,16 @@ class Variant: annotations: AllAnnotations = AllAnnotations() + external_ids: Any = None # list of Annotations + + def get_external_ids(self, title): + result = [] + if self.external_ids is not None: + for external_id in self.external_ids: + if external_id.title == title: + result.append(external_id) + return result + def get_heredicare_consensus_classifications(self): result = [] for heredicare_annotation in self.heredicare_annotations: diff --git a/src/frontend_celery/webapp/tasks.py b/src/frontend_celery/webapp/tasks.py index 909348af..593c3f12 100644 --- a/src/frontend_celery/webapp/tasks.py +++ b/src/frontend_celery/webapp/tasks.py @@ -141,8 +141,8 @@ def import_variants(conn: Connection, user_id, user_roles, min_date, import_queu #all_vids_heredicare, status, message = heredicare_interface.get_vid_list() if status == "success": - - vids_heredivar = conn.get_all_external_ids("heredicare") + annotation_type_id = conn.get_most_recent_annotation_type_id("heredicare_vid") + vids_heredivar = conn.get_all_external_ids_from_annotation_type(annotation_type_id) intersection, heredivar_exclusive_vids, heredicare_exclusive_vids = compare_v_id_lists(all_vids_heredicare, vids_heredivar, vids_heredicare) @@ -230,10 +230,11 @@ def delete_variant_heredicare(self, vid, vids, user_id, user_roles, import_varia message = "Removed heredicare vid" status = "update" - variant_id = conn.get_variant_id_from_external_id(vid, 'heredicare') + annotation_type_id = conn.get_most_recent_annotation_type_id("heredicare_vid") + variant_id = conn.get_variant_id_from_external_id(vid, annotation_type_id) if variant_id is not None: - all_vids_for_variant = conn.get_external_ids_from_variant_id(variant_id, 'heredicare') - conn.delete_external_id(vid, 'heredicare', variant_id = variant_id) + all_vids_for_variant = conn.get_external_ids_from_variant_id(variant_id, annotation_type_id) + conn.delete_external_id(vid, annotation_type_id = annotation_type_id, variant_id = variant_id) if all([v in vids for v in all_vids_for_variant]): status = "deleted" message = "Variant was hidden because it does not have any vids in heredicare anymore" @@ -356,9 +357,10 @@ def fetch_heredicare(vid, heredicare_interface, user_id, conn:Connection, insert return status, message if str(variant.get("VISIBLE", "0")) == "0": - variant_id = conn.get_variant_id_from_external_id(vid, "heredicare") + annotation_type_id = conn.get_most_recent_annotation_type_id("heredicare_vid") + variant_id = conn.get_variant_id_from_external_id(vid, annotation_type_id) if variant_id is not None: - conn.delete_external_id(vid, "heredicare", variant_id) + conn.delete_external_id(vid, annotation_type_id, variant_id) variant_vids = conn.get_external_ids_from_variant_id(variant_id) if len(variant_vids) == 0: conn.hide_variant(variant_id, is_hidden = False) @@ -481,7 +483,8 @@ def map_hg38(variant, user_id, conn:Connection, insert_variant = True, perform_a if variant_id is not None and external_ids is not None: # insert new vid for external_id in external_ids: - conn.insert_external_variant_id(variant_id, external_id, "heredicare") + annotation_type_id = conn.get_most_recent_annotation_type_id("heredicare_vid") + conn.insert_external_variant_id(variant_id, external_id, annotation_type_id) if not was_successful and message == '': new_message = "Not enough data to convert variant!" diff --git a/src/frontend_celery/webapp/templates/macros.html b/src/frontend_celery/webapp/templates/macros.html index 62d2da5e..19ca2355 100644 --- a/src/frontend_celery/webapp/templates/macros.html +++ b/src/frontend_celery/webapp/templates/macros.html @@ -459,14 +459,14 @@
@@ -481,14 +481,14 @@
@@ -783,9 +783,10 @@ {{ consequence.get_num_flags() }} + {% set rsids = variant.get_external_ids('rsid') %} {% set rsid = none %} - {% if variant.annotations.rsid is not none %} - {% set rsid = variant.annotations.rsid.value %} + {% if rsids | length > 0 %} + {% set rsid = rsids[0].value %} {% endif %} {{ google_link("google", rsid = rsid, hgvs_c = consequence.hgvs_c, hgvs_p = consequence.hgvs_p, gene_symbol = consequence.transcript.gene.symbol, is_scholar = False) }} @@ -1018,10 +1019,13 @@
  • {{ external_link("VarSome", "https://varsome.com/security-validation/?next=/variant/hg38/" + variant.chrom[3:]|string + ":" + variant.pos|string + ":" + variant.ref|string + ":" + variant.alt|string) }}
    • - {% if variant.annotations.rsid is not none %} -
  • - {{ external_link("dbSNP", "https://www.ncbi.nlm.nih.gov/snp/rs" + variant.annotations.rsid.value|string) }} + {% set rsids = variant.get_external_ids('rsid') %} + {% if rsids | length > 0 %} + {% for rsid in rsids %} +
  • + {{ external_link("dbSNP", "https://www.ncbi.nlm.nih.gov/snp/rs" + rsid.value|string) }}
    • + {% endfor %} {% endif %} {% if variant.clinvar is not none %}
  • diff --git a/src/frontend_celery/webapp/utils/search_utils.py b/src/frontend_celery/webapp/utils/search_utils.py index 3d33380f..c236d149 100644 --- a/src/frontend_celery/webapp/utils/search_utils.py +++ b/src/frontend_celery/webapp/utils/search_utils.py @@ -92,7 +92,12 @@ def extract_genes(request_obj): flash("You have an error in your genes query(s). Results are not filtered by genes.", "alert-danger") return genes - +def extract_external_ids(request_obj): + external_ids = request_obj.args.get('external_ids', '') + external_ids = preprocess_query(external_ids) + if external_ids is None: + flash("You have an error in your external ID query(s). Results are not filtered by external IDs.", "alert-danger") + return external_ids def extract_consensus_classifications(request_obj, allowed_classes): classes = allowed_classes + ['-'] @@ -103,7 +108,6 @@ def extract_consensus_classifications(request_obj, allowed_classes): consensus_classifications = preprocess_query(consensus_classifications, r'(' + regex_inner + r')?') if consensus_classifications is None: flash("You have an error in your consensus class query(s). It must consist of a number between 1-5, 3+, 3- or M. Results are not filtered by consensus classification.", "alert-danger") - include_heredicare = True if request_obj.args.get('include_heredicare_consensus', 'off') == 'on' else False return consensus_classifications, include_heredicare diff --git a/src/frontend_celery/webapp/variant/variant_routes.py b/src/frontend_celery/webapp/variant/variant_routes.py index 6f4c2cc8..b9d4b923 100644 --- a/src/frontend_celery/webapp/variant/variant_routes.py +++ b/src/frontend_celery/webapp/variant/variant_routes.py @@ -38,6 +38,7 @@ def search(): user_classifications = extract_user_classifications(request, allowed_user_classes) hgvs = extract_hgvs(request) variant_ids_oi = extract_lookup_list(request, user_id, conn) + external_ids = extract_external_ids(request) page = int(request.args.get('page', 1)) sort_bys, page_sizes, selected_page_size, selected_sort_by, include_hidden = extract_search_settings(request) @@ -54,7 +55,8 @@ def search(): user=user_classifications, hgvs=hgvs, variant_ids_oi=variant_ids_oi, - include_heredicare_consensus = include_heredicare_consensus + include_heredicare_consensus = include_heredicare_consensus, + external_ids = external_ids ) lists = conn.get_lists_for_user(user_id) pagination = Pagination(page=page, per_page=selected_page_size, total=total, css_framework='bootstrap5') @@ -213,7 +215,7 @@ def display(variant_id=None, chr=None, pos=None, ref=None, alt=None): clinvar_submission = check_update_clinvar_status(variant_id, conn) - #print(variant.annotations) + print(variant.external_ids) return render_template('variant/variant.html', current_annotation_status=current_annotation_status,