Skip to content

Commit

Permalink
added edit clinvar submissions and status display update
Browse files Browse the repository at this point in the history
  • Loading branch information
@MarvinDo
MarvinDo committed Aug 28, 2024
1 parent 60e4b06 commit 0528c2f
Show file tree
Hide file tree
Showing 13 changed files with 543 additions and 177 deletions.
4 changes: 3 additions & 1 deletion src/common/clinvar_interface.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,7 @@ def get_postable_consensus_classification(self, variant, selected_gene, clinvar_
"data": {"content": data}
}]
}
print(postable_data)
return postable_data

def post_consensus_classification(self, variant, selected_gene, clinvar_accession):
Expand Down Expand Up @@ -233,4 +234,5 @@ def get_clinvar_submission_status(self, submission_id):
clinvar_submission_message = ';'.join(clinvar_submission_messages)
submission_status['message'] = clinvar_submission_message

return submission_status
return submission_status

45 changes: 31 additions & 14 deletions src/common/db_IO.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1447,9 +1447,10 @@ def update_consensus_classification_needs_heredicare_upload(self, consensus_clas
self.conn.commit()

def update_consensus_classification_needs_clinvar_upload(self, consensus_classification_id):
command = "UPDATE consensus_classification SET needs_clinvar_upload = 0 WHERE id = %s"
self.cursor.execute(command, (consensus_classification_id, ))
self.conn.commit()
if consensus_classification_id is not None:
command = "UPDATE consensus_classification SET needs_clinvar_upload = 0 WHERE id = %s"
self.cursor.execute(command, (consensus_classification_id, ))
self.conn.commit()

def get_variant_ids_which_need_heredicare_upload(self, variant_ids_oi = None):
# excludes structural variants and intergenic variants and variants of unfinished submissions
Expand Down Expand Up @@ -3791,7 +3792,7 @@ def get_heredicare_queue_entries(self, publish_queue_ids: list, variant_id):
if len(publish_queue_ids) == 0:
return []
placeholders = self.get_placeholders(len(publish_queue_ids))
command = "SELECT id, status, requested_at, finished_at, message, vid, variant_id, submission_id, consensus_classification_id from publish_heredicare_queue WHERE publish_queue_id IN " + placeholders + " AND variant_id = %s"
command = "SELECT id, status, requested_at, finished_at, message, vid, variant_id, submission_id, consensus_classification_id, publish_queue_id from publish_heredicare_queue WHERE publish_queue_id IN " + placeholders + " AND variant_id = %s"
actual_information = tuple(publish_queue_ids) + (variant_id, )
self.cursor.execute(command, actual_information)
result = self.cursor.fetchall()
Expand Down Expand Up @@ -3860,11 +3861,24 @@ def get_publish_requests_page(self, page, page_size):



def insert_publish_clinvar_request(self, publish_queue_id, variant_id):
command = "INSERT INTO publish_clinvar_queue (publish_queue_id, variant_id) VALUES (%s, %s)"
self.cursor.execute(command, (publish_queue_id, variant_id))
def insert_publish_clinvar_request(self, publish_queue_id, variant_id, manually_added = False):
command = "INSERT INTO publish_clinvar_queue (publish_queue_id, variant_id, manually_added) VALUES (%s, %s, %s)"
self.cursor.execute(command, (publish_queue_id, variant_id, manually_added))
self.conn.commit()
return self.get_last_insert_id()

def is_manual_publish_clinvar_queue(self, publish_queue_id):
command = "SELECT EXISTS (SELECT * FROM publish_clinvar_queue WHERE id = %s AND manually_added = 1)"
self.cursor.execute(command, (publish_queue_id, ))
result = self.cursor.fetchone()[0]
if result == 1:
return True
return False

#def delete_manual_publish_clinvar_queue(self, publish_clinvar_queue_id):
# command = "DELETE FROM publish_clinvar_queue WHERE manually_added = 1 AND id = %s"
# self.cursor.execute(command, (publish_clinvar_queue_id, ))
# self.conn.commit()

def update_publish_clinvar_queue_celery_task_id(self, publish_clinvar_queue_id, celery_task_id):
command = "UPDATE publish_clinvar_queue SET celery_task_id = %s WHERE id = %s"
Expand Down Expand Up @@ -3903,21 +3917,24 @@ def update_publish_clinvar_queue_status(self, publish_clinvar_queue_id, status,


def get_most_recent_publish_queue_ids_clinvar(self, variant_id):
command = "SELECT DISTINCT publish_queue_id FROM publish_clinvar_queue WHERE variant_id = %s AND id >= (SELECT MAX(id) FROM publish_clinvar_queue WHERE variant_id = %s AND status != 'skipped')"
command = "SELECT DISTINCT publish_queue_id FROM publish_clinvar_queue WHERE variant_id = %s AND id >= (SELECT MAX(id) FROM publish_clinvar_queue WHERE variant_id = %s AND status != 'skipped' AND status != 'deleted')"
self.cursor.execute(command, (variant_id, variant_id))
result = self.cursor.fetchall()
if len(result) == 0:
command = "SELECT DISTINCT publish_queue_id from publish_clinvar_queue WHERE variant_id = %s AND status = 'skipped'"
command = "SELECT DISTINCT publish_queue_id from publish_clinvar_queue WHERE variant_id = %s AND status = 'skipped' AND status != 'deleted'"
self.cursor.execute(command, (variant_id, ))
result = self.cursor.fetchall()
return [x[0] for x in result]

def get_clinvar_queue_entries(self, publish_queue_ids: list, variant_id):
if len(publish_queue_ids) == 0:
return []
placeholders = self.get_placeholders(len(publish_queue_ids))
command = "SELECT id, publish_queue_id, requested_at, status, message, submission_id, accession_id, last_updated, celery_task_id, consensus_classification_id FROM publish_clinvar_queue WHERE publish_queue_id IN " + placeholders + " AND variant_id = %s"
actual_information = tuple(publish_queue_ids) + (variant_id, )
command = "SELECT id, publish_queue_id, requested_at, status, message, submission_id, accession_id, last_updated, celery_task_id, consensus_classification_id, manually_added FROM publish_clinvar_queue WHERE variant_id = %s"
actual_information = (variant_id, )
if publish_queue_ids is not None:
if len(publish_queue_ids) == 0:
return []
placeholders = self.get_placeholders(len(publish_queue_ids))
command += " AND publish_queue_id IN " + placeholders
actual_information += tuple(publish_queue_ids)
self.cursor.execute(command, actual_information)
result = self.cursor.fetchall()
return result
Expand Down
28 changes: 28 additions & 0 deletions src/frontend_celery/webapp/tasks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -412,6 +412,28 @@ def skip_hg38(variant) -> bool:
return False


def is_sv(variant) -> bool:
ref_19 = functions.none2default(variant.get('REF_HG19'), "")
alt_19 = functions.none2default(variant.get('ALT_HG19'), "")
ref_38 = functions.none2default(variant.get('REF_HG38'), "")
alt_38 = functions.none2default(variant.get('ALT_HG38'), "")

sv_indicators = ["<", ">"]
unallowed_sv_indicators = ["g."]

if any([indicator in ref_19 for indicator in sv_indicators]) and not any([indicator in ref_19 for indicator in unallowed_sv_indicators]):
return True
if any([indicator in alt_19 for indicator in sv_indicators]) and not any([indicator in alt_19 for indicator in unallowed_sv_indicators]):
return True
if any([indicator in ref_38 for indicator in sv_indicators]) and not any([indicator in ref_38 for indicator in unallowed_sv_indicators]):
return True
if any([indicator in alt_38 for indicator in sv_indicators]) and not any([indicator in alt_38 for indicator in unallowed_sv_indicators]):
return True
if any([len(seq) > 1000 for seq in [ref_19, alt_19, ref_38, alt_38]]):
return True

return False


# this is the main add variant from heredicare function! -> sanitizes and inserts variant
def map_hg38(variant, user_id, conn:Connection, insert_variant = True, perform_annotation = True, external_ids = None): # the task worker
Expand All @@ -421,6 +443,12 @@ def map_hg38(variant, user_id, conn:Connection, insert_variant = True, perform_a

allowed_sequence_letters = "ACGT"

if is_sv(variant): # skip structural variants
status = "error"
message = "Structural variants are not supported for import from HerediCaRe"
return status, message


# first check if the hg38 information is there
chrom = variant.get('CHROM')
pos = variant.get('POS_HG38')
Expand Down
110 changes: 110 additions & 0 deletions src/frontend_celery/webapp/templates/main/changelog.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,116 @@
<h1 class="bst"> {% block title %} Changelog {% endblock %} </h1>



<div class="underline">v 1.13.6 (16.08.2024)</div>
<div>
General changes:
<ul>
<li>Improved API security</li>
<li>Added variant upload and variant check through api</li>
</ul>
Bugfixes:
<ul>
<li>Fixed a visual bug when the request to start a new upload to HerediCaRe task was erroneous, but the ClinVar upload was successful</li>
</ul>
</div>


<div class="underline">v 1.13.5 (15.08.2024)</div>
<div>
Bugfixes:
<ul>
<li>When using a non-default redis port background tasks would not work correctly. This was resolved.</li>
<li>Fixed a bug when uploading variants to HerediCaRe that are unknown by HerediCaRe</li>
</ul>
</div>

<div class="underline">v 1.13.4 (25.07.2024)</div>
<div>
General changes:
<ul>
<li>Added full import of HerediCaRe VIDs button to admin dashboard alongside the preexisting incremental update</li>
<li>Updated HerediClassify to v1.0.4</li>
<li>Improved admin dashboard variant import summary page. It now loads way faster</li>
</ul>
Bugfixes:
<ul>
<li>Fixed a rare bug where HerediCaRe VIDs were inconsistently annotated</li>
<li>pm2_pp + 1x pvs now correctly classifies as likely benign in ATM v1.3.0 scheme</li>
<li>Fixed a bug that the annotation would fail when the consequence annotation tool would not get any consequences from ensembl</li>
<li>Very large variants would fail to recieve literature from litvar2 which would result in an erroneous annotation</li>
</ul>
</div>

<div class="underline">v 1.13.3 (23.07.2024)</div>
<div>
Bugfixes:
<ul>
<li>Searching for specific variants would not apply filters correctly</li>
</ul>
</div>

<div class="underline">v 1.13.2 (22.07.2024)</div>
<div>
General changes:
<ul>
<li>Added retry import button for single failed HerediCaRe VIDs</li>
<li>Improved HerediCaRe VID information page</li>
<li>Added more fine grained variant type search: added insertion, deletion and delins. Small variants are now single nucleotide variants.</li>
</ul>
Bugfixes:
<ul>
<li>Fixed a bug that CrossMap could not be invoked when lifting between genome versions</li>
<li>Variant annotation would fail when the HerediCaRe API returned a bad status code</li>
<li>Fixed a bug where variants could not be hidden</li>
</ul>
</div>

<div class="underline">v 1.13.1 (18.07.2024)</div>
<div>
General changes:
<ul>
<li>Updated ClinVar to version 2024-07-16</li>
<li>Updated HerediClassify to version 1.0.3</li>
<li>Improved security for flashed messages</li>
<li>Added HerediVar link to variant banner on multiple pages</li>
</ul>
Bugfixes:
<ul>
<li>Fixed a bug where users were unable to add variants to lists from public editable databases created by another user</li>
</ul>
</div>

<div class="underline">v 1.13 (05.07.2024)</div>
<div>
General changes:
<ul>
<li>Publish and insert queue history pages are now paginated for better scalability</li>
<li>Improved single HerediCaRe VID import</li>
<li>Updated PFAM protein domain external links to link to InterPro</li>
<li>Increased login idle timer</li>
<li>Now the REVEL score of the best transcript is shown first</li>
<li>Enabled HerediCaRe uploads</li>
</ul>
Bugfixes:
<ul>
<li>Resolved an issue where the consensus classification needs upload warning sign was shown on requested publish status</li>
<li>The ClinVar and HerediCaRe status showed status requesting when any variant was uploaded to heredicare/clinvar</li>
<li>The ClinVar and HerediCaRe status pills did not show the error message in specific cases</li>
<li>Fixed a bug that caused dynamically generated VCFs to not show the download diaglogue</li>
<li>Removed wrong chromosome from check variants page</li>
<li>Classification criteria were not wrapping properly on the variant history page</li>
<li>In specific cases the cancerhotspots annotations were incorrect</li>
<li>Fixed an issue when two criteria had the same mutually exclusive criterion target and both were selected. If the use would unselect one of them the mutually exclusive criterion would be enabled for selection even though it shouldn't</li>
<li>When a variant was not annotated or did not have any consequences the "no scheme" would be preselected on the variant classify page. Now, the correct selected default classification scheme is selected</li>
<li>Fixed an issue where the previous status of ClinVar and HerediCaRe uploads would be shown on the variant display page upon status updates</li>
<li>Fixed a bug where sometimes wrong COSMIC links would be generated</li>
</ul>
</div>




<div class="underline">v 1.12 (17.06.2024)</div>
<div>
General changes:
Expand Down
107 changes: 0 additions & 107 deletions src/frontend_celery/webapp/templates/main/documentation.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,113 +9,6 @@
<h1> {% block title %} Welcome to the HerediVar Wiki {% endblock %} </h1>


<div class="underline">v 1.13.6 (16.08.2024)</div>
<div>
General changes:
<ul>
<li>Improved API security</li>
<li>Added variant upload and variant check through api</li>
</ul>
Bugfixes:
<ul>
<li>Fixed a visual bug when the request to start a new upload to HerediCaRe task was erroneous, but the ClinVar upload was successful</li>
</ul>
</div>


<div class="underline">v 1.13.5 (15.08.2024)</div>
<div>
Bugfixes:
<ul>
<li>When using a non-default redis port background tasks would not work correctly. This was resolved.</li>
<li>Fixed a bug when uploading variants to HerediCaRe that are unknown by HerediCaRe</li>
</ul>
</div>

<div class="underline">v 1.13.4 (25.07.2024)</div>
<div>
General changes:
<ul>
<li>Added full import of HerediCaRe VIDs button to admin dashboard alongside the preexisting incremental update</li>
<li>Updated HerediClassify to v1.0.4</li>
<li>Improved admin dashboard variant import summary page. It now loads way faster</li>
</ul>
Bugfixes:
<ul>
<li>Fixed a rare bug where HerediCaRe VIDs were inconsistently annotated</li>
<li>pm2_pp + 1x pvs now correctly classifies as likely benign in ATM v1.3.0 scheme</li>
<li>Fixed a bug that the annotation would fail when the consequence annotation tool would not get any consequences from ensembl</li>
<li>Very large variants would fail to recieve literature from litvar2 which would result in an erroneous annotation</li>
</ul>
</div>

<div class="underline">v 1.13.3 (23.07.2024)</div>
<div>
Bugfixes:
<ul>
<li>Searching for specific variants would not apply filters correctly</li>
</ul>
</div>

<div class="underline">v 1.13.2 (22.07.2024)</div>
<div>
General changes:
<ul>
<li>Added retry import button for single failed HerediCaRe VIDs</li>
<li>Improved HerediCaRe VID information page</li>
<li>Added more fine grained variant type search: added insertion, deletion and delins. Small variants are now single nucleotide variants.</li>
</ul>
Bugfixes:
<ul>
<li>Fixed a bug that CrossMap could not be invoked when lifting between genome versions</li>
<li>Variant annotation would fail when the HerediCaRe API returned a bad status code</li>
<li>Fixed a bug where variants could not be hidden</li>
</ul>
</div>

<div class="underline">v 1.13.1 (18.07.2024)</div>
<div>
General changes:
<ul>
<li>Updated ClinVar to version 2024-07-16</li>
<li>Updated HerediClassify to version 1.0.3</li>
<li>Improved security for flashed messages</li>
<li>Added HerediVar link to variant banner on multiple pages</li>
</ul>
Bugfixes:
<ul>
<li>Fixed a bug where users were unable to add variants to lists from public editable databases created by another user</li>
</ul>
</div>

<div class="underline">v 1.13 (05.07.2024)</div>
<div>
General changes:
<ul>
<li>Publish and insert queue history pages are now paginated for better scalability</li>
<li>Improved single HerediCaRe VID import</li>
<li>Updated PFAM protein domain external links to link to InterPro</li>
<li>Increased login idle timer</li>
<li>Now the REVEL score of the best transcript is shown first</li>
<li>Enabled HerediCaRe uploads</li>
</ul>
Bugfixes:
<ul>
<li>Resolved an issue where the consensus classification needs upload warning sign was shown on requested publish status</li>
<li>The ClinVar and HerediCaRe status showed status requesting when any variant was uploaded to heredicare/clinvar</li>
<li>The ClinVar and HerediCaRe status pills did not show the error message in specific cases</li>
<li>Fixed a bug that caused dynamically generated VCFs to not show the download diaglogue</li>
<li>Removed wrong chromosome from check variants page</li>
<li>Classification criteria were not wrapping properly on the variant history page</li>
<li>In specific cases the cancerhotspots annotations were incorrect</li>
<li>Fixed an issue when two criteria had the same mutually exclusive criterion target and both were selected. If the use would unselect one of them the mutually exclusive criterion would be enabled for selection even though it shouldn't</li>
<li>When a variant was not annotated or did not have any consequences the "no scheme" would be preselected on the variant classify page. Now, the correct selected default classification scheme is selected</li>
<li>Fixed an issue where the previous status of ClinVar and HerediCaRe uploads would be shown on the variant display page upon status updates</li>
<li>Fixed a bug where sometimes wrong COSMIC links would be generated</li>
</ul>
</div>




<div class="row p-relative">
Expand Down
Loading

0 comments on commit 0528c2f

Please sign in to comment.