From 85471c91fa4453f9121c7f25f4af9bbeb2452924 Mon Sep 17 00:00:00 2001 From: MarvinDo Date: Thu, 13 Jun 2024 14:20:26 +0200 Subject: [PATCH] improved session cookie security & first implementation of api --- .../update_database/update_database_10.sql | 3 + src/common/db_IO.py | 13 ++ src/common/heredicare_interface.py | 20 +-- src/frontend_celery/config.py | 31 +++-- src/frontend_celery/webapp/__init__.py | 2 + src/frontend_celery/webapp/api/__init__.py | 3 + src/frontend_celery/webapp/api/api_routes.py | 82 ++++++++++++ .../webapp/auth/auth_routes.py | 52 ++------ .../templates/auth/generate_api_key.html | 33 +++++ .../webapp/templates/auth/profile.html | 12 +- .../webapp/templates/index.html | 3 + .../webapp/templates/main/documentation.html | 118 +++++++++++++++++- .../webapp/templates/upload/publish.html | 2 +- .../webapp/templates/variant/variant.html | 4 +- .../webapp/upload/upload_tasks.py | 4 +- .../webapp/utils/decorators.py | 47 +++++++ .../webapp/variant/variant_routes.py | 2 +- 17 files changed, 359 insertions(+), 72 deletions(-) create mode 100644 src/frontend_celery/webapp/api/__init__.py create mode 100644 src/frontend_celery/webapp/api/api_routes.py create mode 100644 src/frontend_celery/webapp/templates/auth/generate_api_key.html diff --git a/resources/update_database/update_database_10.sql b/resources/update_database/update_database_10.sql index a9a627ff..68b3b92c 100644 --- a/resources/update_database/update_database_10.sql +++ b/resources/update_database/update_database_10.sql @@ -4,3 +4,6 @@ ADD COLUMN `version` VARCHAR(45) NOT NULL AFTER `alias`; ALTER TABLE `HerediVar_ahdoebm1`.`classification_scheme_alias` ADD UNIQUE INDEX `classification_scheme_alias_unique` (`alias` ASC, `version` ASC); ; + +ALTER TABLE `HerediVar_ahdoebm1`.`user` +ADD COLUMN `api_key` VARCHAR(64) NULL AFTER `affiliation`; diff --git a/src/common/db_IO.py b/src/common/db_IO.py index 33492d07..b6788bd0 100644 --- a/src/common/db_IO.py +++ b/src/common/db_IO.py @@ -1998,6 +1998,19 @@ def get_user(self, user_id): result = self.cursor.fetchone() return result + def set_api_key(self, username, api_key): + command = "UPDATE user SET api_key = %s WHERE username = %s" + self.cursor.execute(command, (api_key, username)) + self.conn.commit() + + def check_api_key(self, api_key: str, username: str) -> bool: + command = "SELECT EXISTS (SELECT * FROM user WHERE api_key = %s AND username = %s)" + self.cursor.execute(command, (api_key, username)) + result = self.cursor.fetchone()[0] + if result == 1: + return True + return False + def parse_raw_user(self, raw_user): return models.User(id = raw_user[0], full_name = raw_user[2] + ' ' + raw_user[3], diff --git a/src/common/heredicare_interface.py b/src/common/heredicare_interface.py index e222a370..d101d165 100644 --- a/src/common/heredicare_interface.py +++ b/src/common/heredicare_interface.py @@ -352,10 +352,8 @@ def get_submission_status(self, submission_id): message = "ERROR: HerediCare API getsubmission id endpoint endpoint returned an HTTP " + str(resp.status_code) + " error: " + self.extract_error_message(resp.text) status = "api_error" else: # success - print(resp.text) resp = resp.json(strict=False) items = resp["items"] - print(items) if len(items) == 0: # submission id was generated but no data was posted yet status = "pending" @@ -446,21 +444,10 @@ def get_variant_items(self, variant, vid, submission_id, post_regexes, transacti for consequence in consequences: if consequence.transcript.gene is None: continue - if consequence.transcript.gene == preferred_gene: + if consequence.transcript.gene.symbol == preferred_gene: preferred_consequence = consequence preferred_transcript = consequence.transcript.name break - #preferred_consequences = variant.get_preferred_transcripts(within_gene = True) - #if preferred_consequences is not None: - # preferred_genes = functions.get_preferred_genes() - # for consequence in preferred_consequences: - # if consequence.transcript.gene.symbol in preferred_genes: - # preferred_consequence = consequence - # break - # elif preferred_consequence is None: - # preferred_consequence = consequence - # elif preferred_consequence.length < consequence.length: - # preferred_consequence = consequence if preferred_consequence is None: status = "skipped" @@ -681,8 +668,9 @@ def get_data(self, variant, vid, options): # tinker all items together data = {'items': all_items} - with open('/mnt/storage2/users/ahdoebm1/HerediVar/src/common/heredicare_interface_debug/sub' + str(submission_id) + '.json', "w") as f: - functions.prettyprint_json(data, f.write) + if os.environ.get('WEBAPP_ENV', '') == 'dev': + with open('/mnt/storage2/users/ahdoebm1/HerediVar/src/common/heredicare_interface_debug/sub' + str(submission_id) + '.json', "w") as f: + functions.prettyprint_json(data, f.write) data = json.dumps(data) return data, vid, submission_id, status, message diff --git a/src/frontend_celery/config.py b/src/frontend_celery/config.py index 33b4e0ba..5bc13e22 100644 --- a/src/frontend_celery/config.py +++ b/src/frontend_celery/config.py @@ -3,7 +3,15 @@ import sys sys.path.append(path.dirname(path.dirname(path.abspath(__file__)))) import common.functions as functions - +from redis import Redis +from redis.backoff import ExponentialBackoff +from redis.retry import Retry +from redis.client import Redis +from redis.exceptions import ( + BusyLoadingError, + ConnectionError, + TimeoutError +) functions.read_dotenv() @@ -40,11 +48,17 @@ class Config(object): DISCOVERYURL = f'{ISSUER}/.well-known/openid-configuration' # configuration of server side session from flask-session module - SESSION_PERMANENT = False - SESSION_TYPE = "filesystem" - #SESSION_COOKIE_SECURE = True - SESSION_USE_SIGNER = True - SESSION_FILE_DIR = os.path.dirname(os.path.abspath(__file__)) + "/flask_sessions" + SESSION_PERMANENT = True + + SESSION_TYPE = 'redis' + retry = Retry(ExponentialBackoff(), 3) + SESSION_REDIS = Redis.from_url('redis://localhost:6379/0', retry=retry, retry_on_error=[BusyLoadingError, ConnectionError, TimeoutError]) # use exponential backoff with 3 tries on specific redis errors + + #SESSION_USE_SIGNER = True # deprecated + #SESSION_TYPE = "filesystem" + #SESSION_FILE_DIR = os.path.dirname(os.path.abspath(__file__)) + "/flask_sessions" + SESSION_COOKIE_SAMESITE = "Lax" + SESSION_COOKIE_HTTPONLY = True # other folders #RESOURCES_FOLDER = 'resources/' @@ -71,8 +85,8 @@ class Config(object): class ProdConfig(Config): - KEYCLOAK_HOST = os.environ.get('KEYCLOAK_HOST', 'localhost') # keycloak + KEYCLOAK_HOST = os.environ.get('KEYCLOAK_HOST', 'localhost') KEYCLOAK_PORT = '8080' KEYCLOAK_BASEPATH = "https://"+KEYCLOAK_HOST+"/kc" ISSUER = os.environ.get('ISSUER', KEYCLOAK_BASEPATH + "/realms/HerediVar") @@ -80,6 +94,9 @@ class ProdConfig(Config): CLIENTSECRET = os.environ.get('CLIENT_SECRET') DISCOVERYURL = f'{ISSUER}/.well-known/openid-configuration' + # sesssion + SESSION_COOKIE_SECURE = True + #SESSION_COOKIE_NAME = "__Host-" class DevConfig(Config): diff --git a/src/frontend_celery/webapp/__init__.py b/src/frontend_celery/webapp/__init__.py index 720ea95f..21079fc9 100644 --- a/src/frontend_celery/webapp/__init__.py +++ b/src/frontend_celery/webapp/__init__.py @@ -70,6 +70,7 @@ def create_app(): from .user import create_module as user_create_module from .errorhandlers import create_module as errorhandlers_create_module from .upload import create_module as upload_create_module + from .api import create_module as api_create_module main_create_module(app) variant_create_module(app) @@ -79,6 +80,7 @@ def create_app(): user_create_module(app) errorhandlers_create_module(app) upload_create_module(app) + api_create_module(app) configure_logging(app) diff --git a/src/frontend_celery/webapp/api/__init__.py b/src/frontend_celery/webapp/api/__init__.py new file mode 100644 index 00000000..52adf225 --- /dev/null +++ b/src/frontend_celery/webapp/api/__init__.py @@ -0,0 +1,3 @@ +def create_module(app, **kwargs): + from .api_routes import api_blueprint + app.register_blueprint(api_blueprint) \ No newline at end of file diff --git a/src/frontend_celery/webapp/api/api_routes.py b/src/frontend_celery/webapp/api/api_routes.py new file mode 100644 index 00000000..0da635fe --- /dev/null +++ b/src/frontend_celery/webapp/api/api_routes.py @@ -0,0 +1,82 @@ +import json +import os +import sys +import requests +from flask import url_for, session, request, Blueprint, current_app +from flask import render_template, redirect, jsonify +from flask_session import Session +sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))) +import common.functions as functions +from common.db_IO import Connection +from ..utils import * + + +api_blueprint = Blueprint( + 'api', + __name__ +) + + +@api_blueprint.route('/api/v1.0/get/consensus_classification', methods=['GET']) +@accept_token +def consensus_classification(): + + conn = Connection(roles = ["read_only"]) + + variant_id = request.args.get('variant_id') + if variant_id is None: + chrom = request.args.get('chrom') + pos = request.args.get('pos') + ref = request.args.get('ref') + alt = request.args.get('alt') + + variant_id = conn.get_variant_id(chrom, pos, ref, alt) + + if variant_id is None: + conn.close() + abort(404, "Requested variant does not exist or missing variant information") + + variant = conn.get_variant(variant_id, include_annotations = False, include_consensus = True, include_user_classifications = False, include_heredicare_classifications=False, include_automatic_classification=False, include_clinvar=False, include_consequences=False, include_assays=False, include_literature=False, include_external_ids=False) + conn.close() + + + v_res = prepare_variant(variant) + mrcc = variant.get_recent_consensus_classification() + mrcc_res = prepare_classification(mrcc) + + result = { + "variant": v_res, + "classification": mrcc_res + } + + return jsonify(result) + + +def prepare_variant(variant): + result = { + "id": variant.id, + "chrom": variant.chrom, + "pos": variant.pos, + "ref": variant.ref, + "alt": variant.alt, + "variant_type": variant.variant_type, + "hidden": variant.is_hidden + } + return result + +def prepare_classification(classification): + result = { + "comment": classification.comment, + "date": classification.date, + "literature": classification.literature, + "scheme": {"name": classification.scheme.display_name, "reference": classification.scheme.reference}, + "criteria": classification.scheme.criteria, + "class_by_scheme": classification.scheme.selected_class, + "selected_class": classification.selected_class, + "classification_type": classification.type + } + return result + + + + diff --git a/src/frontend_celery/webapp/auth/auth_routes.py b/src/frontend_celery/webapp/auth/auth_routes.py index d2522643..8cb66d0e 100644 --- a/src/frontend_celery/webapp/auth/auth_routes.py +++ b/src/frontend_celery/webapp/auth/auth_routes.py @@ -395,8 +395,18 @@ def profile(): return render_template('auth/profile.html', user = user) +@auth_blueprint.route('/generate_api_key', methods=['GET', 'POST']) +@require_permission(["read_resources"]) +def generate_api_key(): + username = session['user']['preferred_username'] + new_key = secrets.token_hex(32) + conn = get_connection() + conn.set_api_key(username, new_key) + flash("Successfully updated API key.", "alert-success") + + return render_template('auth/generate_api_key.html', api_key = new_key) @@ -406,46 +416,10 @@ def profile(): -@auth_blueprint.route('/login') -def login(): - ## you can not use the regular login route during testing because this redirects to the keycloak form which can not - ## be filled computationally - #if current_app.config.get('TESTING', False): - # issuer = current_app.config['ISSUER'] - # url = f'{issuer}/protocol/openid-connect/token' - # data = {'client_id':current_app.config['CLIENTID'], 'client_secret': current_app.config['CLIENTSECRET'], 'grant_type': 'password', 'username': 'superuser', 'password': '12345'} - # token_response = requests.post(url = url, data=data) - # assert token_response.status_code == 200 - # session['tokenResponse'] = token_response.json() - # - # url = f'{issuer}/protocol/openid-connect/userinfo' - # data = {'token': session["tokenResponse"]["access_token"], 'token_type_hint': 'access_token', 'client_secret': current_app.config['CLIENTSECRET'], 'client_id': current_app.config['CLIENTID']} - # header = {'Authorization': f'Bearer {session["tokenResponse"]["access_token"]}'} - # user_response = requests.post(url = url, data=data, headers=header) - # assert user_response.status_code == 200 - # user_info = user_response.json() - # - # # this is only to record which user made which actions in the database and has nothing to do with authenitication - # username = user_info['preferred_username'] - # first_name = user_info['given_name'] - # last_name = user_info['family_name'] - # affiliation = user_info.get('affiliation') - # - # # init the session - # session['user'] = user_info - # - # if affiliation is None or affiliation.strip() == '': - # flash('LOGIN ERROR: You are missing the affiliation tag ask a HerediVar administrator to add it!', 'alert-danger') - # current_app.logger.error("Could not login user " + username + ", because the user was missing affiliation tag in keycloak.") - # return redirect(url_for('auth.logout', auto_logout='True')) - # conn = Connection(session['user']['roles']) - # conn.insert_user(username, first_name, last_name, affiliation) # this inserts only if the user is not already in the database and updates the information if the information changed (except for username this one has to stay) - # user_info['user_id'] = conn.get_user_id(username) - # conn.close() - # - # return save_redirect(request.args.get('next_login', url_for('main.index'))) +@auth_blueprint.route('/login') +def login(): # construct redirect uri: first redirect to keycloak login page # then redirect to auth with the next param which defaults to the '/' route # auth itself redirects to next ie. the page which required a login @@ -469,8 +443,6 @@ def auth(): #claims = jwt.decode(token_response['id_token'], keys, claims_cls=CodeIDToken) #claims.validate() - #print(token_response) - if token_response: user_info = token_response['userinfo'] diff --git a/src/frontend_celery/webapp/templates/auth/generate_api_key.html b/src/frontend_celery/webapp/templates/auth/generate_api_key.html new file mode 100644 index 00000000..602d20ec --- /dev/null +++ b/src/frontend_celery/webapp/templates/auth/generate_api_key.html @@ -0,0 +1,33 @@ +{% extends 'base.html' %} + + +{% block content %} + +
+ + +

{% block title %} Edit your personal information {% endblock %}

+ + +
+ Copy the key somewhere safe it is only shown once on this page!
+ If you lose your API key you have to generate a new one. +
+
+
+ {{api_key}} +
+
+ +
+ Documentation on how to access the HerediVar API can be found here +
+ + +
+ + +{% endblock %} + +{% block special_scripts %} +{% endblock %} \ No newline at end of file diff --git a/src/frontend_celery/webapp/templates/auth/profile.html b/src/frontend_celery/webapp/templates/auth/profile.html index e0b30b05..f571ef5b 100644 --- a/src/frontend_celery/webapp/templates/auth/profile.html +++ b/src/frontend_celery/webapp/templates/auth/profile.html @@ -29,10 +29,20 @@

{% block title %} Edit your personal information {% endblock %}

Affiliation + + API key + +
+ +
+ (re-) generate +
+ + - + diff --git a/src/frontend_celery/webapp/templates/index.html b/src/frontend_celery/webapp/templates/index.html index b1a04539..cb900190 100644 --- a/src/frontend_celery/webapp/templates/index.html +++ b/src/frontend_celery/webapp/templates/index.html @@ -111,6 +111,9 @@

Changelog

  • Previously, moderate benign criteria were colored in pink. Now, very strong benign criteria are shown in this color.
  • Removed extra download for structural variants. Instead they are now combined in one file with small variants. Also, IGV now shows one track for all classified variants.
    • +
  • Improved session security
    • +
  • Added an API endpoint for access of consensus classifications programmatically
    • +
  • Added API documentation
    • Bugfixes: