improved session cookie security & first implementation of api

resources/update_database/update_database_10.sql

@@ -4,3 +4,6 @@ ADD COLUMN `version` VARCHAR(45) NOT NULL AFTER `alias`;
 ALTER TABLE `HerediVar_ahdoebm1`.`classification_scheme_alias` 
 ADD UNIQUE INDEX `classification_scheme_alias_unique` (`alias` ASC, `version` ASC);
 ;
+
+ALTER TABLE `HerediVar_ahdoebm1`.`user` 
+ADD COLUMN `api_key` VARCHAR(64) NULL AFTER `affiliation`;

src/common/db_IO.py

@@ -1998,6 +1998,19 @@ def get_user(self, user_id):
         result = self.cursor.fetchone()
         return result
 
+    def set_api_key(self, username, api_key):
+        command = "UPDATE user SET api_key = %s WHERE username = %s"
+        self.cursor.execute(command, (api_key, username))
+        self.conn.commit()
+
+    def check_api_key(self, api_key: str, username: str) -> bool:
+        command = "SELECT EXISTS (SELECT * FROM user WHERE api_key = %s AND username = %s)"
+        self.cursor.execute(command, (api_key, username))
+        result = self.cursor.fetchone()[0]
+        if result == 1:
+            return True
+        return False
+
     def parse_raw_user(self, raw_user):
         return models.User(id = raw_user[0], 
                    full_name = raw_user[2] + ' ' + raw_user[3], 

src/common/heredicare_interface.py

@@ -352,10 +352,8 @@ def get_submission_status(self, submission_id):
             message = "ERROR: HerediCare API getsubmission id endpoint endpoint returned an HTTP " + str(resp.status_code) + " error: " + self.extract_error_message(resp.text)
             status = "api_error"
         else: # success
-            print(resp.text)
             resp = resp.json(strict=False)
             items = resp["items"]
-            print(items)
 
             if len(items) == 0: # submission id was generated but no data was posted yet
                 status = "pending"
@@ -446,21 +444,10 @@ def get_variant_items(self, variant, vid, submission_id, post_regexes, transacti
                 for consequence in consequences:
                     if consequence.transcript.gene is None:
                         continue
-                    if consequence.transcript.gene == preferred_gene:
+                    if consequence.transcript.gene.symbol == preferred_gene:
                         preferred_consequence = consequence
                         preferred_transcript = consequence.transcript.name
                         break
-            #preferred_consequences = variant.get_preferred_transcripts(within_gene = True)
-            #if preferred_consequences is not None:
-            #    preferred_genes = functions.get_preferred_genes()
-            #    for consequence in preferred_consequences:
-            #        if consequence.transcript.gene.symbol in preferred_genes:
-            #            preferred_consequence = consequence
-            #            break
-            #        elif preferred_consequence is None:
-            #            preferred_consequence = consequence
-            #        elif preferred_consequence.length < consequence.length:
-            #            preferred_consequence = consequence
 
             if preferred_consequence is None:
                 status = "skipped"
@@ -681,8 +668,9 @@ def get_data(self, variant, vid, options):
 
         # tinker all items together
         data = {'items': all_items}
-        with open('/mnt/storage2/users/ahdoebm1/HerediVar/src/common/heredicare_interface_debug/sub' + str(submission_id) + '.json', "w") as f:
-            functions.prettyprint_json(data, f.write)
+        if os.environ.get('WEBAPP_ENV', '') == 'dev':
+            with open('/mnt/storage2/users/ahdoebm1/HerediVar/src/common/heredicare_interface_debug/sub' + str(submission_id) + '.json', "w") as f:
+                functions.prettyprint_json(data, f.write)
         data = json.dumps(data)
 
         return data, vid, submission_id, status, message

src/frontend_celery/config.py

@@ -3,7 +3,15 @@
 import sys
 sys.path.append(path.dirname(path.dirname(path.abspath(__file__))))
 import common.functions as functions
-
+from redis import Redis
+from redis.backoff import ExponentialBackoff
+from redis.retry import Retry
+from redis.client import Redis
+from redis.exceptions import (
+    BusyLoadingError,
+    ConnectionError,
+    TimeoutError
+)
 
 functions.read_dotenv()
 
@@ -40,11 +48,17 @@ class Config(object):
     DISCOVERYURL = f'{ISSUER}/.well-known/openid-configuration'
 
     # configuration of server side session from flask-session module
-    SESSION_PERMANENT = False
-    SESSION_TYPE = "filesystem"
-    #SESSION_COOKIE_SECURE = True
-    SESSION_USE_SIGNER = True
-    SESSION_FILE_DIR = os.path.dirname(os.path.abspath(__file__)) + "/flask_sessions"
+    SESSION_PERMANENT = True
+
+    SESSION_TYPE = 'redis'
+    retry = Retry(ExponentialBackoff(), 3)
+    SESSION_REDIS = Redis.from_url('redis://localhost:6379/0', retry=retry, retry_on_error=[BusyLoadingError, ConnectionError, TimeoutError]) # use exponential backoff with 3 tries on specific redis errors
+
+    #SESSION_USE_SIGNER = True # deprecated
+    #SESSION_TYPE = "filesystem"
+    #SESSION_FILE_DIR = os.path.dirname(os.path.abspath(__file__)) + "/flask_sessions"
+    SESSION_COOKIE_SAMESITE = "Lax"
+    SESSION_COOKIE_HTTPONLY = True
 
     # other folders
     #RESOURCES_FOLDER = 'resources/'
@@ -71,15 +85,18 @@ class Config(object):
 
 
 class ProdConfig(Config):
-    KEYCLOAK_HOST = os.environ.get('KEYCLOAK_HOST', 'localhost')
     # keycloak
+    KEYCLOAK_HOST = os.environ.get('KEYCLOAK_HOST', 'localhost')
     KEYCLOAK_PORT = '8080'
     KEYCLOAK_BASEPATH = "https://"+KEYCLOAK_HOST+"/kc"
     ISSUER = os.environ.get('ISSUER', KEYCLOAK_BASEPATH + "/realms/HerediVar")
     CLIENTID = os.environ.get('CLIENT_ID')
     CLIENTSECRET = os.environ.get('CLIENT_SECRET')
     DISCOVERYURL = f'{ISSUER}/.well-known/openid-configuration'
 
+    # sesssion
+    SESSION_COOKIE_SECURE = True
+    #SESSION_COOKIE_NAME = "__Host-"
 
 
 class DevConfig(Config):

src/frontend_celery/webapp/__init__.py

@@ -70,6 +70,7 @@ def create_app():
     from .user import create_module as user_create_module
     from .errorhandlers import create_module as errorhandlers_create_module
     from .upload import create_module as upload_create_module
+    from .api import create_module as api_create_module
 
     main_create_module(app)
     variant_create_module(app)
@@ -79,6 +80,7 @@ def create_app():
     user_create_module(app)
     errorhandlers_create_module(app)
     upload_create_module(app)
+    api_create_module(app)
 
 
     configure_logging(app)

src/frontend_celery/webapp/api/__init__.py

@@ -0,0 +1,3 @@
+def create_module(app, **kwargs):
+    from .api_routes import api_blueprint
+    app.register_blueprint(api_blueprint)

src/frontend_celery/webapp/api/api_routes.py

@@ -0,0 +1,82 @@
+import json
+import os
+import sys
+import requests
+from flask import url_for, session, request, Blueprint, current_app
+from flask import render_template, redirect, jsonify
+from flask_session import Session
+sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
+import common.functions as functions
+from common.db_IO import Connection
+from ..utils import *
+
+
+api_blueprint = Blueprint(
+    'api',
+    __name__
+)
+
+
+@api_blueprint.route('/api/v1.0/get/consensus_classification', methods=['GET'])
+@accept_token
+def consensus_classification():
+
+    conn = Connection(roles = ["read_only"])
+
+    variant_id = request.args.get('variant_id')
+    if variant_id is None:
+        chrom = request.args.get('chrom')
+        pos = request.args.get('pos')
+        ref = request.args.get('ref')
+        alt = request.args.get('alt')
+
+        variant_id = conn.get_variant_id(chrom, pos, ref, alt)
+
+    if variant_id is None:
+        conn.close()
+        abort(404, "Requested variant does not exist or missing variant information")
+
+    variant = conn.get_variant(variant_id, include_annotations = False, include_consensus = True, include_user_classifications = False, include_heredicare_classifications=False, include_automatic_classification=False, include_clinvar=False, include_consequences=False, include_assays=False, include_literature=False, include_external_ids=False)
+    conn.close()
+
+
+    v_res = prepare_variant(variant)
+    mrcc = variant.get_recent_consensus_classification()
+    mrcc_res = prepare_classification(mrcc)
+
+    result = {
+        "variant": v_res,
+        "classification": mrcc_res
+    }
+
+    return jsonify(result)
+
+
+def prepare_variant(variant):
+    result = {
+        "id": variant.id,
+        "chrom": variant.chrom,
+        "pos": variant.pos,
+        "ref": variant.ref,
+        "alt": variant.alt,
+        "variant_type": variant.variant_type,
+        "hidden": variant.is_hidden
+    }
+    return result
+
+def prepare_classification(classification):
+    result = {
+        "comment": classification.comment,
+        "date": classification.date,
+        "literature": classification.literature,
+        "scheme": {"name": classification.scheme.display_name, "reference": classification.scheme.reference},
+        "criteria": classification.scheme.criteria,
+        "class_by_scheme": classification.scheme.selected_class,
+        "selected_class": classification.selected_class,
+        "classification_type": classification.type
+    }
+    return result
+
+
+
+

src/frontend_celery/webapp/auth/auth_routes.py

@@ -395,8 +395,18 @@ def profile():
     return render_template('auth/profile.html', user = user)
 
 
+@auth_blueprint.route('/generate_api_key', methods=['GET', 'POST'])
+@require_permission(["read_resources"])
+def generate_api_key():
+    username = session['user']['preferred_username']
+    new_key = secrets.token_hex(32)
 
+    conn = get_connection()
+    conn.set_api_key(username, new_key)
 
+    flash("Successfully updated API key.", "alert-success")
+
+    return render_template('auth/generate_api_key.html', api_key = new_key)
 
 
 
@@ -406,46 +416,10 @@ def profile():
 
 
 
-@auth_blueprint.route('/login')
-def login():
 
-    ## you can not use the regular login route during testing because this redirects to the keycloak form which can not
-    ## be filled computationally
-    #if current_app.config.get('TESTING', False):
-    #    issuer = current_app.config['ISSUER']
-    #    url = f'{issuer}/protocol/openid-connect/token'
-    #    data = {'client_id':current_app.config['CLIENTID'], 'client_secret': current_app.config['CLIENTSECRET'], 'grant_type': 'password', 'username': 'superuser', 'password': '12345'}
-    #    token_response = requests.post(url = url, data=data)
-    #    assert token_response.status_code == 200
-    #    session['tokenResponse'] = token_response.json()
-    #
-    #    url = f'{issuer}/protocol/openid-connect/userinfo'
-    #    data = {'token': session["tokenResponse"]["access_token"], 'token_type_hint': 'access_token', 'client_secret': current_app.config['CLIENTSECRET'], 'client_id': current_app.config['CLIENTID']}
-    #    header = {'Authorization': f'Bearer {session["tokenResponse"]["access_token"]}'}
-    #    user_response = requests.post(url = url, data=data, headers=header)
-    #    assert user_response.status_code == 200
-    #    user_info = user_response.json()
-    #    
-    #    # this is only to record which user made which actions in the database and has nothing to do with authenitication
-    #    username = user_info['preferred_username']
-    #    first_name = user_info['given_name']
-    #    last_name = user_info['family_name']
-    #    affiliation = user_info.get('affiliation')
-    #
-    #    # init the session
-    #    session['user'] = user_info
-    #
-    #    if affiliation is None or affiliation.strip() == '':
-    #        flash('LOGIN ERROR: You are missing the affiliation tag ask a HerediVar administrator to add it!', 'alert-danger')
-    #        current_app.logger.error("Could not login user " + username + ", because the user was missing affiliation tag in keycloak.")
-    #        return redirect(url_for('auth.logout', auto_logout='True'))
-    #    conn = Connection(session['user']['roles'])
-    #    conn.insert_user(username, first_name, last_name, affiliation) # this inserts only if the user is not already in the database and updates the information if the information changed (except for username this one has to stay)
-    #    user_info['user_id'] = conn.get_user_id(username)
-    #    conn.close()
-    #
-    #    return save_redirect(request.args.get('next_login', url_for('main.index')))
 
+@auth_blueprint.route('/login')
+def login():
     # construct redirect uri: first redirect to keycloak login page
     # then redirect to auth with the next param which defaults to the '/' route
     # auth itself redirects to next ie. the page which required a login
@@ -469,8 +443,6 @@ def auth():
     #claims = jwt.decode(token_response['id_token'], keys, claims_cls=CodeIDToken)
     #claims.validate()
 
-    #print(token_response)
-
 
     if token_response:
         user_info = token_response['userinfo']

src/frontend_celery/webapp/templates/auth/generate_api_key.html

@@ -0,0 +1,33 @@
+{% extends 'base.html' %}
+
+
+{% block content %}
+
+<div class="container">
+
+
+<h1> {% block title %} Edit your personal information {% endblock %} </h1>
+
+
+<div class="alert alert-danger">
+    Copy the key somewhere safe it is only shown <strong>once on this page</strong>! <br>
+    If you lose your API key you have to generate a new one.
+</div>
+<div class="d-flex justify-content-center">
+    <div class="width_very_large alert alert-secondary text_align_center">
+        {{api_key}}
+    </div>
+</div>
+
+<div>
+    Documentation on how to access the HerediVar API can be found <a href="{{url_for('main.documentation')}}">here</a>
+</div>
+
+
+</div>
+
+
+{% endblock %}
+
+{% block special_scripts %}
+{% endblock %}

src/frontend_celery/webapp/templates/auth/profile.html

@@ -29,10 +29,20 @@ <h1> {% block title %} Edit your personal information {% endblock %} </h1>
                     <td class="width_small vertical_align_middle">Affiliation</td>
                     <td><input class="form-control" type="text" value="{{ user.get('attributes', {}).get('affiliation', [''])[0] }}" name="affiliation"></td>
                 </tr>
+                <tr>
+                    <td class="width_small vertical_align_middle">API key</td>
+                    <td>
+                        <div class="d-flex">
+                            <input class="form-control width_very_large" type="text"value="" name="" disabled>
+                            <div class="flex-grow-1"></div>
+                            <a href="{{url_for('auth.generate_api_key')}}" class="btn btn-primary" role="button">(re-) generate</a>
+                        </div>
+                    </td>
+                </tr>
             </tbody>
         </table>
     </div>
-    <button class="btn btn-primary">Submit</button>
+    <button class="btn btn-primary" role="submit">Submit</button>
 </form>
 
 

src/frontend_celery/webapp/templates/index.html

@@ -111,6 +111,9 @@ <h4>Changelog</h4>
             </li>
             <li>Previously, moderate benign criteria were colored in pink. Now, very strong benign criteria are shown in this color.</li>
             <li>Removed extra download for structural variants. Instead they are now combined in one file with small variants. Also, IGV now shows one track for all classified variants.</li>
+            <li>Improved session security</li>
+            <li>Added an API endpoint for access of consensus classifications programmatically</li>
+            <li>Added API documentation</li>
         </ul>
         Bugfixes:
         <ul>