From fa9e18e8880e8da03f7ca7fde33dbcacd3fb13f7 Mon Sep 17 00:00:00 2001 From: john681611 Date: Mon, 23 Oct 2023 16:19:20 +0100 Subject: [PATCH] Added: DB side of part split response logic --- application/database/db.py | 4 +- .../src/pages/GapAnalysis/GapAnalysis.tsx | 1 - application/tests/db_test.py | 103 ++++++++++++++++-- application/tests/web_main_test.py | 32 +++++- application/web/web_main.py | 18 +++ 5 files changed, 138 insertions(+), 20 deletions(-) diff --git a/application/database/db.py b/application/database/db.py index f656cf1b5..717e08bda 100644 --- a/application/database/db.py +++ b/application/database/db.py @@ -1790,8 +1790,8 @@ def gap_analysis( path["score"] = get_path_score(path) del path["start"] if path["score"] <= GA_STRONG_UPPER_LIMIT: - if end_key in extra_paths_dict[key]['paths']: - del extra_paths_dict[key]['paths'][end_key] + if end_key in extra_paths_dict[key]["paths"]: + del extra_paths_dict[key]["paths"][end_key] grouped_paths[key]["extra"] -= 1 if end_key in grouped_paths[key]["paths"]: if grouped_paths[key]["paths"][end_key]["score"] > path["score"]: diff --git a/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx b/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx index b0f29b57a..4f2d12a5f 100644 --- a/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx +++ b/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx @@ -119,7 +119,6 @@ export const GapAnalysis = () => { ); const [gaJob, setgaJob] = useState(''); const [gapAnalysis, setGapAnalysis] = useState>(); - const [activeIndex, SetActiveIndex] = useState(); const [loadingStandards, setLoadingStandards] = useState(false); const [loadingGA, setLoadingGA] = useState(false); const [error, setError] = useState(null); diff --git a/application/tests/db_test.py b/application/tests/db_test.py index 15df2ac89..5142d5011 100644 --- a/application/tests/db_test.py +++ b/application/tests/db_test.py @@ -3,13 +3,14 @@ import os import tempfile import unittest +from unittest import mock from unittest.mock import patch import uuid from copy import copy, deepcopy from pprint import pprint -from pydoc import doc from typing import Any, Dict, List, Union -import neo4j +import redis +from flask import json as flask_json import yaml from application import create_app, sqla # type: ignore @@ -1169,7 +1170,7 @@ def test_gap_analysis_no_links(self, gap_mock): ( ["a", "b"], {1: {"start": defs.CRE(name="bob", id=1), "paths": {}, "extra": 0}}, - {1: {'paths': {}}} + {1: {"paths": {}}}, ), ) @@ -1210,7 +1211,7 @@ def test_gap_analysis_one_link(self, gap_mock): "extra": 0, } }, - {1: {'paths': {}}} + {1: {"paths": {}}}, ) self.assertEqual(db.gap_analysis(collection.neo_db, ["a", "b"]), expected) @@ -1253,7 +1254,13 @@ def test_gap_analysis_one_weak_link(self, gap_mock): expected = ( ["a", "b"], {1: {"start": defs.CRE(name="bob", id=1), "paths": {}, "extra": 1}}, - {1: {'paths': { 2: {"end": defs.CRE(name="bob", id=2), "path": path, "score": 4}}}} + { + 1: { + "paths": { + 2: {"end": defs.CRE(name="bob", id=2), "path": path, "score": 4} + } + } + }, ) self.assertEqual(db.gap_analysis(collection.neo_db, ["a", "b"]), expected) @@ -1312,12 +1319,14 @@ def test_gap_analysis_duplicate_link_path_existing_lower(self, gap_mock): "extra": 0, }, }, - {1: {'paths': {}}} + {1: {"paths": {}}}, ) self.assertEqual(db.gap_analysis(collection.neo_db, ["a", "b"]), expected) @patch.object(db.NEO_DB, "gap_analysis") - def test_gap_analysis_duplicate_link_path_existing_lower_new_in_extras(self, gap_mock): + def test_gap_analysis_duplicate_link_path_existing_lower_new_in_extras( + self, gap_mock + ): collection = db.Node_collection() collection.neo_db.connected = True path = [ @@ -1375,7 +1384,7 @@ def test_gap_analysis_duplicate_link_path_existing_lower_new_in_extras(self, gap "extra": 0, }, }, - {1: {'paths': {}}} + {1: {"paths": {}}}, ) self.assertEqual(db.gap_analysis(collection.neo_db, ["a", "b"]), expected) @@ -1433,12 +1442,14 @@ def test_gap_analysis_duplicate_link_path_existing_higher(self, gap_mock): "extra": 0, } }, - {1: {'paths': {}}} + {1: {"paths": {}}}, ) self.assertEqual(db.gap_analysis(collection.neo_db, ["a", "b"]), expected) @patch.object(db.NEO_DB, "gap_analysis") - def test_gap_analysis_duplicate_link_path_existing_higher_and_in_extras(self, gap_mock): + def test_gap_analysis_duplicate_link_path_existing_higher_and_in_extras( + self, gap_mock + ): collection = db.Node_collection() collection.neo_db.connected = True path = [ @@ -1496,10 +1507,80 @@ def test_gap_analysis_duplicate_link_path_existing_higher_and_in_extras(self, ga "extra": 0, } }, - {1: {'paths': {}}} + {1: {"paths": {}}}, ) self.assertEqual(db.gap_analysis(collection.neo_db, ["a", "b"]), expected) + @patch.object(redis, "from_url") + @patch.object(db.NEO_DB, "gap_analysis") + def test_gap_analysis_dump_to_cache(self, gap_mock, redis_conn_mock): + collection = db.Node_collection() + collection.neo_db.connected = True + path = [ + { + "end": defs.CRE(name="bob", id=1), + "relationship": "LINKED_TO", + "start": defs.CRE(name="bob", id="a"), + }, + { + "end": defs.CRE(name="bob", id=2), + "relationship": "RELATED", + "start": defs.CRE(name="bob", id=1), + }, + { + "end": defs.CRE(name="bob", id=1), + "relationship": "RELATED", + "start": defs.CRE(name="bob", id=2), + }, + { + "end": defs.CRE(name="bob", id=3), + "relationship": "LINKED_TO", + "start": defs.CRE(name="bob", id=2), + }, + ] + gap_mock.return_value = ( + [defs.CRE(name="bob", id="a")], + [ + { + "start": defs.CRE(name="bob", id="a"), + "end": defs.CRE(name="bob", id="b"), + "path": path, + } + ], + ) + + expected_response = ( + ["a", "b"], + {"a": {"start": defs.CRE(name="bob", id="a"), "paths": {}, "extra": 1}}, + { + "a": { + "paths": { + "b": { + "end": defs.CRE(name="bob", id="b"), + "path": path, + "score": 4, + } + } + } + }, + ) + response = db.gap_analysis(collection.neo_db, ["a", "b"], True) + + self.assertEqual(response, (expected_response[0], {}, {})) + + redis_conn_mock.return_value.set.assert_has_calls( + [ + mock.call( + "d8160c9b3dc20d4e931aeb4f45262155", + flask_json.dumps({"result": expected_response[1]}), + ), + mock.call( + "d8160c9b3dc20d4e931aeb4f45262155->a", + flask_json.dumps({"result": expected_response[2]["a"]}), + ), + ] + ) + def test_neo_db_parse_node_code(self): name = "name" id = "id" diff --git a/application/tests/web_main_test.py b/application/tests/web_main_test.py index 30f870c6f..c6f919947 100644 --- a/application/tests/web_main_test.py +++ b/application/tests/web_main_test.py @@ -1,18 +1,15 @@ import re import json -import logging -import os -import tempfile import unittest -from pprint import pprint -from typing import Any, Dict, List +from unittest.mock import patch + +import redis from application import create_app, sqla # type: ignore from application.database import db from application.defs import cre_defs as defs from application.defs import osib_defs from application.web import web_main -from application.utils import mdutils class TestMain(unittest.TestCase): @@ -568,3 +565,26 @@ def test_smartlink(self) -> None: location = head[1] self.assertEqual(location, "") self.assertEqual(404, response.status_code) + + # TODO: (JOHN) Basic gap analysis endpoint tests + + def test_gap_analysis_weak_links_no_cache(self) -> None: + with self.app.test_client() as client: + response = client.get( + "/rest/v1/map_analysis_weak_links?standard=aaa&standard=bbb&key=ccc`", + headers={"Content-Type": "application/json"}, + ) + self.assertEqual(404, response.status_code) + + @patch.object(redis, "from_url") + def test_gap_analysis_weak_links_response(self, redis_conn_mock) -> None: + expected = {"result": "hello"} + redis_conn_mock.return_value.exists.return_value = True + redis_conn_mock.return_value.get.return_value = json.dumps(expected) + with self.app.test_client() as client: + response = client.get( + "/rest/v1/map_analysis_weak_links?standard=aaa&standard=bbb&key=ccc`", + headers={"Content-Type": "application/json"}, + ) + self.assertEqual(200, response.status_code) + self.assertEqual(expected, json.loads(response.data)) diff --git a/application/web/web_main.py b/application/web/web_main.py index 13d71dbc7..1ff4c18d4 100644 --- a/application/web/web_main.py +++ b/application/web/web_main.py @@ -251,6 +251,24 @@ def gap_analysis() -> Any: return jsonify({"job_id": gap_analysis_job.id}) +@app.route("/rest/v1/map_analysis_weak_links", methods=["GET"]) +@cache.cached(timeout=50, query_string=True) +def gap_analysis_weak_links() -> Any: + standards = request.args.getlist("standard") + key = request.args.get("key") + redis_url = os.getenv("REDIS_URL", "redis://localhost:6379") + conn = redis.from_url(redis_url) + standards_hash = make_array_hash(standards) + cache_key = standards_hash + "->" + key + if conn.exists(cache_key): + gap_analysis_results = conn.get(cache_key) + if gap_analysis_results: + gap_analysis_dict = json.loads(gap_analysis_results) + if gap_analysis_dict.get("result"): + return jsonify({"result": gap_analysis_dict.get("result")}) + abort(404, "No such Cache") + + @app.route("/rest/v1/ma_job_results", methods=["GET"]) def fetch_job() -> Any: logger.info("fetching job results")