From 0a7d30dbbfe5e8f64104591846077586ff14e543 Mon Sep 17 00:00:00 2001 From: john681611 Date: Mon, 23 Oct 2023 15:26:41 +0100 Subject: [PATCH] Added: DB seperation for smaller inital response and cache extras --- application/database/db.py | 36 +++++-- application/tests/db_test.py | 186 ++++++++++++++++++++++++++++++++++- 2 files changed, 213 insertions(+), 9 deletions(-) diff --git a/application/database/db.py b/application/database/db.py index 0a4a54653..f656cf1b5 100644 --- a/application/database/db.py +++ b/application/database/db.py @@ -494,6 +494,7 @@ def format_path_record(rec): @classmethod def standards(self) -> List[str]: + # TODO (JOHN) REDUCE DUPLICATION & SIMPLIFY tools = [] for x in db.cypher_query("""MATCH (n:NeoTool) RETURN DISTINCT n.name""")[0]: tools.extend(x) @@ -1774,31 +1775,54 @@ def gap_analysis( if base_standard is None: return None grouped_paths = {} + extra_paths_dict = {} + GA_STRONG_UPPER_LIMIT = 2 + for node in base_standard: key = node.id if key not in grouped_paths: - grouped_paths[key] = {"start": node, "paths": {}} + grouped_paths[key] = {"start": node, "paths": {}, "extra": 0} + extra_paths_dict[key] = {"paths": {}} for path in paths: key = path["start"].id end_key = path["end"].id path["score"] = get_path_score(path) del path["start"] - if end_key in grouped_paths[key]["paths"]: - if grouped_paths[key]["paths"][end_key]["score"] > path["score"]: + if path["score"] <= GA_STRONG_UPPER_LIMIT: + if end_key in extra_paths_dict[key]['paths']: + del extra_paths_dict[key]['paths'][end_key] + grouped_paths[key]["extra"] -= 1 + if end_key in grouped_paths[key]["paths"]: + if grouped_paths[key]["paths"][end_key]["score"] > path["score"]: + grouped_paths[key]["paths"][end_key] = path + else: grouped_paths[key]["paths"][end_key] = path else: - grouped_paths[key]["paths"][end_key] = path + if end_key in grouped_paths[key]["paths"]: + continue + if end_key in extra_paths_dict[key]: + if extra_paths_dict[key]["paths"][end_key]["score"] > path["score"]: + extra_paths_dict[key]["paths"][end_key] = path + else: + extra_paths_dict[key]["paths"][end_key] = path + grouped_paths[key]["extra"] += 1 if ( store_in_cache ): # lightweight memory option to not return potentially huge object and instead store in a cache, # in case this is called via worker, we save both this and the caller memory by avoiding duplicate object in mem + # TODO (JOHN) MOCK AND TEST REDIS CALLS conn = redis.from_url(os.getenv("REDIS_URL", "redis://localhost:6379")) if cache_key == "": cache_key = make_array_hash(node_names) conn.set(cache_key, flask_json.dumps({"result": grouped_paths})) - return (node_names, {}) + for key in extra_paths_dict: + conn.set( + cache_key + "->" + key, + flask_json.dumps({"result": extra_paths_dict[key]}), + ) + return (node_names, {}, {}) - return (node_names, grouped_paths) + return (node_names, grouped_paths, extra_paths_dict) diff --git a/application/tests/db_test.py b/application/tests/db_test.py index 6ab6402c9..15df2ac89 100644 --- a/application/tests/db_test.py +++ b/application/tests/db_test.py @@ -1155,7 +1155,7 @@ def test_gap_analysis_no_nodes(self, gap_mock): gap_mock.return_value = ([], []) self.assertEqual( - db.gap_analysis(collection.neo_db, ["a", "b"]), (["a", "b"], {}) + db.gap_analysis(collection.neo_db, ["a", "b"]), (["a", "b"], {}, {}) ) @patch.object(db.NEO_DB, "gap_analysis") @@ -1166,7 +1166,11 @@ def test_gap_analysis_no_links(self, gap_mock): gap_mock.return_value = ([defs.CRE(name="bob", id=1)], []) self.assertEqual( db.gap_analysis(collection.neo_db, ["a", "b"]), - (["a", "b"], {1: {"start": defs.CRE(name="bob", id=1), "paths": {}}}), + ( + ["a", "b"], + {1: {"start": defs.CRE(name="bob", id=1), "paths": {}, "extra": 0}}, + {1: {'paths': {}}} + ), ) @patch.object(db.NEO_DB, "gap_analysis") @@ -1203,9 +1207,55 @@ def test_gap_analysis_one_link(self, gap_mock): "paths": { 2: {"end": defs.CRE(name="bob", id=2), "path": path, "score": 0} }, + "extra": 0, } }, + {1: {'paths': {}}} + ) + self.assertEqual(db.gap_analysis(collection.neo_db, ["a", "b"]), expected) + + @patch.object(db.NEO_DB, "gap_analysis") + def test_gap_analysis_one_weak_link(self, gap_mock): + collection = db.Node_collection() + collection.neo_db.connected = True + path = [ + { + "end": defs.CRE(name="bob", id=1), + "relationship": "LINKED_TO", + "start": defs.CRE(name="bob", id="a"), + }, + { + "end": defs.CRE(name="bob", id=2), + "relationship": "RELATED", + "start": defs.CRE(name="bob", id=1), + }, + { + "end": defs.CRE(name="bob", id=1), + "relationship": "RELATED", + "start": defs.CRE(name="bob", id=2), + }, + { + "end": defs.CRE(name="bob", id=3), + "relationship": "LINKED_TO", + "start": defs.CRE(name="bob", id=2), + }, + ] + gap_mock.return_value = ( + [defs.CRE(name="bob", id=1)], + [ + { + "start": defs.CRE(name="bob", id=1), + "end": defs.CRE(name="bob", id=2), + "path": path, + } + ], + ) + expected = ( + ["a", "b"], + {1: {"start": defs.CRE(name="bob", id=1), "paths": {}, "extra": 1}}, + {1: {'paths': { 2: {"end": defs.CRE(name="bob", id=2), "path": path, "score": 4}}}} ) + self.assertEqual(db.gap_analysis(collection.neo_db, ["a", "b"]), expected) @patch.object(db.NEO_DB, "gap_analysis") @@ -1259,8 +1309,73 @@ def test_gap_analysis_duplicate_link_path_existing_lower(self, gap_mock): "paths": { 2: {"end": defs.CRE(name="bob", id=2), "path": path, "score": 0} }, - } + "extra": 0, + }, + }, + {1: {'paths': {}}} + ) + self.assertEqual(db.gap_analysis(collection.neo_db, ["a", "b"]), expected) + + @patch.object(db.NEO_DB, "gap_analysis") + def test_gap_analysis_duplicate_link_path_existing_lower_new_in_extras(self, gap_mock): + collection = db.Node_collection() + collection.neo_db.connected = True + path = [ + { + "end": defs.CRE(name="bob", id=1), + "relationship": "LINKED_TO", + "start": defs.CRE(name="bob", id="a"), + }, + { + "end": defs.CRE(name="bob", id=2), + "relationship": "LINKED_TO", + "start": defs.CRE(name="bob", id="a"), + }, + ] + path2 = [ + { + "end": defs.CRE(name="bob", id=1), + "relationship": "LINKED_TO", + "start": defs.CRE(name="bob", id="a"), + }, + { + "end": defs.CRE(name="bob", id=2), + "relationship": "RELATED", + "start": defs.CRE(name="bob", id="a"), + }, + { + "end": defs.CRE(name="bob", id=2), + "relationship": "RELATED", + "start": defs.CRE(name="bob", id="a"), + }, + ] + gap_mock.return_value = ( + [defs.CRE(name="bob", id=1)], + [ + { + "start": defs.CRE(name="bob", id=1), + "end": defs.CRE(name="bob", id=2), + "path": path, + }, + { + "start": defs.CRE(name="bob", id=1), + "end": defs.CRE(name="bob", id=2), + "path": path2, + }, + ], + ) + expected = ( + ["a", "b"], + { + 1: { + "start": defs.CRE(name="bob", id=1), + "paths": { + 2: {"end": defs.CRE(name="bob", id=2), "path": path, "score": 0} + }, + "extra": 0, + }, }, + {1: {'paths': {}}} ) self.assertEqual(db.gap_analysis(collection.neo_db, ["a", "b"]), expected) @@ -1315,8 +1430,73 @@ def test_gap_analysis_duplicate_link_path_existing_higher(self, gap_mock): "paths": { 2: {"end": defs.CRE(name="bob", id=2), "path": path, "score": 0} }, + "extra": 0, + } + }, + {1: {'paths': {}}} + ) + self.assertEqual(db.gap_analysis(collection.neo_db, ["a", "b"]), expected) + + @patch.object(db.NEO_DB, "gap_analysis") + def test_gap_analysis_duplicate_link_path_existing_higher_and_in_extras(self, gap_mock): + collection = db.Node_collection() + collection.neo_db.connected = True + path = [ + { + "end": defs.CRE(name="bob", id=1), + "relationship": "LINKED_TO", + "start": defs.CRE(name="bob", id="a"), + }, + { + "end": defs.CRE(name="bob", id=2), + "relationship": "LINKED_TO", + "start": defs.CRE(name="bob", id="a"), + }, + ] + path2 = [ + { + "end": defs.CRE(name="bob", id=1), + "relationship": "LINKED_TO", + "start": defs.CRE(name="bob", id="a"), + }, + { + "end": defs.CRE(name="bob", id=2), + "relationship": "RELATED", + "start": defs.CRE(name="bob", id="a"), + }, + { + "end": defs.CRE(name="bob", id=2), + "relationship": "RELATED", + "start": defs.CRE(name="bob", id="a"), + }, + ] + gap_mock.return_value = ( + [defs.CRE(name="bob", id=1)], + [ + { + "start": defs.CRE(name="bob", id=1), + "end": defs.CRE(name="bob", id=2), + "path": path2, + }, + { + "start": defs.CRE(name="bob", id=1), + "end": defs.CRE(name="bob", id=2), + "path": path, + }, + ], + ) + expected = ( + ["a", "b"], + { + 1: { + "start": defs.CRE(name="bob", id=1), + "paths": { + 2: {"end": defs.CRE(name="bob", id=2), "path": path, "score": 0} + }, + "extra": 0, } }, + {1: {'paths': {}}} ) self.assertEqual(db.gap_analysis(collection.neo_db, ["a", "b"]), expected)