Skip to content

Commit

Permalink
Added: DB seperation for smaller inital response and cache extras
Browse files Browse the repository at this point in the history
  • Loading branch information
@john681611
john681611 committed Oct 23, 2023
1 parent defc518 commit 0a7d30d
Show file tree
Hide file tree
Showing 2 changed files with 213 additions and 9 deletions.
36 changes: 30 additions & 6 deletions application/database/db.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -494,6 +494,7 @@ def format_path_record(rec):

@classmethod
def standards(self) -> List[str]:
# TODO (JOHN) REDUCE DUPLICATION & SIMPLIFY
tools = []
for x in db.cypher_query("""MATCH (n:NeoTool) RETURN DISTINCT n.name""")[0]:
tools.extend(x)
Expand Down Expand Up @@ -1774,31 +1775,54 @@ def gap_analysis(
if base_standard is None:
return None
grouped_paths = {}
extra_paths_dict = {}
GA_STRONG_UPPER_LIMIT = 2

for node in base_standard:
key = node.id
if key not in grouped_paths:
grouped_paths[key] = {"start": node, "paths": {}}
grouped_paths[key] = {"start": node, "paths": {}, "extra": 0}
extra_paths_dict[key] = {"paths": {}}

for path in paths:
key = path["start"].id
end_key = path["end"].id
path["score"] = get_path_score(path)
del path["start"]
if end_key in grouped_paths[key]["paths"]:
if grouped_paths[key]["paths"][end_key]["score"] > path["score"]:
if path["score"] <= GA_STRONG_UPPER_LIMIT:
if end_key in extra_paths_dict[key]['paths']:
del extra_paths_dict[key]['paths'][end_key]
grouped_paths[key]["extra"] -= 1
if end_key in grouped_paths[key]["paths"]:
if grouped_paths[key]["paths"][end_key]["score"] > path["score"]:
grouped_paths[key]["paths"][end_key] = path
else:
grouped_paths[key]["paths"][end_key] = path
else:
grouped_paths[key]["paths"][end_key] = path
if end_key in grouped_paths[key]["paths"]:
continue
if end_key in extra_paths_dict[key]:
if extra_paths_dict[key]["paths"][end_key]["score"] > path["score"]:
extra_paths_dict[key]["paths"][end_key] = path
else:
extra_paths_dict[key]["paths"][end_key] = path
grouped_paths[key]["extra"] += 1

if (
store_in_cache
): # lightweight memory option to not return potentially huge object and instead store in a cache,
# in case this is called via worker, we save both this and the caller memory by avoiding duplicate object in mem
# TODO (JOHN) MOCK AND TEST REDIS CALLS
conn = redis.from_url(os.getenv("REDIS_URL", "redis://localhost:6379"))
if cache_key == "":
cache_key = make_array_hash(node_names)

conn.set(cache_key, flask_json.dumps({"result": grouped_paths}))
return (node_names, {})
for key in extra_paths_dict:
conn.set(
cache_key + "->" + key,
flask_json.dumps({"result": extra_paths_dict[key]}),
)
return (node_names, {}, {})

return (node_names, grouped_paths)
return (node_names, grouped_paths, extra_paths_dict)
186 changes: 183 additions & 3 deletions application/tests/db_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1155,7 +1155,7 @@ def test_gap_analysis_no_nodes(self, gap_mock):

gap_mock.return_value = ([], [])
self.assertEqual(
db.gap_analysis(collection.neo_db, ["a", "b"]), (["a", "b"], {})
db.gap_analysis(collection.neo_db, ["a", "b"]), (["a", "b"], {}, {})
)

@patch.object(db.NEO_DB, "gap_analysis")
Expand All @@ -1166,7 +1166,11 @@ def test_gap_analysis_no_links(self, gap_mock):
gap_mock.return_value = ([defs.CRE(name="bob", id=1)], [])
self.assertEqual(
db.gap_analysis(collection.neo_db, ["a", "b"]),
(["a", "b"], {1: {"start": defs.CRE(name="bob", id=1), "paths": {}}}),
(
["a", "b"],
{1: {"start": defs.CRE(name="bob", id=1), "paths": {}, "extra": 0}},
{1: {'paths': {}}}
),
)

@patch.object(db.NEO_DB, "gap_analysis")
Expand Down Expand Up @@ -1203,9 +1207,55 @@ def test_gap_analysis_one_link(self, gap_mock):
"paths": {
2: {"end": defs.CRE(name="bob", id=2), "path": path, "score": 0}
},
"extra": 0,
}
},
{1: {'paths': {}}}
)
self.assertEqual(db.gap_analysis(collection.neo_db, ["a", "b"]), expected)

@patch.object(db.NEO_DB, "gap_analysis")
def test_gap_analysis_one_weak_link(self, gap_mock):
collection = db.Node_collection()
collection.neo_db.connected = True
path = [
{
"end": defs.CRE(name="bob", id=1),
"relationship": "LINKED_TO",
"start": defs.CRE(name="bob", id="a"),
},
{
"end": defs.CRE(name="bob", id=2),
"relationship": "RELATED",
"start": defs.CRE(name="bob", id=1),
},
{
"end": defs.CRE(name="bob", id=1),
"relationship": "RELATED",
"start": defs.CRE(name="bob", id=2),
},
{
"end": defs.CRE(name="bob", id=3),
"relationship": "LINKED_TO",
"start": defs.CRE(name="bob", id=2),
},
]
gap_mock.return_value = (
[defs.CRE(name="bob", id=1)],
[
{
"start": defs.CRE(name="bob", id=1),
"end": defs.CRE(name="bob", id=2),
"path": path,
}
],
)
expected = (
["a", "b"],
{1: {"start": defs.CRE(name="bob", id=1), "paths": {}, "extra": 1}},
{1: {'paths': { 2: {"end": defs.CRE(name="bob", id=2), "path": path, "score": 4}}}}
)

self.assertEqual(db.gap_analysis(collection.neo_db, ["a", "b"]), expected)

@patch.object(db.NEO_DB, "gap_analysis")
Expand Down Expand Up @@ -1259,8 +1309,73 @@ def test_gap_analysis_duplicate_link_path_existing_lower(self, gap_mock):
"paths": {
2: {"end": defs.CRE(name="bob", id=2), "path": path, "score": 0}
},
}
"extra": 0,
},
},
{1: {'paths': {}}}
)
self.assertEqual(db.gap_analysis(collection.neo_db, ["a", "b"]), expected)

@patch.object(db.NEO_DB, "gap_analysis")
def test_gap_analysis_duplicate_link_path_existing_lower_new_in_extras(self, gap_mock):
collection = db.Node_collection()
collection.neo_db.connected = True
path = [
{
"end": defs.CRE(name="bob", id=1),
"relationship": "LINKED_TO",
"start": defs.CRE(name="bob", id="a"),
},
{
"end": defs.CRE(name="bob", id=2),
"relationship": "LINKED_TO",
"start": defs.CRE(name="bob", id="a"),
},
]
path2 = [
{
"end": defs.CRE(name="bob", id=1),
"relationship": "LINKED_TO",
"start": defs.CRE(name="bob", id="a"),
},
{
"end": defs.CRE(name="bob", id=2),
"relationship": "RELATED",
"start": defs.CRE(name="bob", id="a"),
},
{
"end": defs.CRE(name="bob", id=2),
"relationship": "RELATED",
"start": defs.CRE(name="bob", id="a"),
},
]
gap_mock.return_value = (
[defs.CRE(name="bob", id=1)],
[
{
"start": defs.CRE(name="bob", id=1),
"end": defs.CRE(name="bob", id=2),
"path": path,
},
{
"start": defs.CRE(name="bob", id=1),
"end": defs.CRE(name="bob", id=2),
"path": path2,
},
],
)
expected = (
["a", "b"],
{
1: {
"start": defs.CRE(name="bob", id=1),
"paths": {
2: {"end": defs.CRE(name="bob", id=2), "path": path, "score": 0}
},
"extra": 0,
},
},
{1: {'paths': {}}}
)
self.assertEqual(db.gap_analysis(collection.neo_db, ["a", "b"]), expected)

Expand Down Expand Up @@ -1315,8 +1430,73 @@ def test_gap_analysis_duplicate_link_path_existing_higher(self, gap_mock):
"paths": {
2: {"end": defs.CRE(name="bob", id=2), "path": path, "score": 0}
},
"extra": 0,
}
},
{1: {'paths': {}}}
)
self.assertEqual(db.gap_analysis(collection.neo_db, ["a", "b"]), expected)

@patch.object(db.NEO_DB, "gap_analysis")
def test_gap_analysis_duplicate_link_path_existing_higher_and_in_extras(self, gap_mock):
collection = db.Node_collection()
collection.neo_db.connected = True
path = [
{
"end": defs.CRE(name="bob", id=1),
"relationship": "LINKED_TO",
"start": defs.CRE(name="bob", id="a"),
},
{
"end": defs.CRE(name="bob", id=2),
"relationship": "LINKED_TO",
"start": defs.CRE(name="bob", id="a"),
},
]
path2 = [
{
"end": defs.CRE(name="bob", id=1),
"relationship": "LINKED_TO",
"start": defs.CRE(name="bob", id="a"),
},
{
"end": defs.CRE(name="bob", id=2),
"relationship": "RELATED",
"start": defs.CRE(name="bob", id="a"),
},
{
"end": defs.CRE(name="bob", id=2),
"relationship": "RELATED",
"start": defs.CRE(name="bob", id="a"),
},
]
gap_mock.return_value = (
[defs.CRE(name="bob", id=1)],
[
{
"start": defs.CRE(name="bob", id=1),
"end": defs.CRE(name="bob", id=2),
"path": path2,
},
{
"start": defs.CRE(name="bob", id=1),
"end": defs.CRE(name="bob", id=2),
"path": path,
},
],
)
expected = (
["a", "b"],
{
1: {
"start": defs.CRE(name="bob", id=1),
"paths": {
2: {"end": defs.CRE(name="bob", id=2), "path": path, "score": 0}
},
"extra": 0,
}
},
{1: {'paths': {}}}
)
self.assertEqual(db.gap_analysis(collection.neo_db, ["a", "b"]), expected)

Expand Down

0 comments on commit 0a7d30d

Please sign in to comment.