Skip to content

Commit

Permalink
More progress in conversion
Browse files Browse the repository at this point in the history
  • Loading branch information
@john681611
john681611 committed Oct 3, 2023
1 parent 0f73e1a commit d0c8683
Show file tree
Hide file tree
Showing 2 changed files with 110 additions and 100 deletions.
209 changes: 109 additions & 100 deletions application/database/db.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
Relationship,
RelationshipTo,
ArrayProperty,
StructuredRel,
db,
)
from sqlalchemy.orm import aliased
import os
Expand Down Expand Up @@ -168,43 +170,110 @@ class Embeddings(BaseModel): # type: ignore
)


class RelatedRel(StructuredRel):
pass


class ContainsRel(StructuredRel):
pass


class LinkedToRel(StructuredRel):
pass


class SameRel(StructuredRel):
pass


class NeoDocument(StructuredNode):
id = UniqueIdProperty()
name = StringProperty(required=True)
description = StringProperty(required=True)
tags = ArrayProperty(StringProperty())
doctype = StringProperty(required=True)
related = Relationship("NeoDocument", "RELATED")
related = Relationship("NeoDocument", "RELATED", model=RelatedRel)

@classmethod
def to_cre_def(self):
raise Exception(f"Shouldn't be parsing a NeoDocument")


class NeoNode(NeoDocument):
doctype = StringProperty()
version = StringProperty(required=True)
hyperlink = StringProperty()

@classmethod
def to_cre_def(self):
raise Exception(f"Shouldn't be parsing a NeoNode")


class NeoStandard(NeoNode):
section = StringProperty()
subsection = StringProperty(required=True)
section_id = StringProperty()

@classmethod
def to_cre_def(self) -> cre_defs.Standard:
return cre_defs.Standard(
name=self.name,
id=self.id,
description=self.description,
tags=self.tags,
hyperlink=self.hyperlink,
version=self.version,
section=self.section,
sectionID=self.section_id,
subsection=self.subsection,
)


class NeoTool(NeoStandard):
tooltype = StringProperty(required=True)

@classmethod
def to_cre_def(self) -> cre_defs.Tool:
return cre_defs.Tool(
name=self.name,
id=self.id,
description=self.description,
tags=self.tags,
hyperlink=self.hyperlink,
version=self.version,
section=self.section,
sectionID=self.section_id,
subsection=self.subsection,
)


class NeoCode(NeoNode):
pass
@classmethod
def to_cre_def(self) -> cre_defs.Code:
return cre_defs.Code(
name=self.name,
id=self.id,
description=self.description,
tags=self.tags,
hyperlink=self.hyperlink,
version=self.version,
)


class NeoCRE(NeoDocument): # type: ignore
id = UniqueIdProperty()
external_id = StringProperty()
description = StringProperty()
name = StringProperty(required=True)
contains = RelationshipTo("NeoCRE", "CONTAINS")
linked = RelationshipTo("NeoStandard", "LINKED_TO")
same_as = RelationshipTo("NeoStandard", "SAME")
contains = RelationshipTo("NeoCRE", "CONTAINS", model=ContainsRel)
linked = RelationshipTo("NeoStandard", "LINKED_TO", model=LinkedToRel)
same_as = RelationshipTo("NeoStandard", "SAME", model=SameRel)

@classmethod
def to_cre_def(self) -> cre_defs.CRE:
return cre_defs.CRE(
name=self.name,
id=self.id,
description=self.description,
tags=self.tags,
)


class NEO_DB:
Expand Down Expand Up @@ -298,7 +367,7 @@ def add_dbnode(self, dbnode: Node):
"version": dbnode.version or "",
"section": dbnode.section or "",
"sectionID": dbnode.section_id or "",
"subsection": dbnode.subsection or ""
"subsection": dbnode.subsection or "",
}
)
return
Expand Down Expand Up @@ -371,47 +440,46 @@ def link_CRE_to_Node(self, CRE_id, node_id, link_type):
def gap_analysis(self, name_1, name_2):
if not self.connected:
return None, None
base_standard, _, _ = self.driver.execute_query(
"""
MATCH (BaseStandard:Standard|Tool {name: $name1})
RETURN BaseStandard
""",
name1=name_1,
database_="neo4j",
)
base_standard = NeoStandard.nodes.filter(name=name_1)

path_records_all, _, _ = self.driver.execute_query(
path_records_all, _ = db.cypher_query(
"""
OPTIONAL MATCH (BaseStandard:Standard|Tool {name: $name1})
OPTIONAL MATCH (CompareStandard:Standard|Tool {name: $name2})
OPTIONAL MATCH (BaseStandard:NeoStandard {name: $name1})
OPTIONAL MATCH (CompareStandard:NeoStandard {name: $name2})
OPTIONAL MATCH p = shortestPath((BaseStandard)-[*..20]-(CompareStandard))
WITH p
WHERE length(p) > 1 AND ALL(n in NODES(p) WHERE n:CRE or n = BaseStandard or n = CompareStandard)
WHERE length(p) > 1 AND ALL(n in NODES(p) WHERE n:NeoCRE or n = BaseStandard or n = CompareStandard)
RETURN p
""",
name1=name_1,
name2=name_2,
database_="neo4j",
{"name1": name_1, "name2": name_2},
resolve_objects=True,
)
path_records, _, _ = self.driver.execute_query(

path_records, _ = db.cypher_query(
"""
OPTIONAL MATCH (BaseStandard:Standard|Tool {name: $name1})
OPTIONAL MATCH (CompareStandard:Standard|Tool {name: $name2})
OPTIONAL MATCH (BaseStandard:NeoStandard {name: $name1})
OPTIONAL MATCH (CompareStandard:NeoStandard {name: $name2})
OPTIONAL MATCH p = shortestPath((BaseStandard)-[:(LINKED_TO|CONTAINS)*..20]-(CompareStandard))
WITH p
WHERE length(p) > 1 AND ALL(n in NODES(p) WHERE n:CRE or n = BaseStandard or n = CompareStandard)
WHERE length(p) > 1 AND ALL(n in NODES(p) WHERE n:NeoCRE or n = BaseStandard or n = CompareStandard)
RETURN p
""",
name1=name_1,
name2=name_2,
database_="neo4j",
{"name1": name_1, "name2": name_2},
resolve_objects=True,
)

def format_segment(seg):
def format_segment(seg: StructuredRel):
relation_map = {
RelatedRel: "RELATED",
ContainsRel: "CONTAINS",
LinkedToRel: "LINKED_TO",
SameRel: "SAME",
}

return {
"start": NEO_DB.parse_node(seg.start_node),
"end": NEO_DB.parse_node(seg.end_node),
"relationship": seg.type,
"start": NEO_DB.parse_node(seg.start_node()),
"end": NEO_DB.parse_node(seg.end_node()),
"relationship": relation_map[type(seg)],
}

def format_path_record(rec):
Expand All @@ -421,8 +489,8 @@ def format_path_record(rec):
"path": [format_segment(seg) for seg in rec.relationships],
}

return [NEO_DB.parse_node(rec["BaseStandard"]) for rec in base_standard], [
format_path_record(rec["p"]) for rec in (path_records + path_records_all)
return [NEO_DB.parse_node(rec) for rec in base_standard], [
format_path_record(rec[0]) for rec in (path_records + path_records_all)
]

@classmethod
Expand All @@ -431,71 +499,12 @@ def standards(self) -> List[str]:
return
tools = NeoTool.nodes.all()
standards = NeoStandard.nodes.all()

return list(set([x.name for x in tools] + [x.name for x in standards]))

@staticmethod
def parse_node(node: neo4j.graph.Node) -> cre_defs.Document:
# TODO: Parse via NeoDocument classes
name = node["name"]
id = node["id"] if "id" in node else None
description = node["description"] if "description" in node else None
# links = [self.parse_link(link) for link in node["links"]]
tags = node["tags"]
# metadata = node["metadata"]
if cre_defs.Credoctypes.Code.value in node.labels:
return cre_defs.Code(
name=name,
id=id,
description=description,
# links=links,
tags=tags,
# metadata=metadata,
# hyperlink=(node["hyperlink"] if "hyperlink" in node else None),
version=(node["version"] if "version" in node else None),
)
if cre_defs.Credoctypes.Standard.value in node.labels:
return cre_defs.Standard(
name=name,
id=id,
description=description,
# links=links,
tags=tags,
# metadata=metadata,
# hyperlink=(node["hyperlink"] if "hyperlink" in node else None),
version=(node["version"] if "version" in node else None),
section=node["section"],
sectionID=node["sectionID"],
subsection=(node["subsection"] if "subsection" in node else None),
)
if cre_defs.Credoctypes.Tool.value in node.labels:
return cre_defs.Tool(
name=name,
id=id,
description=description,
# links=links,
tags=tags,
# metadata=metadata,
# hyperlink=(node["hyperlink"] if "hyperlink" in node else None),
version=(node["version"] if "version" in node else None),
section=node["section"],
sectionID=node["sectionID"],
subsection=(node["subsection"] if "subsection" in node else None),
)
if cre_defs.Credoctypes.CRE.value in node.labels:
return cre_defs.CRE(
name=name,
id=id,
description=description,
# links=links,
tags=tags,
# metadata=metadata,
)
raise Exception(f"Unknown node {node.labels}")
return list(set([x.name for x in tools] + [x.name for x in standards]))

# @classmethod
# def parse_link(self, link):
# return cre_defs.Link(ltype=link["ltype"], tags=link["tags"])
@staticmethod
def parse_node(node: NeoDocument) -> cre_defs.Document:
return node.to_cre_def()


class CRE_Graph:
Expand Down
1 change: 1 addition & 0 deletions application/web/web_main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,7 @@ def gap_analysis() -> Any:
gap_analysis = database.gap_analysis(standards)
if gap_analysis is None:
return neo4j_not_running_rejection()
print(gap_analysis)
return jsonify(gap_analysis)


Expand Down

0 comments on commit d0c8683

Please sign in to comment.