diff --git a/arches/app/models/graph.py b/arches/app/models/graph.py index 7baa63b510..71150b166a 100644 --- a/arches/app/models/graph.py +++ b/arches/app/models/graph.py @@ -30,7 +30,7 @@ from arches.app.datatypes.datatypes import DataTypeFactory from arches.app.etl_modules.bulk_data_deletion import BulkDataDeletion from arches.app.utils.betterJSONSerializer import JSONSerializer, JSONDeserializer -from arches.app.utils.editable_future_graph import update_editable_future_nodegroups +from arches.app.utils import editable_future_graph_utils from arches.app.search.search_engine_factory import SearchEngineFactory from arches.app.utils.i18n import LanguageSynchronizer from django.utils.translation import gettext as _ @@ -2430,305 +2430,82 @@ def update_from_editable_future_graph(self): raise Exception(_("No identifiable future Graph")) with transaction.atomic(): - self.root.set_relatable_resources( - [ - node.pk - for node in editable_future_graph.root.get_relatable_resources() - ] + current_nodegroups = self.get_nodegroups(force_recalculation=True) + editable_future_nodegroups = editable_future_graph.get_nodegroups( + force_recalculation=True ) - previous_card_ids = {str(card.pk) for card in self.cards.values()} - previous_node_ids = {str(node.pk) for node in self.nodes.values()} - previous_edge_ids = {str(edge.pk) for edge in self.edges.values()} - previous_widget_ids = {str(widget.pk) for widget in self.widgets.values()} - - self.cards = {} - self.nodes = {} - self.edges = {} - self.widgets = {} - # BEGIN update related models - # Iterates over cards, nodes, and edges of the editable_future_graph. If the item - # has a `source_identifier` attribute, it represents an item related to the source - # graph ( the graph mapped to `self` ); we iterate over the item attributes and map - # them to source item. If the item does not have a `source_identifier` attribute, it - # has been newly created; we update the `graph_id` to match the source graph. We are - # not saving in this block so updates can accur in any order. - update_editable_future_nodegroups( - editable_future_graph.get_nodegroups(force_recalculation=True) - ) - - for future_widget in list(editable_future_graph.widgets.values()): - source_widget = future_widget.source_identifier - - if future_widget.source_identifier_id: - for key in vars(source_widget).keys(): - if key not in [ - "_state", - "id", - "node_id", - "card_id", - "source_identifier_id", - ]: - setattr(source_widget, key, getattr(future_widget, key)) - - if future_widget.card.source_identifier_id: - source_widget.card_id = future_widget.card.source_identifier_id - if future_widget.node.source_identifier_id: - source_widget.node_id = future_widget.node.source_identifier_id - - self.widgets[source_widget.pk] = source_widget - else: # newly-created widget - future_widget.source_identifier_id = None - - if future_widget.card.source_identifier_id: - future_widget.card_id = future_widget.card.source_identifier_id - if future_widget.node.source_identifier_id: - future_widget.node_id = future_widget.node.source_identifier_id - - del editable_future_graph.widgets[future_widget.pk] - self.widgets[future_widget.pk] = future_widget - - for future_card in list(editable_future_graph.cards.values()): - future_card_nodegroup_node = models.Node.objects.get( - pk=future_card.nodegroup.pk - ) - - if future_card.source_identifier: - source_card = future_card.source_identifier - - for key in vars(source_card).keys(): - if key not in [ - "graph_id", - "cardid", - "nodegroup_id", - "source_identifier_id", - ]: - if ( - key == "config" - and str(future_card.component_id) - == "2f9054d8-de57-45cd-8a9c-58bbb1619030" - ): # grouping card - grouped_card_ids = [] - for grouped_card_id in future_card.config[ - "groupedCardIds" - ]: - grouped_card = Card.objects.get(pk=grouped_card_id) - grouped_card_ids.append( - str(grouped_card.source_identifier_id) - ) - - source_card.config["groupedCardIds"] = grouped_card_ids - - sorted_widget_ids = [] - for node_id in future_card.config["sortedWidgetIds"]: - sorted_widget = models.Node.objects.get(pk=node_id) - sorted_widget_ids.append( - str(sorted_widget.source_identifier_id) - ) - - source_card.config["sortedWidgetIds"] = ( - sorted_widget_ids - ) - else: - setattr(source_card, key, getattr(future_card, key)) - - source_card.nodegroup_id = future_card_nodegroup_node.nodegroup_id - if future_card_nodegroup_node.source_identifier_id: - source_card.nodegroup_id = ( - future_card_nodegroup_node.source_identifier_id - ) - - self.cards[source_card.pk] = source_card - else: # newly-created card - future_card.graph_id = self.pk - future_card.source_identifier_id = None - - future_card.nodegroup_id = future_card_nodegroup_node.nodegroup_id - if future_card_nodegroup_node.source_identifier_id: - future_card.nodegroup_id = ( - future_card_nodegroup_node.source_identifier_id - ) - - del editable_future_graph.cards[future_card.pk] - self.cards[future_card.pk] = future_card - - for future_edge in list(editable_future_graph.edges.values()): - if future_edge.source_identifier_id: - source_edge = future_edge.source_identifier - - for key in vars(source_edge).keys(): - if key not in [ - "domainnode_id", - "edgeid", - "graph_id", - "rangenode_id", - "source_identifier_id", - ]: - setattr(source_edge, key, getattr(future_edge, key)) - - source_edge.domainnode_id = future_edge.domainnode_id - if future_edge.domainnode.source_identifier: - source_edge.domainnode_id = ( - future_edge.domainnode.source_identifier.pk - ) - - source_edge.rangenode_id = future_edge.rangenode_id - if future_edge.rangenode.source_identifier: - source_edge.rangenode_id = ( - future_edge.rangenode.source_identifier.pk - ) - - self.edges[source_edge.pk] = source_edge - else: # newly-created edge - future_edge.graph_id = self.pk - future_edge.source_identfier_id = None - - if future_edge.domainnode.source_identifier_id: - future_edge.domainnode_id = ( - future_edge.domainnode.source_identifier_id - ) - - if future_edge.rangenode.source_identifier_id: - future_edge.rangenode_id = ( - future_edge.rangenode.source_identifier_id - ) - - del editable_future_graph.edges[future_edge.pk] - self.edges[future_edge.pk] = future_edge - - for future_node in list(editable_future_graph.nodes.values()): - future_node_nodegroup_node = ( - models.Node.objects.get(pk=future_node.nodegroup.pk) - if future_node.nodegroup - else None - ) - - if future_node.source_identifier: - source_node = future_node.source_identifier - - for key in vars(source_node).keys(): - if key not in [ - "graph_id", - "nodeid", - "nodegroup_id", - "source_identifier_id", - ]: - setattr(source_node, key, getattr(future_node, key)) - - source_node.nodegroup_id = future_node.nodegroup_id - if ( - future_node_nodegroup_node - and future_node_nodegroup_node.source_identifier_id - ): - source_node.nodegroup_id = ( - future_node_nodegroup_node.source_identifier_id - ) - - self.nodes[source_node.pk] = source_node - else: # newly-created node - future_node.graph_id = self.pk - future_node.source_identifier_id = None - - if ( - future_node_nodegroup_node - and future_node_nodegroup_node.source_identifier_id - ): - future_node.nodegroup_id = ( - future_node_nodegroup_node.source_identifier_id - ) + EXCLUDED_NODEGROUP_KEYS = ["nodegroupid", "parentnodegroup_id"] + EXCLUDED_WIDGET_KEYS = [ + "_state", + "id", + "node_id", + "card_id", + "source_identifier_id", + ] + EXCLUDED_CARD_KEYS = [ + "graph_id", + "cardid", + "nodegroup_id", + "source_identifier_id", + ] + EXCLUDED_EDGE_KEYS = [ + "domainnode_id", + "edgeid", + "graph_id", + "rangenode_id", + "source_identifier_id", + ] + EXCLUDED_NODE_KEYS = [ + "graph_id", + "nodeid", + "nodegroup_id", + "source_identifier_id", + ] + EXCLUDED_GRAPH_ATTRIBUTES_KEYS = [ + "_state", + "graphid", + "cards", + "nodes", + "edges", + "widgets", + "root", + "source_identifier", + "source_identifier_id", + "resource_instance_lifecycle", + "resource_instance_lifecycle_id", + "publication_id", + "_nodegroups_to_delete", + "_functions", + "_card_constraints", + "_constraints_x_nodes", + "serialized_graph", + ] - del editable_future_graph.nodes[future_node.pk] - self.nodes[future_node.pk] = future_node + editable_future_graph_utils.update_nodegroups( + self, editable_future_graph, EXCLUDED_NODEGROUP_KEYS + ) + editable_future_graph_utils.update_widgets( + self, editable_future_graph, EXCLUDED_WIDGET_KEYS + ) + editable_future_graph_utils.update_cards( + self, editable_future_graph, EXCLUDED_CARD_KEYS + ) + editable_future_graph_utils.update_edges( + self, editable_future_graph, EXCLUDED_EDGE_KEYS + ) + editable_future_graph_utils.update_nodes( + self, editable_future_graph, EXCLUDED_NODE_KEYS + ) # END update related models # BEGIN copy attrs from editable_future_graph to source_graph - for key, value in vars(editable_future_graph).items(): - if key not in [ - "_state", - "graphid", - "cards", - "nodes", - "edges", - "widgets", - "root", - "source_identifier", - "source_identifier_id", - "resource_instance_lifecycle", - "resource_instance_lifecycle_id", - "publication_id", - "_nodegroups_to_delete", - "_functions", - "_card_constraints", - "_constraints_x_nodes", - "serialized_graph", - ]: - setattr(self, key, value) - - self.root = self.nodes[self.root.pk] + editable_future_graph_utils.update_source_graph( + self, editable_future_graph, EXCLUDED_GRAPH_ATTRIBUTES_KEYS + ) # END copy attrs from editable_future_graph to source_graph - # BEGIN delete superflous models - # Compares UUIDs between models related to the source graph and models related to - # the editable_future_graph. If the item related to the source graph exists, but the item - # related to the editable_future_graph does not exist, the item related to the source graph - # should be deleted. - updated_widget_ids = { - str(widget.pk) for widget in editable_future_graph.widgets.values() - } - - updated_card_ids = { - str(card.source_identifier_id) - for card in editable_future_graph.cards.values() - } - - updated_edge_ids = { - str(edge.source_identifier_id) - for edge in editable_future_graph.edges.values() - } - - updated_node_ids = { - str(node.source_identifier_id) - for node in editable_future_graph.nodes.values() - } - updated_node_ids.add(str(self.root.pk)) - - models.CardXNodeXWidget.objects.filter( - pk__in=set(previous_widget_ids - updated_widget_ids) - | {widget.pk for widget in editable_future_graph.widgets.values()} - ).delete() - - models.CardModel.objects.filter( - pk__in=set(previous_card_ids - updated_card_ids) - | {card.pk for card in editable_future_graph.cards.values()} - ).delete() - - models.Edge.objects.filter( - pk__in=set(previous_edge_ids - updated_edge_ids) - | {edge.pk for edge in editable_future_graph.edges.values()} - ).delete() - - models.Node.objects.filter( - pk__in=set(previous_node_ids - updated_node_ids) - | {node.pk for node in editable_future_graph.nodes.values()} - ).delete() - # END delete superflous models - - # BEGIN save related models - # save order is _very_ important! - for widget in self.widgets.values(): - widget.save() - - for card in self.cards.values(): - card.save() - - for edge in self.edges.values(): - edge.save() - - for node in self.nodes.values(): - node.save() - # END save related models - self.save(validate=False) # This ensures essential objects that have been re-assigned to the `source_graph` @@ -2740,21 +2517,14 @@ def update_from_editable_future_graph(self): editable_future_graph.delete() - graph_from_database = type(self).objects.get( - pk=self.pk - ) # returns an updated copy of self - graph_from_database.create_editable_future_graph() + # delete any nodegroups that are no longer associated with the graph + for nodegroup in current_nodegroups + editable_future_nodegroups: + if not models.Node.objects.filter(pk=nodegroup.pk).exists(): + nodegroup.delete() - # TODO: This is a temporary fix to remove nodegroups that are no longer in use. It should be replaced with a more performant solution. - with connection.cursor() as cursor: - cursor.execute( - """ - DELETE FROM node_groups - WHERE NOT EXISTS ( - SELECT 1 FROM nodes WHERE nodes.nodeid = node_groups.nodegroupid - ); - """ - ) + # returns an updated copy of self + graph_from_database = type(self).objects.get(pk=self.pk) + graph_from_database.create_editable_future_graph() return graph_from_database diff --git a/arches/app/utils/editable_future_graph.py b/arches/app/utils/editable_future_graph.py deleted file mode 100644 index 8c898ee3a7..0000000000 --- a/arches/app/utils/editable_future_graph.py +++ /dev/null @@ -1,54 +0,0 @@ -from arches.app.models import models - - -def update_editable_future_nodegroups(editable_future_nodegroups): - """ - Update the nodegroups in the source_graph to match those in the - editable_future_graph. - """ - - def _update_source_nodegroup_hierarchy(nodegroup): - if not nodegroup: - return None - - node = models.Node.objects.get(pk=nodegroup.pk) - if node.source_identifier_id: - source_nodegroup = models.NodeGroup.objects.get( - pk=node.source_identifier_id - ) - - source_nodegroup.cardinality = nodegroup.cardinality - source_nodegroup.legacygroupid = nodegroup.legacygroupid - - if nodegroup.parentnodegroup_id: - nodegroup_parent_node = models.Node.objects.get( - pk=nodegroup.parentnodegroup_id - ) - - if nodegroup_parent_node.source_identifier_id: - source_nodegroup.parentnodegroup_id = ( - nodegroup_parent_node.source_identifier_id - ) - - source_nodegroup.save() - - if nodegroup.parentnodegroup: - _update_source_nodegroup_hierarchy(nodegroup=nodegroup.parentnodegroup) - - # creates `NodeGroup`s for source_nodegroups_nodes whose editable_future_node has been used to create a new card - for editable_future_node in models.Node.objects.filter( - nodegroup__in=editable_future_nodegroups - ): - if ( - editable_future_node.source_identifier - and editable_future_node.pk == editable_future_node.nodegroup_id - and editable_future_node.source_identifier.pk - != editable_future_node.source_identifier.nodegroup_id - ): - models.NodeGroup.objects.create( - nodegroupid=editable_future_node.source_identifier.pk, - cardinality=editable_future_node.nodegroup.cardinality, - ) - - for editable_future_nodegroup in editable_future_nodegroups: - _update_source_nodegroup_hierarchy(editable_future_nodegroup) diff --git a/arches/app/utils/editable_future_graph_utils.py b/arches/app/utils/editable_future_graph_utils.py new file mode 100644 index 0000000000..30eafc6462 --- /dev/null +++ b/arches/app/utils/editable_future_graph_utils.py @@ -0,0 +1,268 @@ +from arches.app.models import models + + +def update_nodegroups(current_graph, editable_future_graph, excluded_keys): + """ + Update the nodegroups in the source_graph to match those in the editable_future_graph. + """ + + def _update_source_nodegroup_hierarchy(nodegroup): + if not nodegroup: + return None + + node = models.Node.objects.filter(pk=nodegroup.pk).first() + source_nodegroup = None + + # updates source_nodegroup if node has a source_identifier_id + if node and node.source_identifier_id: + source_nodegroup = models.NodeGroup.objects.get( + pk=node.source_identifier_id + ) + for key, value in vars(nodegroup).items(): + if key not in excluded_keys: + setattr(source_nodegroup, key, value) + + # recursively update parent nodegroup relationships if parentnodegroup_id exists + if nodegroup.parentnodegroup_id: + parent_node = models.Node.objects.filter( + pk=nodegroup.parentnodegroup_id + ).first() + if parent_node and parent_node.source_identifier_id: + if source_nodegroup: + source_nodegroup.parentnodegroup_id = ( + parent_node.source_identifier_id + ) + source_nodegroup.save() + + nodegroup.parentnodegroup_id = parent_node.source_identifier_id + nodegroup.save() + + _update_source_nodegroup_hierarchy(nodegroup=nodegroup.parentnodegroup) + + editable_future_nodegroups = editable_future_graph.get_nodegroups( + force_recalculation=True + ) + + for editable_future_node in models.Node.objects.filter( + nodegroup__in=editable_future_nodegroups + ): + if ( + editable_future_node.source_identifier + and editable_future_node.pk == editable_future_node.nodegroup_id + and editable_future_node.source_identifier.pk + != editable_future_node.source_identifier.nodegroup_id + ): + models.NodeGroup.objects.create( + nodegroupid=editable_future_node.source_identifier.pk, + cardinality=editable_future_node.nodegroup.cardinality, + ) + + for editable_future_nodegroup in editable_future_nodegroups: + _update_source_nodegroup_hierarchy(editable_future_nodegroup) + + +def update_nodes(current_graph, editable_future_graph, excluded_keys): + previous_node_ids = {str(node.pk) for node in current_graph.nodes.values()} + current_graph.nodes = {} + + for future_node in list(editable_future_graph.nodes.values()): + future_node_nodegroup_node = ( + models.Node.objects.get(pk=future_node.nodegroup.pk) + if future_node.nodegroup + else None + ) + + if future_node.source_identifier: + source_node = future_node.source_identifier + + for key in vars(source_node).keys(): + if key not in excluded_keys: + setattr(source_node, key, getattr(future_node, key)) + + source_node.nodegroup_id = future_node.nodegroup_id + if ( + future_node_nodegroup_node + and future_node_nodegroup_node.source_identifier_id + ): + source_node.nodegroup_id = ( + future_node_nodegroup_node.source_identifier_id + ) + + current_graph.nodes[source_node.pk] = source_node + else: + future_node.graph_id = current_graph.pk + future_node.source_identifier_id = None + + if ( + future_node_nodegroup_node + and future_node_nodegroup_node.source_identifier_id + ): + future_node.nodegroup_id = ( + future_node_nodegroup_node.source_identifier_id + ) + + del editable_future_graph.nodes[future_node.pk] + current_graph.nodes[future_node.pk] = future_node + + updated_node_ids = { + str(node.source_identifier_id) for node in editable_future_graph.nodes.values() + } + updated_node_ids.add(str(current_graph.root.pk)) + + models.Node.objects.filter( + pk__in=set(previous_node_ids - updated_node_ids) + | {node.pk for node in editable_future_graph.nodes.values()} + ).delete() + + for node in current_graph.nodes.values(): + node.save() + + +def update_edges(current_graph, editable_future_graph, excluded_keys): + previous_edge_ids = {str(edge.pk) for edge in current_graph.edges.values()} + current_graph.edges = {} + + for future_edge in list(editable_future_graph.edges.values()): + if future_edge.source_identifier_id: + source_edge = future_edge.source_identifier + + for key in vars(source_edge).keys(): + if key not in excluded_keys: + setattr(source_edge, key, getattr(future_edge, key)) + + source_edge.domainnode_id = future_edge.domainnode_id + if future_edge.domainnode.source_identifier: + source_edge.domainnode_id = future_edge.domainnode.source_identifier.pk + + source_edge.rangenode_id = future_edge.rangenode_id + if future_edge.rangenode.source_identifier: + source_edge.rangenode_id = future_edge.rangenode.source_identifier.pk + + current_graph.edges[source_edge.pk] = source_edge + else: + future_edge.graph_id = current_graph.pk + future_edge.source_identifier_id = None + + if future_edge.domainnode.source_identifier_id: + future_edge.domainnode_id = future_edge.domainnode.source_identifier_id + + if future_edge.rangenode.source_identifier_id: + future_edge.rangenode_id = future_edge.rangenode.source_identifier_id + + del editable_future_graph.edges[future_edge.pk] + current_graph.edges[future_edge.pk] = future_edge + + updated_edge_ids = { + str(edge.source_identifier_id) for edge in editable_future_graph.edges.values() + } + + models.Edge.objects.filter( + pk__in=set(previous_edge_ids - updated_edge_ids) + | {edge.pk for edge in editable_future_graph.edges.values()} + ).delete() + + for edge in current_graph.edges.values(): + edge.save() + + +def update_cards(current_graph, editable_future_graph, excluded_keys): + previous_card_ids = {str(card.pk) for card in current_graph.cards.values()} + current_graph.cards = {} + + for future_card in list(editable_future_graph.cards.values()): + future_card_nodegroup_node = models.Node.objects.get( + pk=future_card.nodegroup.pk + ) + + if future_card.source_identifier: + source_card = future_card.source_identifier + + for key in vars(source_card).keys(): + if key not in excluded_keys: + setattr(source_card, key, getattr(future_card, key)) + + source_card.nodegroup_id = future_card_nodegroup_node.nodegroup_id + if future_card_nodegroup_node.source_identifier_id: + source_card.nodegroup_id = ( + future_card_nodegroup_node.source_identifier_id + ) + + current_graph.cards[source_card.pk] = source_card + else: + future_card.graph_id = current_graph.pk + future_card.source_identifier_id = None + + future_card.nodegroup_id = future_card_nodegroup_node.nodegroup_id + if future_card_nodegroup_node.source_identifier_id: + future_card.nodegroup_id = ( + future_card_nodegroup_node.source_identifier_id + ) + + del editable_future_graph.cards[future_card.pk] + current_graph.cards[future_card.pk] = future_card + + updated_card_ids = { + str(card.source_identifier_id) for card in editable_future_graph.cards.values() + } + + models.CardModel.objects.filter( + pk__in=set(previous_card_ids - updated_card_ids) + | {card.pk for card in editable_future_graph.cards.values()} + ).delete() + + for card in current_graph.cards.values(): + card.save() + + +def update_widgets(current_graph, editable_future_graph, excluded_keys): + previous_widget_ids = {str(widget.pk) for widget in current_graph.widgets.values()} + current_graph.widgets = {} + + for future_widget in list(editable_future_graph.widgets.values()): + if future_widget.source_identifier_id: + source_widget = future_widget.source_identifier + + for key in vars(source_widget).keys(): + if key not in excluded_keys: + setattr(source_widget, key, getattr(future_widget, key)) + + if future_widget.card.source_identifier_id: + source_widget.card_id = future_widget.card.source_identifier_id + if future_widget.node.source_identifier_id: + source_widget.node_id = future_widget.node.source_identifier_id + + current_graph.widgets[source_widget.pk] = source_widget + else: + future_widget.source_identifier_id = None + + if future_widget.card.source_identifier_id: + future_widget.card_id = future_widget.card.source_identifier_id + if future_widget.node.source_identifier_id: + future_widget.node_id = future_widget.node.source_identifier_id + + del editable_future_graph.widgets[future_widget.pk] + current_graph.widgets[future_widget.pk] = future_widget + + updated_widget_ids = { + str(widget.source_identifier_id) + for widget in editable_future_graph.widgets.values() + } + + models.CardXNodeXWidget.objects.filter( + pk__in=set(previous_widget_ids - updated_widget_ids) + | {widget.pk for widget in editable_future_graph.widgets.values()} + ).delete() + + for widget in current_graph.widgets.values(): + widget.save() + + +def update_source_graph(current_graph, editable_future_graph, excluded_keys): + for key, value in vars(editable_future_graph).items(): + if key not in excluded_keys: + setattr(current_graph, key, value) + + current_graph.root = current_graph.nodes[current_graph.root.pk] + current_graph.root.set_relatable_resources( + [node.pk for node in editable_future_graph.root.get_relatable_resources()] + )