Merge pull request #5261 from neos/task/contentCacheFlusher-followup

FEATURE: Speedup content cache flush by using cte in `findAncestorNodeAggregateIds`

Neos.ContentGraph.DoctrineDbalAdapter/Tests/Behavior/Features/Bootstrap/ProjectionIntegrityViolationDetectionTrait.php

@@ -103,6 +103,31 @@ public function iAddTheFollowingHierarchyRelation(TableNode $payloadTable): void
         );
     }
 
+    /**
+     * @When /^I change the following hierarchy relation's parent:$/
+     * @throws DBALException
+     */
+    public function iChangeTheFollowingHierarchyRelationsParent(TableNode $payloadTable): void
+    {
+        $dataset = $this->transformPayloadTableToDataset($payloadTable);
+        $record = $this->transformDatasetToHierarchyRelationRecord($dataset);
+        unset($record['position']);
+
+        $newParentHierarchyRelation = $this->findHierarchyRelationByIds(
+            ContentStreamId::fromString($dataset['contentStreamId']),
+            DimensionSpacePoint::fromArray($dataset['dimensionSpacePoint']),
+            NodeAggregateId::fromString($dataset['newParentNodeAggregateId'])
+        );
+
+        $this->dbal->update(
+            $this->tableNames()->hierarchyRelation(),
+            [
+                'parentnodeanchor' => $newParentHierarchyRelation['childnodeanchor']
+            ],
+            $record
+        );
+    }
+
     /**
      * @When /^I change the following hierarchy relation's dimension space point hash:$/
      * @param TableNode $payloadTable

Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/ProjectionIntegrityViolationDetection/AllNodesAreConnectedToARootNodePerSubgraph.feature → Neos.ContentGraph.DoctrineDbalAdapter/Tests/Behavior/Features/Projection/ProjectionIntegrityViolationDetection/AllNodesAreConnectedToARootNodePerSubgraph.feature

@@ -10,6 +10,9 @@ Feature: Run projection integrity violation detection regarding root connection
       | language   | de, gsw | gsw->de         |
     And using the following node types:
     """yaml
+    'Neos.ContentRepository.Testing:Root':
+      superTypes:
+        'Neos.ContentRepository:Root': true
     'Neos.ContentRepository.Testing:Document': []
     """
     And using identifier "default", I define a content repository
@@ -20,45 +23,54 @@ Feature: Run projection integrity violation detection regarding root connection
       | newContentStreamId   | "cs-identifier"      |
 
   Scenario: Create a cycle
-    When the event RootNodeAggregateWithNodeWasCreated was published with payload:
+    When the command CreateRootNodeAggregateWithNode is executed with payload:
       | Key                         | Value                                     |
       | workspaceName               | "live"                                    |
-      | contentStreamId             | "cs-identifier"                           |
       | nodeAggregateId             | "lady-eleonode-rootford"                  |
-      | nodeTypeName                | "Neos.ContentRepository.Testing:Document" |
+      | nodeTypeName                | "Neos.ContentRepository.Testing:Root"     |
       | coveredDimensionSpacePoints | [{"language":"de"},{"language":"gsw"}]    |
       | nodeAggregateClassification | "root"                                    |
-    When the event NodeAggregateWithNodeWasCreated was published with payload:
+    When the command CreateNodeAggregateWithNode is executed with payload:
       | Key                         | Value                                     |
       | workspaceName               | "live"                                    |
-      | contentStreamId             | "cs-identifier"                           |
       | nodeAggregateId             | "sir-david-nodenborough"                  |
       | nodeTypeName                | "Neos.ContentRepository.Testing:Document" |
       | originDimensionSpacePoint   | {"language":"de"}                         |
       | coveredDimensionSpacePoints | [{"language":"de"},{"language":"gsw"}]    |
       | parentNodeAggregateId       | "lady-eleonode-rootford"                  |
       | nodeName                    | "document"                                |
       | nodeAggregateClassification | "regular"                                 |
-    And the event NodeAggregateWithNodeWasCreated was published with payload:
+    When the command CreateNodeAggregateWithNode is executed with payload:
       | Key                         | Value                                     |
       | workspaceName               | "live"                                    |
-      | contentStreamId             | "cs-identifier"                           |
       | nodeAggregateId             | "nody-mc-nodeface"                        |
       | nodeTypeName                | "Neos.ContentRepository.Testing:Document" |
       | originDimensionSpacePoint   | {"language":"de"}                         |
       | coveredDimensionSpacePoints | [{"language":"de"},{"language":"gsw"}]    |
       | parentNodeAggregateId       | "sir-david-nodenborough"                  |
       | nodeName                    | "child-document"                          |
       | nodeAggregateClassification | "regular"                                 |
-    And the event NodeAggregateWasMoved was published with payload:
-      | Key                           | Value                                                                                                                                  |
-      | workspaceName                 | "live"                                                                                                                                 |
-      | contentStreamId               | "cs-identifier"                                                                                                                        |
-      | nodeAggregateId               | "sir-david-nodenborough"                                                                                                               |
-      | newParentNodeAggregateId      | "nody-mc-nodeface"                                                                                                                     |
-      | succeedingSiblingsForCoverage | [{"dimensionSpacePoint":{"language":"de"},"nodeAggregateId": null},{"dimensionSpacePoint":{"language":"gsw"},"nodeAggregateId": null}] |
+
+    When I change the following hierarchy relation's parent:
+      | Key                        | Value                                      |
+      | contentStreamId            | "cs-identifier"                            |
+      | dimensionSpacePoint        | {"language":"de"}                          |
+      | parentNodeAggregateId      | "lady-eleonode-rootford"                   |
+      | childNodeAggregateId       | "sir-david-nodenborough"                   |
+      | newParentNodeAggregateId   | "nody-mc-nodeface"                         |
+    And I run integrity violation detection
+    Then I expect the integrity violation detection result to contain exactly 1 errors
+    And I expect integrity violation detection result error number 1 to have code 1597754245
+
+    # Another error. One error per subgraph
+    When I change the following hierarchy relation's parent:
+      | Key                        | Value                                       |
+      | contentStreamId            | "cs-identifier"                             |
+      | dimensionSpacePoint        | {"language":"gsw"}                          |
+      | parentNodeAggregateId      | "lady-eleonode-rootford"                    |
+      | childNodeAggregateId       | "sir-david-nodenborough"                    |
+      | newParentNodeAggregateId   | "nody-mc-nodeface"                          |
     And I run integrity violation detection
-    # one error per subgraph
     Then I expect the integrity violation detection result to contain exactly 2 errors
     And I expect integrity violation detection result error number 1 to have code 1597754245
     And I expect integrity violation detection result error number 2 to have code 1597754245

Neos.ContentGraph.DoctrineDbalAdapter/src/Domain/Repository/ContentGraph.php

@@ -35,6 +35,7 @@
 use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId;
 use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateClassification;
 use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateId;
+use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateIds;
 use Neos\ContentRepository\Core\SharedModel\Node\NodeName;
 use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId;
 use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName;
@@ -54,6 +55,7 @@
  *    - cn -> child node
  *    - h -> the hierarchy edge connecting parent and child
  *    - ph -> the hierarchy edge incoming to the parent (sometimes relevant)
+ *    - ch -> the hierarchy edge of the child (sometimes relevant)
  *    - dsp -> dimension space point, resolves hashes to full dimension coordinates
  *    - cdsp -> child dimension space point, same as dsp for child queries
  *    - pdsp -> parent dimension space point, same as dsp for parent queries
@@ -187,6 +189,38 @@ public function findParentNodeAggregates(
         return $this->mapQueryBuilderToNodeAggregates($queryBuilder);
     }
 
+    public function findAncestorNodeAggregateIds(NodeAggregateId $entryNodeAggregateId): NodeAggregateIds
+    {
+        $queryBuilderInitial = $this->createQueryBuilder()
+            ->select('ch.parentnodeanchor')
+            ->from($this->nodeQueryBuilder->tableNames->hierarchyRelation(), 'ch')
+            ->innerJoin('ch', $this->nodeQueryBuilder->tableNames->node(), 'c', 'c.relationanchorpoint = ch.childnodeanchor')
+            ->where('ch.contentstreamid = :contentStreamId')
+            ->andWhere('c.nodeaggregateid = :entryNodeAggregateId');
+
+        $queryBuilderRecursive = $this->createQueryBuilder()
+            ->select('ph.parentnodeanchor')
+            ->from('ancestry', 'ch')
+            ->innerJoin('ch', $this->nodeQueryBuilder->tableNames->hierarchyRelation(), 'ph', 'ph.childnodeanchor = ch.parentnodeanchor')
+            ->where('ph.contentstreamid = :contentStreamId');
+
+        $queryBuilderCte = $this->createQueryBuilder()
+            ->select('n.nodeAggregateId')
+            ->from('ancestry', 'a')
+            ->innerJoin('a', $this->nodeQueryBuilder->tableNames->node(), 'n', 'n.relationanchorpoint = a.parentnodeanchor')
+            ->setParameter('contentStreamId', $this->contentStreamId->value)
+            ->setParameter('entryNodeAggregateId', $entryNodeAggregateId->value);
+
+        $nodeAggregateIdRows = $this->fetchCteResults(
+            $queryBuilderInitial,
+            $queryBuilderRecursive,
+            $queryBuilderCte,
+            'ancestry'
+        );
+
+        return NodeAggregateIds::fromArray(array_map(fn(array $row) => NodeAggregateId::fromString($row['nodeAggregateId']), $nodeAggregateIdRows));
+    }
+
     public function findChildNodeAggregates(
         NodeAggregateId $parentNodeAggregateId
     ): NodeAggregates {
@@ -324,6 +358,28 @@ private function fetchRows(QueryBuilder $queryBuilder): array
         }
     }
 
+    /**
+     * @return array<int, array<string, mixed>>
+     */
+    private function fetchCteResults(QueryBuilder $queryBuilderInitial, QueryBuilder $queryBuilderRecursive, QueryBuilder $queryBuilderCte, string $cteTableName = 'cte'): array
+    {
+        $query = <<<SQL
+            WITH RECURSIVE {$cteTableName} AS (
+                {$queryBuilderInitial->getSQL()}
+                UNION
+                {$queryBuilderRecursive->getSQL()}
+            )
+            {$queryBuilderCte->getSQL()}
+        SQL;
+        $parameters = array_merge($queryBuilderInitial->getParameters(), $queryBuilderRecursive->getParameters(), $queryBuilderCte->getParameters());
+        $parameterTypes = array_merge($queryBuilderInitial->getParameterTypes(), $queryBuilderRecursive->getParameterTypes(), $queryBuilderCte->getParameterTypes());
+        try {
+            return $this->dbal->fetchAllAssociative($query, $parameters, $parameterTypes);
+        } catch (DBALException $e) {
+            throw new \RuntimeException(sprintf('Failed to fetch CTE result: %s', $e->getMessage()), 1678358108, $e);
+        }
+    }
+
     public function getContentStreamId(): ContentStreamId
     {
         return $this->contentStreamId;

Neos.ContentGraph.DoctrineDbalAdapter/src/Domain/Repository/ContentSubgraph.php

@@ -74,6 +74,7 @@
  *   - cn -> child node
  *   - h -> the hierarchy edge connecting parent and child
  *   - ph -> the hierarchy edge incoming to the parent (sometimes relevant)
+ *   - ch -> the hierarchy edge of the child (sometimes relevant)
  *
  *  - if more than one node (source-destination)
  *   - sn -> source node
@@ -570,8 +571,8 @@ private function buildAncestorNodesQueries(NodeAggregateId $entryNodeAggregateId
 
         $queryBuilderRecursive = $this->createQueryBuilder()
             ->select('pn.*, h.subtreetags, h.parentnodeanchor')
-            ->from('ancestry', 'cn')
-            ->innerJoin('cn', $this->nodeQueryBuilder->tableNames->node(), 'pn', 'pn.relationanchorpoint = cn.parentnodeanchor')
+            ->from('ancestry', 'ch')
+            ->innerJoin('ch', $this->nodeQueryBuilder->tableNames->node(), 'pn', 'pn.relationanchorpoint = ch.parentnodeanchor')
             ->innerJoin('pn', $this->nodeQueryBuilder->tableNames->hierarchyRelation(), 'h', 'h.childnodeanchor = pn.relationanchorpoint')
             ->where('h.contentstreamid = :contentStreamId')
             ->andWhere('h.dimensionspacepointhash = :dimensionSpacePointHash');

Neos.ContentGraph.PostgreSQLAdapter/src/Domain/Repository/ContentHypergraph.php

@@ -32,6 +32,7 @@
 use Neos\ContentRepository\Core\Projection\ContentGraph\VisibilityConstraints;
 use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId;
 use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateId;
+use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateIds;
 use Neos\ContentRepository\Core\SharedModel\Node\NodeName;
 use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId;
 use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName;
@@ -183,6 +184,19 @@ public function findParentNodeAggregates(
         );
     }
 
+    public function findAncestorNodeAggregateIds(NodeAggregateId $entryNodeAggregateId): NodeAggregateIds
+    {
+        $stack = iterator_to_array($this->findParentNodeAggregates($entryNodeAggregateId));
+
+        $ancestorNodeAggregateIds = [];
+        while ($stack !== []) {
+            $nodeAggregate = array_shift($stack);
+            $ancestorNodeAggregateIds[] = $nodeAggregate->nodeAggregateId;
+            array_push($stack, ...iterator_to_array($this->findParentNodeAggregates($nodeAggregate->nodeAggregateId)));
+        }
+        return NodeAggregateIds::fromArray($ancestorNodeAggregateIds);
+    }
+
     public function findChildNodeAggregates(
         NodeAggregateId $parentNodeAggregateId
     ): NodeAggregates {

Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/NodeTraversal/AncestorNodes.feature

@@ -1,6 +1,6 @@
 @contentrepository @adapters=DoctrineDBAL
   # TODO implement for Postgres
-Feature: Find and count nodes using the findAncestorNodes and countAncestorNodes queries
+Feature: Find and count nodes using the findAncestorNodes, countAncestorNodes and findAncestorNodeAggregateIds queries
 
   Background:
     Given using the following content dimensions:
@@ -64,6 +64,7 @@ Feature: Find and count nodes using the findAncestorNodes and countAncestorNodes
       | a2a2            | a2a2     | Neos.ContentRepository.Testing:Page        | a2a                    | {}                    | {}                                       |
       | a2a2a           | a2a2a    | Neos.ContentRepository.Testing:Page        | a2a2                   | {}                    | {}                                       |
       | a2a2b           | a2a2b    | Neos.ContentRepository.Testing:Page        | a2a2                   | {}                    | {}                                       |
+      | a2a2c           | a2a2c    | Neos.ContentRepository.Testing:Page        | a2a2                   | {}                    | {}                                       |
       | a2b             | a2b      | Neos.ContentRepository.Testing:Page        | a2                     | {}                    | {}                                       |
       | a2b1            | a2b1     | Neos.ContentRepository.Testing:Page        | a2b                    | {}                    | {}                                       |
       | b               | b        | Neos.ContentRepository.Testing:Page        | home                   | {}                    | {}                                       |
@@ -75,8 +76,15 @@ Feature: Find and count nodes using the findAncestorNodes and countAncestorNodes
       | Key                          | Value         |
       | nodeAggregateId              | "a2b"         |
       | nodeVariantSelectionStrategy | "allVariants" |
-
+    And the command MoveNodeAggregate is executed with payload:
+      | Key                          | Value                   |
+      | workspaceName                | "live"                  |
+      | dimensionSpacePoint          | {"language": "ch"}      |
+      | relationDistributionStrategy | "scatter"               |
+      | nodeAggregateId              | "a2a2c"                 |
+      | newParentNodeAggregateId     | "b"                     |
   Scenario:
+    Subgraph queries
     # findAncestorNodes queries without results
     When I execute the findAncestorNodes query for entry node aggregate id "non-existing" I expect no nodes to be returned
     # a2a2a is disabled
@@ -87,3 +95,23 @@ Feature: Find and count nodes using the findAncestorNodes and countAncestorNodes
     # findAncestorNodes queries with results
     When I execute the findAncestorNodes query for entry node aggregate id "a2a2b" I expect the nodes "a2a2,a2a,a2,a,home,lady-eleonode-rootford" to be returned and the total count to be 6
     When I execute the findAncestorNodes query for entry node aggregate id "a2a2b" and filter '{"nodeTypes": "Neos.ContentRepository.Testing:Page"}' I expect the nodes "a2a2,a2,a" to be returned and the total count to be 3
+
+    # a2a2c lives in dimension space ch beneath b
+    When I execute the findAncestorNodes query for entry node aggregate id "a2a2c" I expect the nodes "a2a2,a2a,a2,a,home,lady-eleonode-rootford" to be returned
+    And I am in dimension space point {"language":"ch"}
+    When I execute the findAncestorNodes query for entry node aggregate id "a2a2c" I expect the nodes "b,home,lady-eleonode-rootford" to be returned
+
+  Scenario:
+    Contentgraph queries
+    # findAncestorNodes queries without results
+    When I execute the findAncestorNodeAggregateIds query for entry node aggregate id "non-existing" I expect no nodes to be returned
+
+    # findAncestorNodes queries with results
+    # a2a2a is disabled
+    When I execute the findAncestorNodeAggregateIds query for entry node aggregate id "a2a2a" I expect the nodes "a2a2,a2a,a2,a,home,lady-eleonode-rootford" to be returned in any order
+    # a2b is disabled
+    When I execute the findAncestorNodeAggregateIds query for entry node aggregate id "a2b1" I expect the nodes "a2b,a2,a,home,lady-eleonode-rootford" to be returned in any order
+    # a2a2c lives in dimension space ch beneath b
+    When I execute the findAncestorNodeAggregateIds query for entry node aggregate id "a2a2c" I expect the nodes "a2a2,a2a,a2,b,a,home,lady-eleonode-rootford" to be returned in any order
+
+    When I execute the findAncestorNodeAggregateIds query for entry node aggregate id "a2a2b" I expect the nodes "a2a2,a2a,a2,a,home,lady-eleonode-rootford" to be returned in any order

Neos.ContentRepository.Core/Classes/Projection/ContentGraph/ContentGraphInterface.php

@@ -23,6 +23,7 @@
 use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId;
 use Neos\ContentRepository\Core\SharedModel\Exception\NodeAggregatesTypeIsAmbiguous;
 use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateId;
+use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateIds;
 use Neos\ContentRepository\Core\SharedModel\Node\NodeName;
 use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId;
 use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName;
@@ -108,6 +109,13 @@ public function findParentNodeAggregates(
         NodeAggregateId $childNodeAggregateId
     ): NodeAggregates;
 
+    /**
+     * @internal the returned order of node aggregate ids is undefined and not to be relied upon
+     */
+    public function findAncestorNodeAggregateIds(
+        NodeAggregateId $entryNodeAggregateId
+    ): NodeAggregateIds;
+
     /**
      * @internal only for consumption inside the Command Handler
      */