From 5d60c0c32c3b41122ab4af81fa01bb4a5d8ff5b5 Mon Sep 17 00:00:00 2001 From: Hong Liu Date: Wed, 6 Nov 2024 15:05:41 -0800 Subject: [PATCH] Add the process of Relationship V2 in EbeanLocalRelationshipWriterDAO (#467) * add support to process Relationship V2 * address comments * address comments --- .../metadata/dao/utils/GraphUtils.java | 47 +++++++- .../metadata/dao/utils/ModelUtils.java | 4 +- .../metadata/dao/utils/GraphUtilsTest.java | 113 ++++++++++++++++-- .../dao/EbeanLocalRelationshipWriterDAO.java | 69 ++++++++--- .../EbeanLocalRelationshipWriterDAOTest.java | 85 +++++++++++++ ...l-with-non-dollar-virtual-column-names.sql | 15 +++ ...bean-local-relationship-dao-create-all.sql | 14 +++ 7 files changed, 315 insertions(+), 32 deletions(-) diff --git a/dao-api/src/main/java/com/linkedin/metadata/dao/utils/GraphUtils.java b/dao-api/src/main/java/com/linkedin/metadata/dao/utils/GraphUtils.java index a5ff92222..61c40d6f5 100644 --- a/dao-api/src/main/java/com/linkedin/metadata/dao/utils/GraphUtils.java +++ b/dao-api/src/main/java/com/linkedin/metadata/dao/utils/GraphUtils.java @@ -5,27 +5,34 @@ import com.linkedin.metadata.dao.internal.BaseGraphWriterDAO; import java.util.List; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import static com.linkedin.metadata.dao.utils.ModelUtils.*; public class GraphUtils { + private static final String SOURCE = "source"; private GraphUtils() { // Util class } /** * Check if a group relationship shares the same source urn, destination urn or both based on the remove option. + * @param relationships list of relationships + * @param removalOption removal option to specify which relationships to be removed + * @param sourceField name of the source field + * @param destinationField name of the destination field + * @param urn source urn to compare. Optional for V1. Needed for V2. */ public static void checkSameUrn(@Nonnull final List relationships, - @Nonnull final BaseGraphWriterDAO.RemovalOption removalOption, final String sourceField, final String destinationField) { + @Nonnull final BaseGraphWriterDAO.RemovalOption removalOption, @Nonnull final String sourceField, + @Nonnull final String destinationField, @Nullable Urn urn) { if (relationships.isEmpty()) { return; } - // ToDo: how to handle this for Relationship V2? - final Urn sourceUrn = getSourceUrnFromRelationship(relationships.get(0)); + final Urn sourceUrn = getSourceUrnBasedOnRelationshipVersion(relationships.get(0), urn); final Urn destinationUrn = getDestinationUrnFromRelationship(relationships.get(0)); if (removalOption == BaseGraphWriterDAO.RemovalOption.REMOVE_ALL_EDGES_FROM_SOURCE) { @@ -38,9 +45,43 @@ public static void checkSameUrn(@Nonnull final List re } } + /** + * Get the source asset's urn for a given relationship. + * @param relationship Relationship. The relationship can be in model V1 or V2. + * @param urn The source asset urn. Optional for V1. Must for V2. Exception will be thrown if urn is not provided for V2. + * @return The source asset urn. + */ + public static Urn getSourceUrnBasedOnRelationshipVersion( + @Nonnull RELATIONSHIP relationship, @Nullable Urn urn) { + Urn sourceUrn; + boolean isRelationshipInV2 = ModelUtils.isRelationshipInV2(relationship.schema()); + if (isRelationshipInV2 && urn != null) { + // if relationship model in V2 and urn is not null, get the sourceUrn from the input urn + sourceUrn = urn; + } else if (!isRelationshipInV2) { + // if relationship model in V1, get the sourceUrn from relationship + sourceUrn = getSourceUrnFromRelationship(relationship); + } else { + // throw exception if relationship in V2 but source urn not provided + throw new IllegalArgumentException("Source urn is needed for Relationship V2"); + } + return sourceUrn; + } + + public static void checkSameUrn(@Nonnull final List relationships, + @Nonnull final BaseGraphWriterDAO.RemovalOption removalOption, @Nonnull final String sourceField, + @Nonnull final String destinationField) { + checkSameUrn(relationships, removalOption, sourceField, destinationField, null); + } + private static void checkSameUrn(@Nonnull List records, @Nonnull String field, @Nonnull Urn compare) { for (RecordTemplate relation : records) { + if (ModelUtils.isRelationshipInV2(relation.schema()) && field.equals(SOURCE)) { + // Skip source urn check for V2 relationships since they don't have source field + // ToDo: enhance the source check for V2 relationships + return; + } if (!compare.equals(ModelUtils.getUrnFromRelationship(relation, field))) { throw new IllegalArgumentException("Records have different " + field + " urn"); } diff --git a/dao-api/src/main/java/com/linkedin/metadata/dao/utils/ModelUtils.java b/dao-api/src/main/java/com/linkedin/metadata/dao/utils/ModelUtils.java index eb98acc37..d95c37229 100644 --- a/dao-api/src/main/java/com/linkedin/metadata/dao/utils/ModelUtils.java +++ b/dao-api/src/main/java/com/linkedin/metadata/dao/utils/ModelUtils.java @@ -874,7 +874,7 @@ public static boolean isCommonAspect(@Nonnull Class cl * @param relationship must be a valid relationship model defined in com.linkedin.metadata.relationship * @return boolean. True if the relationship is in MG model V2. */ - static boolean isRelationshipInV2(Class relationship) { + public static boolean isRelationshipInV2(Class relationship) { final RecordDataSchema schema = ValidationUtils.getRecordSchema(relationship); return isRelationshipInV2(schema); } @@ -887,7 +887,7 @@ static boolean isRelationshipInV2(Class field.getName().equals(SOURCE_FIELD)) diff --git a/dao-api/src/test/java/com/linkedin/metadata/dao/utils/GraphUtilsTest.java b/dao-api/src/test/java/com/linkedin/metadata/dao/utils/GraphUtilsTest.java index 8696c537d..2750e1f08 100644 --- a/dao-api/src/test/java/com/linkedin/metadata/dao/utils/GraphUtilsTest.java +++ b/dao-api/src/test/java/com/linkedin/metadata/dao/utils/GraphUtilsTest.java @@ -22,13 +22,13 @@ public class GraphUtilsTest { public void testCheckSameUrnWithEmptyRelationships() { List relationships = Collections.emptyList(); GraphUtils.checkSameUrn(relationships, BaseGraphWriterDAO.RemovalOption.REMOVE_ALL_EDGES_FROM_SOURCE, "source", "destination"); + GraphUtils.checkSameUrn(relationships, BaseGraphWriterDAO.RemovalOption.REMOVE_ALL_EDGES_TO_DESTINATION, "source", "destination", new BarUrn(1)); // No exception should be thrown } @Test public void testCheckSameUrnWithSameSourceUrn() { - // ToDo: Add test cases for relationship V2 - + // Test cases for relationship V1 RelationshipFoo relationship; try { relationship = mockRelationshipFoo(new FooUrn(1), new BarUrn(2)); @@ -42,10 +42,36 @@ public void testCheckSameUrnWithSameSourceUrn() { } catch (IllegalArgumentException e) { fail("Expected no IllegalArgumentException to be thrown, but got: " + e.getMessage()); } + + // when urn is provided for relationship V1, the provided urn should be ignored + try { + GraphUtils.checkSameUrn(relationships, BaseGraphWriterDAO.RemovalOption.REMOVE_ALL_EDGES_FROM_SOURCE, "source", "destination", new BarUrn(10)); + } catch (IllegalArgumentException e) { + fail("Expected no IllegalArgumentException to be thrown, but got: " + e.getMessage()); + } + + // Test cases for relationship V2 + RelationshipV2Bar relationshipV2 = mockRelationshipV2Bar(new BarUrn(2)); + List relationshipsV2 = Lists.newArrayList(relationshipV2, relationshipV2); + // when urn is provided for relationship V2, the check should pass + try { + GraphUtils.checkSameUrn(relationshipsV2, BaseGraphWriterDAO.RemovalOption.REMOVE_ALL_EDGES_FROM_SOURCE, "source", "destination", new BarUrn(2)); + } catch (IllegalArgumentException e) { + fail("Expected no IllegalArgumentException to be thrown, but got: " + e.getMessage()); + } + // when urn is not provided for relationship V2, it should throw IllegalArgumentException + assertThrows(IllegalArgumentException.class, + () -> GraphUtils.checkSameUrn( + relationshipsV2, + BaseGraphWriterDAO.RemovalOption.REMOVE_ALL_EDGES_FROM_SOURCE, + "source", + "destination") + ); } @Test public void testCheckSameUrnWithDifferentSourceUrn() { + // Test cases for relationship V1 RecordTemplate relationship1; RecordTemplate relationship2; try { @@ -63,49 +89,116 @@ public void testCheckSameUrnWithDifferentSourceUrn() { "source", "destination") ); + // when urn is provided for relationship V1, the provided urn should be ignored + assertThrows(IllegalArgumentException.class, + () -> GraphUtils.checkSameUrn( + relationships, + BaseGraphWriterDAO.RemovalOption.REMOVE_ALL_EDGES_FROM_SOURCE, + "source", + "destination", + new BarUrn(10)) + ); + + // ToDo: add test cases for V2. Right now it check if a list of relationships have the same source urn. } @Test public void testCheckSameUrnWithSameDestinationUrn() { + // Test cases for relationship V1 RelationshipFoo relationship1; - RelationshipV2Bar relationship2; try { relationship1 = mockRelationshipFoo(new FooUrn(1), new BarUrn(2)); - relationship2 = mockRelationshipV2Bar(new BarUrn(2)); } catch (URISyntaxException e) { throw new RuntimeException(e); } - List relationships = Lists.newArrayList(relationship1, relationship2); + List relationships1 = Lists.newArrayList(relationship1, relationship1); try { - GraphUtils.checkSameUrn(relationships, BaseGraphWriterDAO.RemovalOption.REMOVE_ALL_EDGES_TO_DESTINATION, "source", + GraphUtils.checkSameUrn(relationships1, BaseGraphWriterDAO.RemovalOption.REMOVE_ALL_EDGES_TO_DESTINATION, "source", "destination"); } catch (IllegalArgumentException e) { fail("Expected no IllegalArgumentException to be thrown, but got: " + e.getMessage()); } + + // when urn is provided for relationship V1, the provided urn should be ignored + try { + GraphUtils.checkSameUrn(relationships1, BaseGraphWriterDAO.RemovalOption.REMOVE_ALL_EDGES_TO_DESTINATION, "source", + "destination", new BarUrn(10)); + } catch (IllegalArgumentException e) { + fail("Expected no IllegalArgumentException to be thrown, but got: " + e.getMessage()); + } + + // Test cases for relationship V2 + RelationshipV2Bar relationship2 = mockRelationshipV2Bar(new BarUrn(2)); + List relationships2 = Lists.newArrayList(relationship2, relationship2); + + // throws exception if V2 relationships without source urn provided + assertThrows(IllegalArgumentException.class, + () -> GraphUtils.checkSameUrn( + relationships2, + BaseGraphWriterDAO.RemovalOption.REMOVE_ALL_EDGES_TO_DESTINATION, + "source", + "destination") + ); + + try { + GraphUtils.checkSameUrn(relationships2, BaseGraphWriterDAO.RemovalOption.REMOVE_ALL_EDGES_TO_DESTINATION, "source", + "destination", new BarUrn(10)); + } catch (IllegalArgumentException e) { + fail("Expected no IllegalArgumentException to be thrown, but got: " + e.getMessage()); + } } @Test public void testCheckSameUrnWithDifferentDestinationUrn() { + // Test cases for relationship V1 RelationshipFoo relationship1; RelationshipBar relationship2; - RelationshipV2Bar relationship3; try { relationship1 = mockRelationshipFoo(new FooUrn(1), new BarUrn(2)); relationship2 = new RelationshipBar().setSource(new FooUrn(4)).setDestination(new BazUrn(2)); - relationship3 = mockRelationshipV2Bar(new BarUrn(3)); } catch (URISyntaxException e) { throw new RuntimeException(e); } - List relationships = Lists.newArrayList(relationship1, relationship2, relationship3); + List relationships1 = Lists.newArrayList(relationship1, relationship2); assertThrows(IllegalArgumentException.class, () -> GraphUtils.checkSameUrn( - relationships, + relationships1, BaseGraphWriterDAO.RemovalOption.REMOVE_ALL_EDGES_TO_DESTINATION, "source", "destination") ); + + assertThrows(IllegalArgumentException.class, + () -> GraphUtils.checkSameUrn( + relationships1, + BaseGraphWriterDAO.RemovalOption.REMOVE_ALL_EDGES_TO_DESTINATION, + "source", + "destination", + new BarUrn(10)) + ); + + // Test cases for relationship V2 + RelationshipV2Bar relationship3 = mockRelationshipV2Bar(new BarUrn(3)); + RelationshipV2Bar relationship4 = mockRelationshipV2Bar(new BarUrn(4)); + List relationships2 = Lists.newArrayList(relationship3, relationship4); + assertThrows(IllegalArgumentException.class, + () -> GraphUtils.checkSameUrn( + relationships2, + BaseGraphWriterDAO.RemovalOption.REMOVE_ALL_EDGES_TO_DESTINATION, + "source", + "destination") + ); + + assertThrows(IllegalArgumentException.class, + () -> GraphUtils.checkSameUrn( + relationships2, + BaseGraphWriterDAO.RemovalOption.REMOVE_ALL_EDGES_TO_DESTINATION, + "source", + "destination", + new BarUrn(10)) + ); } private RelationshipFoo mockRelationshipFoo(FooUrn expectedSource, BarUrn expectedDestination) { diff --git a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalRelationshipWriterDAO.java b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalRelationshipWriterDAO.java index f590e37e1..586eb210b 100644 --- a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalRelationshipWriterDAO.java +++ b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalRelationshipWriterDAO.java @@ -18,7 +18,7 @@ import java.util.Map; import java.util.stream.Collectors; import javax.annotation.Nonnull; -import javax.annotation.ParametersAreNonnullByDefault; +import javax.annotation.Nullable; import static com.linkedin.metadata.dao.utils.ModelUtils.*; @@ -35,8 +35,6 @@ private static class CommonColumnName { private static final String METADATA = "metadata"; private static final String LAST_MODIFIED_ON = "lastmodifiedon"; private static final String LAST_MODIFIED_BY = "lastmodifiedby"; - private static final String DELETED_TS = "deleted_ts"; - private static final String ASPECT = "aspect"; } public EbeanLocalRelationshipWriterDAO(EbeanServer server) { @@ -45,17 +43,19 @@ public EbeanLocalRelationshipWriterDAO(EbeanServer server) { /** * Process the local relationship updates with transaction guarantee. + * @param urn Urn of the entity to update relationships. * @param relationshipUpdates Updates to local relationship tables. + * @param isTestMode whether to use test schema */ @Transactional public void processLocalRelationshipUpdates(@Nonnull Urn urn, - @Nonnull List relationshipUpdates, boolean isTestMode) { + @Nonnull List relationshipUpdates, @Nonnull boolean isTestMode) { for (LocalRelationshipUpdates relationshipUpdate : relationshipUpdates) { if (relationshipUpdate.getRelationships().isEmpty()) { clearRelationshipsByEntity(urn, relationshipUpdate.getRelationshipClass(), relationshipUpdate.getRemovalOption(), isTestMode); } else { - addRelationships(relationshipUpdate.getRelationships(), relationshipUpdate.getRemovalOption(), isTestMode); + addRelationships(relationshipUpdate.getRelationships(), relationshipUpdate.getRemovalOption(), isTestMode, urn); } } } @@ -64,9 +64,12 @@ public void processLocalRelationshipUpdates(@Nonnull Urn urn, * This method is to serve for the purpose to clear all the relationships from a source entity urn. * @param urn entity urn could be either source or destination, depends on the RemovalOption * @param relationshipClass relationship that needs to be cleared + * @param removalOption removal option to specify which relationships to be removed + * @param isTestMode whether to use test schema */ public void clearRelationshipsByEntity(@Nonnull Urn urn, - @Nonnull Class relationshipClass, @Nonnull RemovalOption removalOption, boolean isTestMode) { + @Nonnull Class relationshipClass, @Nonnull RemovalOption removalOption, + @Nonnull boolean isTestMode) { if (removalOption == RemovalOption.REMOVE_NONE || removalOption == RemovalOption.REMOVE_ALL_EDGES_FROM_SOURCE_TO_DESTINATION) { // this method is to handle the case of adding empty relationship list to clear relationships of an entity urn @@ -85,22 +88,36 @@ public void clearRelationshipsByEntity(@Nonnull Urn urn, deletionSQL.execute(); } - @Override + /** + * Persist the given list of relationships to the local relationship tables. + * @param relationships the list of relationships to be persisted + * @param removalOption whether to remove existing relationship of the same type + * @param isTestMode whether to use test schema + * @param urn Urn of the entity to update relationships. + * For Relationship V1: Optional, can be source or destination urn. + * For Relationship V2: Required, is the source urn. + */ public void addRelationships(@Nonnull List relationships, - @Nonnull RemovalOption removalOption, boolean isTestMode) { + @Nonnull RemovalOption removalOption, @Nonnull boolean isTestMode, @Nullable Urn urn) { // split relationships by relationship type Map> relationshipGroupMap = relationships.stream() .collect(Collectors.groupingBy(relationship -> relationship.getClass().getCanonicalName())); // validate if all relationship groups have valid urns relationshipGroupMap.values().forEach(relationshipGroup - -> GraphUtils.checkSameUrn(relationshipGroup, removalOption, CommonColumnName.SOURCE, CommonColumnName.DESTINATION)); + -> GraphUtils.checkSameUrn(relationshipGroup, removalOption, CommonColumnName.SOURCE, CommonColumnName.DESTINATION, urn)); relationshipGroupMap.values().forEach(relationshipGroup -> { - addRelationshipGroup(relationshipGroup, removalOption, isTestMode); + addRelationshipGroup(relationshipGroup, removalOption, isTestMode, urn); }); } + @Override + public void addRelationships(@Nonnull List relationships, + @Nonnull RemovalOption removalOption, @Nonnull boolean isTestMode) { + addRelationships(relationships, removalOption, isTestMode, null); + } + @Override public void removeRelationships(@Nonnull List relationships) { for (RELATIONSHIP relationship : relationships) { @@ -122,8 +139,16 @@ public void removeEntities(@Nonnull List urns) { throw new UnsupportedOperationException("Local relationship does not support removing entity. Please consider using metadata entity table."); } + /** + * Add the given list of relationships to the local relationship tables. + * @param relationshipGroup the list of relationships to be persisted + * @param removalOption whether to remove existing relationship of the same type + * @param isTestMode whether to use test schema + * @param urn the source urn to be used for the relationships. Optional for Relationship V1. + * Needed for Relationship V2 because source is not included in the relationshipV2 metadata. + */ private void addRelationshipGroup(@Nonnull final List relationshipGroup, - @Nonnull RemovalOption removalOption, boolean isTestMode) { + @Nonnull RemovalOption removalOption, @Nonnull boolean isTestMode, @Nullable Urn urn) { if (relationshipGroup.size() == 0) { return; } @@ -133,12 +158,15 @@ private void addRelationshipGroup(@Nonnull // Process remove option to delete some local relationships if needed before adding new relationships. processRemovalOption(isTestMode ? SQLSchemaUtils.getTestRelationshipTableName(firstRelationship) - : SQLSchemaUtils.getRelationshipTableName(firstRelationship), firstRelationship, removalOption); + : SQLSchemaUtils.getRelationshipTableName(firstRelationship), firstRelationship, removalOption, urn); long now = Instant.now().toEpochMilli(); for (RELATIONSHIP relationship : relationshipGroup) { - Urn source = getSourceUrnFromRelationship(relationship); + // Relationship model V2 doesn't include source urn, it needs to be passed in. + // For relationship model V1, this given urn can be source urn or destination urn. + // For relationship model V2, this given urn can only be source urn. + Urn source = GraphUtils.getSourceUrnBasedOnRelationshipVersion(relationship, urn); Urn destination = getDestinationUrnFromRelationship(relationship); _server.createSqlUpdate(SQLStatementUtils.insertLocalRelationshipSQL( @@ -155,16 +183,23 @@ private void addRelationshipGroup(@Nonnull } } - @ParametersAreNonnullByDefault - private void processRemovalOption(String tableName, RELATIONSHIP relationship, - RemovalOption removalOption) { + /** + * Process the relationship removal in the DB tableName based on the removal option. + * @param tableName the table name of the relationship + * @param relationship the relationship to be removed + * @param removalOption the removal option + * @param urn the source urn to be used for the relationships. Optional for Relationship V1. + * Needed for Relationship V2 because source is not included in the relationshipV2 metadata. + */ + private void processRemovalOption(@Nonnull String tableName, + @Nonnull RELATIONSHIP relationship, @Nonnull RemovalOption removalOption, @Nullable Urn urn) { if (removalOption == RemovalOption.REMOVE_NONE) { return; } SqlUpdate deletionSQL = _server.createSqlUpdate(SQLStatementUtils.deleteLocalRelationshipSQL(tableName, removalOption)); - Urn source = getSourceUrnFromRelationship(relationship); + Urn source = GraphUtils.getSourceUrnBasedOnRelationshipVersion(relationship, urn); Urn destination = getDestinationUrnFromRelationship(relationship); if (removalOption == RemovalOption.REMOVE_ALL_EDGES_FROM_SOURCE_TO_DESTINATION) { diff --git a/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/localrelationship/EbeanLocalRelationshipWriterDAOTest.java b/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/localrelationship/EbeanLocalRelationshipWriterDAOTest.java index 7ba0cde13..6ac2f17ec 100644 --- a/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/localrelationship/EbeanLocalRelationshipWriterDAOTest.java +++ b/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/localrelationship/EbeanLocalRelationshipWriterDAOTest.java @@ -10,6 +10,7 @@ import com.linkedin.metadata.dao.utils.EmbeddedMariaInstance; import com.linkedin.metadata.dao.builder.BaseLocalRelationshipBuilder.LocalRelationshipUpdates; import com.linkedin.testing.BarUrnArray; +import com.linkedin.testing.RelationshipV2Bar; import com.linkedin.testing.localrelationship.AspectFooBar; import com.linkedin.testing.localrelationship.PairsWith; import com.linkedin.testing.urn.BarUrn; @@ -20,6 +21,7 @@ import java.io.IOException; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import org.testng.annotations.BeforeClass; @@ -179,6 +181,11 @@ public void testAddRelationshipWithRemoveAllEdgesFromSourceToDestination() throw @Test public void testAddRelationshipWithRemoveAllEdgesFromSource() throws URISyntaxException { + // Test cases for Relationship Model V1 + // set 3 existing relationships + // 1) "urn:li:bar:123" -> "urn:li:foo:123" + // 2) "urn:li:bar:123" -> "urn:li:foo:000" + // 3) "urn:li:bar:000" -> "urn:li:foo:123" _server.execute(Ebean.createSqlUpdate(insertRelationships("metadata_relationship_versionof", "urn:li:bar:123", "bar", "urn:li:foo:123", "foo"))); @@ -188,6 +195,8 @@ public void testAddRelationshipWithRemoveAllEdgesFromSource() throws URISyntaxEx _server.execute(Ebean.createSqlUpdate(insertRelationships("metadata_relationship_versionof", "urn:li:bar:000", "bar", "urn:li:foo:123", "foo"))); + // mock a new relationship update + // 1) "urn:li:bar:123" -> "urn:li:foo:123" AspectFooBar aspectFooBar = new AspectFooBar().setBars(new BarUrnArray(BarUrn.createFromString("urn:li:bar:123"))); List updates = new VersionOfLocalRelationshipBuilder(AspectFooBar.class) @@ -200,6 +209,11 @@ public void testAddRelationshipWithRemoveAllEdgesFromSource() throws URISyntaxEx _localRelationshipWriterDAO.processLocalRelationshipUpdates(FooUrn.createFromString("urn:li:foo:123"), updates, false); // After processing verification + // now the relationship table should have the following relationships: + // 1) "urn:li:bar:123" -> "urn:li:foo:123" (soft-deleted) + // 2) "urn:li:bar:123" -> "urn:li:foo:000" (soft-deleted) + // 3) "urn:li:bar:000" -> "urn:li:foo:123" + // 4) "urn:li:bar:123" -> "urn:li:foo:123" (newly inserted) List all = _server.createSqlQuery("select * from metadata_relationship_versionof").findList(); assertEquals(all.size(), 4); // Total number of edges @@ -216,6 +230,77 @@ public void testAddRelationshipWithRemoveAllEdgesFromSource() throws URISyntaxEx // Clean up _server.execute(Ebean.createSqlUpdate("truncate metadata_relationship_versionof")); + + // Test cases for Relationship Model V2 + // set 3 existing relationships + // 1) "urn:li:foo:1" -> "urn:li:bar:1" + // 2) "urn:li:foo:1" -> "urn:li:bar:2" + // 3) "urn:li:foo:2" -> "urn:li:bar:2" + _server.execute(Ebean.createSqlUpdate(insertRelationships("metadata_relationship_relationshipv2bar", + "urn:li:foo:1", "foo", + "urn:li:bar:1", "bar"))); + + _server.execute(Ebean.createSqlUpdate(insertRelationships("metadata_relationship_relationshipv2bar", + "urn:li:foo:1", "foo", + "urn:li:bar:2", "bar"))); + + _server.execute(Ebean.createSqlUpdate(insertRelationships("metadata_relationship_relationshipv2bar", + "urn:li:foo:2", "foo", + "urn:li:bar:2", "bar"))); + + // mock 3 new relationships + // 1) "urn:li:foo:1" -> "urn:li:bar:1" + // 2) "urn:li:foo:1" -> "urn:li:bar:22" + // 3) "urn:li:foo:1" -> "urn:li:bar:23" + List relationships = new ArrayList<>(); + RelationshipV2Bar relationship21 = new RelationshipV2Bar().setDestination( + RelationshipV2Bar.Destination.createWithDestinationBar(new BarUrn(1))); + RelationshipV2Bar relationship22 = new RelationshipV2Bar().setDestination( + RelationshipV2Bar.Destination.createWithDestinationBar(new BarUrn(22))); + RelationshipV2Bar relationship23 = new RelationshipV2Bar().setDestination( + RelationshipV2Bar.Destination.createWithDestinationBar(new BarUrn(23))); + relationships.add(relationship21); + relationships.add(relationship22); + relationships.add(relationship23); + + LocalRelationshipUpdates localRelationshipUpdates2 = new LocalRelationshipUpdates(relationships, + RelationshipV2Bar.class, + BaseGraphWriterDAO.RemovalOption.REMOVE_ALL_EDGES_FROM_SOURCE); + + List updates2 = new ArrayList<>(); + updates2.add(localRelationshipUpdates2); + + // Before processing new relationships, there should be 3 existing relationships + List before2 = _server.createSqlQuery("select * from metadata_relationship_relationshipv2bar").findList(); + assertEquals(before2.size(), 3); + + // process the 3 new relationships + // then the relationship table should have the following records: + // 1) "urn:li:foo:1" -> "urn:li:bar:1" (soft-deleted) + // 2) "urn:li:foo:1" -> "urn:li:bar:2" (soft-deleted) + // 3) "urn:li:foo:2" -> "urn:li:bar:2" + // 4) "urn:li:foo:1" -> "urn:li:bar:1" (newly inserted) + // 5) "urn:li:foo:1" -> "urn:li:bar:22" (newly inserted) + // 6) "urn:li:foo:1" -> "urn:li:bar:23" (newly inserted) + _localRelationshipWriterDAO.processLocalRelationshipUpdates(FooUrn.createFromString("urn:li:foo:1"), updates2, false); + + // After processing verification + List all2 = _server.createSqlQuery("select * from metadata_relationship_relationshipv2bar").findList(); + assertEquals(all2.size(), 6); // Total number of edges + + List softDeleted2 = _server.createSqlQuery("select * from metadata_relationship_relationshipv2bar where deleted_ts IS NOT NULL").findList(); + assertEquals(softDeleted2.size(), 2); // 2 edges are soft-deleted + + List newEdge2 = _server.createSqlQuery( + "select * from metadata_relationship_relationshipv2bar where source='urn:li:foo:1' and deleted_ts IS NULL").findList(); + assertEquals(newEdge2.size(), 3); // newly insert 3 edge + + List oldEdge2 = _server.createSqlQuery( + "select * from metadata_relationship_relationshipv2bar where source='urn:li:foo:2' and deleted_ts IS NULL").findList(); + assertEquals(oldEdge2.size(), 1); // untouched record + + // Clean up + _server.execute(Ebean.createSqlUpdate("truncate metadata_relationship_relationshipv2bar")); } diff --git a/dao-impl/ebean-dao/src/test/resources/ebean-local-relationship-create-all-with-non-dollar-virtual-column-names.sql b/dao-impl/ebean-dao/src/test/resources/ebean-local-relationship-create-all-with-non-dollar-virtual-column-names.sql index 4adddeaa9..48ffab9ca 100644 --- a/dao-impl/ebean-dao/src/test/resources/ebean-local-relationship-create-all-with-non-dollar-virtual-column-names.sql +++ b/dao-impl/ebean-dao/src/test/resources/ebean-local-relationship-create-all-with-non-dollar-virtual-column-names.sql @@ -4,6 +4,7 @@ DROP TABLE IF EXISTS metadata_relationship_ownedby; DROP TABLE IF EXISTS metadata_relationship_pairswith; DROP TABLE IF EXISTS metadata_relationship_versionof; DROP TABLE IF EXISTS metadata_relationship_consumefrom; +DROP TABLE IF EXISTS metadata_relationship_relationshipv2bar; DROP TABLE IF EXISTS metadata_entity_foo; DROP TABLE IF EXISTS metadata_entity_bar; @@ -85,6 +86,20 @@ CREATE TABLE IF NOT EXISTS metadata_relationship_consumefrom ( PRIMARY KEY (id) ); +CREATE TABLE IF NOT EXISTS metadata_relationship_relationshipv2bar ( + id BIGINT NOT NULL AUTO_INCREMENT, + metadata LONGTEXT NOT NULL, + source VARCHAR(1000) NOT NULL, + source_type VARCHAR(100) NOT NULL, + destination VARCHAR(1000) NOT NULL, + destination_type VARCHAR(100) NOT NULL, + lastmodifiedon TIMESTAMP NOT NULL, + lastmodifiedby VARCHAR(255) NOT NULL, + deleted_ts DATETIME(6) DEFAULT NULL, + PRIMARY KEY (id) + ); + + -- initialize foo entity table CREATE TABLE IF NOT EXISTS metadata_entity_foo ( urn VARCHAR(100) NOT NULL, diff --git a/dao-impl/ebean-dao/src/test/resources/ebean-local-relationship-dao-create-all.sql b/dao-impl/ebean-dao/src/test/resources/ebean-local-relationship-dao-create-all.sql index d80c95e81..f627627a3 100644 --- a/dao-impl/ebean-dao/src/test/resources/ebean-local-relationship-dao-create-all.sql +++ b/dao-impl/ebean-dao/src/test/resources/ebean-local-relationship-dao-create-all.sql @@ -4,6 +4,7 @@ DROP TABLE IF EXISTS metadata_relationship_ownedby; DROP TABLE IF EXISTS metadata_relationship_pairswith; DROP TABLE IF EXISTS metadata_relationship_versionof; DROP TABLE IF EXISTS metadata_relationship_consumefrom; +DROP TABLE IF EXISTS metadata_relationship_relationshipv2bar; DROP TABLE IF EXISTS metadata_entity_foo; DROP TABLE IF EXISTS metadata_entity_bar; @@ -99,6 +100,19 @@ CREATE TABLE IF NOT EXISTS metadata_relationship_consumefrom ( PRIMARY KEY (id) ); +CREATE TABLE IF NOT EXISTS metadata_relationship_relationshipv2bar ( + id BIGINT NOT NULL AUTO_INCREMENT, + metadata LONGTEXT NOT NULL, + source VARCHAR(1000) NOT NULL, + source_type VARCHAR(100) NOT NULL, + destination VARCHAR(1000) NOT NULL, + destination_type VARCHAR(100) NOT NULL, + lastmodifiedon TIMESTAMP NOT NULL, + lastmodifiedby VARCHAR(255) NOT NULL, + deleted_ts DATETIME(6) DEFAULT NULL, + PRIMARY KEY (id) +); + -- initialize foo entity table CREATE TABLE IF NOT EXISTS metadata_entity_foo ( urn VARCHAR(100) NOT NULL,