fix(backfill): ingest relationships + use createdon desc during entit…

…y table backfill (#487) Co-authored-by: Justin Donn <[email protected]>
linkedin · Dec 11, 2024 · b710127 · b710127
1 parent 9adcb21
commit b710127
Show file tree

Hide file tree

Showing 2 changed files with 22 additions and 6 deletions.
diff --git a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalDAO.java b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalDAO.java
@@ -640,13 +640,18 @@ public <ASPECT extends RecordTemplate> void updateEntityTables(@Nonnull URN urn,
     }
     PrimaryKey key = new PrimaryKey(urn.toString(), aspectClass.getCanonicalName(), LATEST_VERSION);
     runInTransactionWithRetry(() -> {
-      // use forUpdate() to lock the row during this transaction so that we can guarantee a consistent update
-      EbeanMetadataAspect result = _server.createQuery(EbeanMetadataAspect.class).setId(key).forUpdate().findOne();
+      // use forUpdate() to lock the row during this transaction so that we can guarantee a consistent update.
+      // order by createdon desc to get the latest value in the case where there are multiple results
+      EbeanMetadataAspect result = _server.createQuery(EbeanMetadataAspect.class).setId(key).orderBy().desc("createdon").forUpdate().findOne();
       if (result == null) {
         return null; // unused
       }
       AuditStamp auditStamp = makeAuditStamp(result);
-      _localAccess.add(urn, toRecordTemplate(aspectClass, result).orElse(null), aspectClass, auditStamp, null, false);
+      ASPECT aspect = toRecordTemplate(aspectClass, result).orElse(null);
+      _localAccess.add(urn, aspect, aspectClass, auditStamp, null, false);
+
+      // also insert any relationships associated with this aspect
+      handleRelationshipIngestion(urn, aspect, null, aspectClass, false);
       return null; // unused
     }, 1);
   }

diff --git a/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/EbeanLocalDAOTest.java b/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/EbeanLocalDAOTest.java
@@ -3021,10 +3021,12 @@ public void testUpdateEntityTables() throws URISyntaxException {
     // fill in old schema
     FooUrn urn1 = new FooUrn(1);
     AspectFoo foo = new AspectFoo().setValue("foo");
+    AspectFooBar fooBar = new AspectFooBar().setBars(new BarUrnArray(BarUrn.createFromString("urn:li:bar:1")));
     // this function only adds to old schema
-    String aspectName = AspectFoo.class.getCanonicalName();
-    EbeanMetadataAspect ema = getMetadata(urn1, aspectName, 0, foo);
-    _server.save(ema);
+    EbeanMetadataAspect ema1 = getMetadata(urn1, AspectFoo.class.getCanonicalName(), 0, foo);
+    _server.save(ema1);
+    EbeanMetadataAspect ema2 = getMetadata(urn1, AspectFooBar.class.getCanonicalName(), 0, fooBar);
+    _server.save(ema2);
 
     // check that there is nothing in the entity table right now
     if (_schemaConfig != SchemaConfig.OLD_SCHEMA_ONLY) {
@@ -3034,7 +3036,9 @@ public void testUpdateEntityTables() throws URISyntaxException {
 
     // perform the migration
     try {
+      dao.setLocalRelationshipBuilderRegistry(new SampleLocalRelationshipRegistryImpl());
       dao.updateEntityTables(urn1, AspectFoo.class);
+      dao.updateEntityTables(urn1, AspectFooBar.class);
       if (_schemaConfig == SchemaConfig.OLD_SCHEMA_ONLY) {
         // expect an exception here since there is no new schema to update
         fail();
@@ -3052,7 +3056,14 @@ public void testUpdateEntityTables() throws URISyntaxException {
     assertEquals(result.size(), 1);
     assertEquals(result.get(0).get("urn"), "urn:li:foo:1");
     assertNotNull(result.get(0).get("a_aspectfoo"));
+    assertNotNull(result.get(0).get("a_aspectfoobar"));
     assertNull(result.get(0).get("a_aspectbar"));
+
+    // make sure relationships are ingested too
+    result = _server.createSqlQuery("SELECT * FROM metadata_relationship_belongsto").findList();
+    assertEquals(result.size(), 1);
+    assertEquals(result.get(0).get("source"), "urn:li:foo:1");
+    assertEquals(result.get(0).get("destination"), "urn:li:bar:1");
   }
 
   @Test