From c6be65728c4be54148b2260f4483f7f239c3567e Mon Sep 17 00:00:00 2001 From: Yang Yang Date: Tue, 8 Oct 2024 13:25:48 -0700 Subject: [PATCH] Consolidate resource-lix into datahub-gma (#446) Co-authored-by: Yang Yang --- .../metadata/restli/BaseEntityResource.java | 4 +- ...sourceLix.java => LegacyResourceImpl.java} | 10 +- .../restli/lix/RampedResourceImpl.java | 86 +++++++++++++ .../restli/BaseEntityResourceTest.java | 114 ++++-------------- .../BaseEntitySimpleKeyResourceTest.java | 86 +------------ .../restli/lix/DummyResourceLixTest.java | 31 +++++ 6 files changed, 155 insertions(+), 176 deletions(-) rename restli-resources/src/main/java/com/linkedin/metadata/restli/lix/{DummyResourceLix.java => LegacyResourceImpl.java} (81%) create mode 100644 restli-resources/src/main/java/com/linkedin/metadata/restli/lix/RampedResourceImpl.java create mode 100644 restli-resources/src/test/java/com/linkedin/metadata/restli/lix/DummyResourceLixTest.java diff --git a/restli-resources/src/main/java/com/linkedin/metadata/restli/BaseEntityResource.java b/restli-resources/src/main/java/com/linkedin/metadata/restli/BaseEntityResource.java index da5f3154e..8d29c98d8 100644 --- a/restli-resources/src/main/java/com/linkedin/metadata/restli/BaseEntityResource.java +++ b/restli-resources/src/main/java/com/linkedin/metadata/restli/BaseEntityResource.java @@ -21,7 +21,7 @@ import com.linkedin.metadata.query.IndexSortCriterion; import com.linkedin.metadata.query.ListResultMetadata; import com.linkedin.metadata.query.MapMetadata; -import com.linkedin.metadata.restli.lix.DummyResourceLix; +import com.linkedin.metadata.restli.lix.RampedResourceImpl; import com.linkedin.metadata.restli.lix.ResourceLix; import com.linkedin.parseq.Task; import com.linkedin.restli.common.EmptyRecord; @@ -96,7 +96,7 @@ public abstract class BaseEntityResource< private final Class _assetClass; protected final Class _urnClass; protected BaseTrackingManager _trackingManager = null; - private ResourceLix _defaultResourceLix = new DummyResourceLix(); + private ResourceLix _defaultResourceLix = new RampedResourceImpl(); /** * This method is to be overriden by specific resource endpoint implementation with real lix impl. diff --git a/restli-resources/src/main/java/com/linkedin/metadata/restli/lix/DummyResourceLix.java b/restli-resources/src/main/java/com/linkedin/metadata/restli/lix/LegacyResourceImpl.java similarity index 81% rename from restli-resources/src/main/java/com/linkedin/metadata/restli/lix/DummyResourceLix.java rename to restli-resources/src/main/java/com/linkedin/metadata/restli/lix/LegacyResourceImpl.java index 1dbad9f39..aac20fbaf 100644 --- a/restli-resources/src/main/java/com/linkedin/metadata/restli/lix/DummyResourceLix.java +++ b/restli-resources/src/main/java/com/linkedin/metadata/restli/lix/LegacyResourceImpl.java @@ -4,8 +4,12 @@ import javax.annotation.Nullable; -public class DummyResourceLix implements ResourceLix { - +/** + * Legacy Resource will always choose using the old MG Kernel logic, equivalent to Lix is always 'control'. Legacy + * Resource is only used for the GMSes are on the maintenance mode (using legacy models Snapshot, Aspect Union) + * and are not planning to evolve (e.g. the legacy. AIM ). Usage: assign to 'resourceLix' fields at each GMS entity resource. + */ +public class LegacyResourceImpl implements ResourceLix { @Override public boolean testGet(@Nonnull String urn, @Nonnull String entityType) { return false; @@ -80,4 +84,4 @@ public boolean testSearch(@Nullable String urnType) { public boolean testSearchV2(@Nullable String urnType) { return false; } -} \ No newline at end of file +} diff --git a/restli-resources/src/main/java/com/linkedin/metadata/restli/lix/RampedResourceImpl.java b/restli-resources/src/main/java/com/linkedin/metadata/restli/lix/RampedResourceImpl.java new file mode 100644 index 000000000..a6783709b --- /dev/null +++ b/restli-resources/src/main/java/com/linkedin/metadata/restli/lix/RampedResourceImpl.java @@ -0,0 +1,86 @@ +package com.linkedin.metadata.restli.lix; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + + +/** + * Ramped Resource will always choose using the new MG Kernel logic, equivalent to Lix is always 'treatment'. + * Usage: assign to 'resourceLix' fields at each GMS entity resource. + */ +public class RampedResourceImpl implements ResourceLix { + @Override + public boolean testGet(@Nonnull String urn, @Nonnull String entityType) { + return true; + } + + @Override + public boolean testBatchGet(@Nullable String urn, @Nullable String entityType) { + return true; + } + + @Override + public boolean testBatchGetWithErrors(@Nullable String urn, @Nullable String type) { + return true; + } + + @Override + public boolean testGetSnapshot(@Nullable String urn, @Nullable String entityType) { + return true; + } + + @Override + public boolean testBackfillLegacy(@Nullable String urn, @Nullable String entityType) { + return true; + } + + @Override + public boolean testBackfillWithUrns(@Nullable String urn, @Nullable String entityType) { + return true; + } + + @Override + public boolean testEmitNoChangeMetadataAuditEvent(@Nullable String urn, @Nullable String entityType) { + return true; + } + + @Override + public boolean testBackfillWithNewValue(@Nullable String urn, @Nullable String entityType) { + return true; + } + + @Override + public boolean testBackfillEntityTables(@Nullable String urn, @Nullable String entityType) { + return true; + } + + @Override + public boolean testBackfillRelationshipTables(@Nullable String urn, @Nullable String entityType) { + return true; + } + + @Override + public boolean testBackfill(@Nonnull String assetType, @Nonnull String mode) { + return true; + } + + @Override + public boolean testFilter(@Nonnull String assetType) { + return true; + } + + @Override + public boolean testGetAll(@Nullable String urnType) { + return true; + } + + @Override + public boolean testSearch(@Nullable String urnType) { + return true; + } + + @Override + public boolean testSearchV2(@Nullable String urnType) { + return true; + } +} diff --git a/restli-resources/src/test/java/com/linkedin/metadata/restli/BaseEntityResourceTest.java b/restli-resources/src/test/java/com/linkedin/metadata/restli/BaseEntityResourceTest.java index 44f0400fd..7085e49fe 100644 --- a/restli-resources/src/test/java/com/linkedin/metadata/restli/BaseEntityResourceTest.java +++ b/restli-resources/src/test/java/com/linkedin/metadata/restli/BaseEntityResourceTest.java @@ -24,6 +24,7 @@ import com.linkedin.metadata.query.IndexSortCriterion; import com.linkedin.metadata.query.MapMetadata; import com.linkedin.metadata.query.SortOrder; +import com.linkedin.metadata.restli.lix.LegacyResourceImpl; import com.linkedin.metadata.restli.lix.ResourceLix; import com.linkedin.parseq.BaseEngineTest; import com.linkedin.restli.common.ComplexResourceKey; @@ -62,7 +63,6 @@ import java.util.Set; import java.util.stream.Collectors; import javax.annotation.Nonnull; -import javax.annotation.Nullable; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -82,6 +82,11 @@ class TestResource extends BaseEntityResource, EntityValue, FooUrn, EntitySnapshot, EntityAspectUnion, InternalEntitySnapshot, InternalEntityAspectUnion, EntityAsset> { + @Override + protected ResourceLix getResourceLix() { + return new LegacyResourceImpl(); + } + public TestResource() { super(EntitySnapshot.class, EntityAspectUnion.class, FooUrn.class, InternalEntitySnapshot.class, InternalEntityAspectUnion.class, EntityAsset.class); @@ -160,86 +165,6 @@ class TestInternalResource extends BaseEntityResource, EntityValue, FooUrn, EntitySnapshot, EntityAspectUnion, InternalEntitySnapshot, InternalEntityAspectUnion, EntityAsset> { - @Override - protected ResourceLix getResourceLix() { - return new ResourceLix() { - @Override - public boolean testGet(@Nonnull String urn, @Nonnull String entityType) { - return true; - } - - @Override - public boolean testBatchGet(@Nullable String urn, @Nullable String entityType) { - return true; - } - - @Override - public boolean testBatchGetWithErrors(@Nullable String urn, @Nullable String type) { - return false; - } - - @Override - public boolean testGetSnapshot(@Nullable String urn, @Nullable String entityType) { - return true; - } - - @Override - public boolean testBackfillLegacy(@Nullable String urn, @Nullable String entityType) { - return false; - } - - @Override - public boolean testBackfillWithUrns(@Nullable String urn, @Nullable String entityType) { - return false; - } - - @Override - public boolean testEmitNoChangeMetadataAuditEvent(@Nullable String urn, @Nullable String entityType) { - return false; - } - - @Override - public boolean testBackfillWithNewValue(@Nullable String urn, @Nullable String entityType) { - return false; - } - - @Override - public boolean testBackfillEntityTables(@Nullable String urn, @Nullable String entityType) { - return false; - } - - @Override - public boolean testBackfillRelationshipTables(@Nullable String urn, @Nullable String entityType) { - return false; - } - - @Override - public boolean testBackfill(@Nonnull String assetType, @Nonnull String mode) { - return false; - } - - @Override - public boolean testFilter(@Nonnull String assetType) { - return true; - } - - @Override - public boolean testGetAll(@Nullable String urnType) { - return false; - } - - @Override - public boolean testSearch(@Nullable String urnType) { - return false; - } - - @Override - public boolean testSearchV2(@Nullable String urnType) { - return false; - } - }; - } - public TestInternalResource() { super(EntitySnapshot.class, EntityAspectUnion.class, FooUrn.class, InternalEntitySnapshot.class, InternalEntityAspectUnion.class, EntityAsset.class); @@ -518,7 +443,8 @@ public void testBatchGetWithErrorsUrnsNotFound() { .thenReturn(Collections.emptyMap()); BatchResult, EntityValue> result = - runAndWait(_resource.batchGetWithErrors(ImmutableSet.of(makeResourceKey(urn1), makeResourceKey(urn2)), aspectNames)); + runAndWait( + _resource.batchGetWithErrors(ImmutableSet.of(makeResourceKey(urn1), makeResourceKey(urn2)), aspectNames)); // convert BatchResult, EntityValue> to BatchResult BatchResult batchResultMap = convertBatchResult(result); @@ -596,7 +522,8 @@ public void testBatchGetWithErrorsSpecificAspectsPartialSuccess() { .thenReturn(ImmutableMap.of(aspectFooKey1, Optional.of(foo), aspectBarKey2, Optional.of(bar))); BatchResult, EntityValue> result = - runAndWait(_resource.batchGetWithErrors(ImmutableSet.of(makeResourceKey(urn1), makeResourceKey(urn2)), aspectNames)); + runAndWait( + _resource.batchGetWithErrors(ImmutableSet.of(makeResourceKey(urn1), makeResourceKey(urn2)), aspectNames)); // convert BatchResult, EntityValue> to BatchResult BatchResult batchResultMap = convertBatchResult(result); @@ -635,7 +562,8 @@ public void testBatchGetWithErrorsUrnsPartialSuccess() { .thenReturn(ImmutableMap.of(aspectFooKey1, Optional.of(foo), aspectBarKey2, Optional.empty())); BatchResult, EntityValue> result = - runAndWait(_resource.batchGetWithErrors(ImmutableSet.of(makeResourceKey(urn1), makeResourceKey(urn2)), aspectNames)); + runAndWait( + _resource.batchGetWithErrors(ImmutableSet.of(makeResourceKey(urn1), makeResourceKey(urn2)), aspectNames)); // convert BatchResult, EntityValue> to BatchResult BatchResult batchResultMap = convertBatchResult(result); @@ -962,7 +890,8 @@ public void testBackfillWithNewValue() { AspectBar bar1 = new AspectBar().setValue("bar1"); AspectBar bar2 = new AspectBar().setValue("bar2"); String[] aspects = new String[]{"com.linkedin.testing.AspectFoo", "com.linkedin.testing.AspectBar"}; - when(_mockLocalDAO.backfillWithNewValue(_resource.parseAspectsParam(aspects, false), ImmutableSet.of(urn1, urn2))) + when(_mockLocalDAO.backfillWithNewValue( + _resource.parseAspectsParam(aspects, false), ImmutableSet.of(urn1, urn2))) .thenReturn( ImmutableMap.of(urn1, ImmutableMap.of(AspectFoo.class, Optional.of(foo1), AspectBar.class, Optional.of(bar1)), urn2, ImmutableMap.of(AspectBar.class, Optional.of(bar2))) @@ -1001,7 +930,8 @@ public void testEmitNoChangeMetadataAuditEvent() { ); BackfillResult backfillResult = - runAndWait(_resource.emitNoChangeMetadataAuditEvent(new String[]{urn1.toString(), urn2.toString()}, aspects, + runAndWait( + _resource.emitNoChangeMetadataAuditEvent(new String[]{urn1.toString(), urn2.toString()}, aspects, IngestionMode.BACKFILL)); assertEquals(backfillResult.getEntities().size(), 2); @@ -1034,7 +964,8 @@ public void testEmitNoChangeMetadataAuditEventBootstrap() { ); BackfillResult backfillResult = - runAndWait(_resource.emitNoChangeMetadataAuditEvent(new String[]{urn1.toString(), urn2.toString()}, aspects, + runAndWait( + _resource.emitNoChangeMetadataAuditEvent(new String[]{urn1.toString(), urn2.toString()}, aspects, IngestionMode.BOOTSTRAP)); assertEquals(backfillResult.getEntities().size(), 2); @@ -1062,7 +993,8 @@ public void testEmitNoChangeMetadataAuditEventNoResult() { String[] aspects = new String[]{"com.linkedin.testing.AspectFoo", "com.linkedin.testing.AspectBar"}; BackfillResult backfillResult = - runAndWait(_resource.emitNoChangeMetadataAuditEvent(new String[]{urn1.toString(), urn2.toString()}, aspects, + runAndWait( + _resource.emitNoChangeMetadataAuditEvent(new String[]{urn1.toString(), urn2.toString()}, aspects, IngestionMode.LIVE)); verify(_mockLocalDAO, times(0)).backfill(any(BackfillMode.class), any(Set.class), any(Set.class)); assertFalse(backfillResult.hasEntities()); @@ -1082,7 +1014,8 @@ public void testBackfillRelationshipTables() { List relationships = Collections.singletonList(updates); when(_mockLocalDAO.backfillLocalRelationships(fooUrn, AspectFoo.class)).thenReturn(relationships); - BackfillResult backfillResult = runAndWait(_resource.backfillRelationshipTables(new String[]{fooUrn.toString()}, aspects)); + BackfillResult backfillResult = runAndWait( + _resource.backfillRelationshipTables(new String[]{fooUrn.toString()}, aspects)); assertTrue(backfillResult.hasRelationships()); assertEquals(backfillResult.getRelationships().size(), 1); @@ -1171,7 +1104,8 @@ public void testFilterFromIndexEmptyAspects() { .build(); when(_mockLocalDAO.listUrns(null, indexSortCriterion, 0, 2)).thenReturn(urnsListResult); ListResult - listResultActual = runAndWait(_resource.filter(null, indexSortCriterion, new String[0], new PagingContext(0, 2))); + listResultActual = runAndWait( + _resource.filter(null, indexSortCriterion, new String[0], new PagingContext(0, 2))); List actualValues = listResultActual.getValues(); assertEquals(actualValues.size(), 2); assertEquals(actualValues.get(0), new EntityValue()); diff --git a/restli-resources/src/test/java/com/linkedin/metadata/restli/BaseEntitySimpleKeyResourceTest.java b/restli-resources/src/test/java/com/linkedin/metadata/restli/BaseEntitySimpleKeyResourceTest.java index 3ccaf9fe3..3ccfb9244 100644 --- a/restli-resources/src/test/java/com/linkedin/metadata/restli/BaseEntitySimpleKeyResourceTest.java +++ b/restli-resources/src/test/java/com/linkedin/metadata/restli/BaseEntitySimpleKeyResourceTest.java @@ -9,6 +9,7 @@ import com.linkedin.metadata.dao.BaseLocalDAO; import com.linkedin.metadata.dao.utils.ModelUtils; import com.linkedin.metadata.dao.utils.RecordUtils; +import com.linkedin.metadata.restli.lix.LegacyResourceImpl; import com.linkedin.metadata.restli.lix.ResourceLix; import com.linkedin.parseq.BaseEngineTest; import com.linkedin.restli.common.HttpStatus; @@ -38,7 +39,6 @@ import java.util.Set; import java.util.stream.Collectors; import javax.annotation.Nonnull; -import javax.annotation.Nullable; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -408,6 +408,10 @@ private class TestResource extends InternalEntityAspectUnion.class, EntityAsset.class); } + @Override + protected ResourceLix getResourceLix() { + return new LegacyResourceImpl(); + } @Override @@ -481,86 +485,6 @@ private class TestInternalResource extends BaseEntityResource { - @Override - protected ResourceLix getResourceLix() { - return new ResourceLix() { - @Override - public boolean testGet(@Nonnull String urn, @Nonnull String entityType) { - return false; - } - - @Override - public boolean testBatchGet(@Nullable String urn, @Nullable String entityType) { - return true; - } - - @Override - public boolean testBatchGetWithErrors(@Nullable String urn, @Nullable String type) { - return false; - } - - @Override - public boolean testGetSnapshot(@Nullable String urn, @Nullable String entityType) { - return false; - } - - @Override - public boolean testBackfillLegacy(@Nullable String urn, @Nullable String entityType) { - return false; - } - - @Override - public boolean testBackfillWithUrns(@Nullable String urn, @Nullable String entityType) { - return false; - } - - @Override - public boolean testEmitNoChangeMetadataAuditEvent(@Nullable String urn, @Nullable String entityType) { - return false; - } - - @Override - public boolean testBackfillWithNewValue(@Nullable String urn, @Nullable String entityType) { - return false; - } - - @Override - public boolean testBackfillEntityTables(@Nullable String urn, @Nullable String entityType) { - return false; - } - - @Override - public boolean testBackfillRelationshipTables(@Nullable String urn, @Nullable String entityType) { - return false; - } - - @Override - public boolean testBackfill(@Nonnull String assetType, @Nonnull String mode) { - return false; - } - - @Override - public boolean testFilter(@Nonnull String assetType) { - return false; - } - - @Override - public boolean testGetAll(@Nullable String urnType) { - return false; - } - - @Override - public boolean testSearch(@Nullable String urnType) { - return false; - } - - @Override - public boolean testSearchV2(@Nullable String urnType) { - return false; - } - }; - } - TestInternalResource() { super(EntitySnapshot.class, EntityAspectUnion.class, InternalEntitySnapshot.class, InternalEntityAspectUnion.class, EntityAsset.class); diff --git a/restli-resources/src/test/java/com/linkedin/metadata/restli/lix/DummyResourceLixTest.java b/restli-resources/src/test/java/com/linkedin/metadata/restli/lix/DummyResourceLixTest.java new file mode 100644 index 000000000..f95798a4e --- /dev/null +++ b/restli-resources/src/test/java/com/linkedin/metadata/restli/lix/DummyResourceLixTest.java @@ -0,0 +1,31 @@ +package com.linkedin.metadata.restli.lix; + +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import javax.annotation.Nonnull; +import org.testng.annotations.Test; + +import static org.testng.Assert.*; + + +public class DummyResourceLixTest { + @Test + public void testDefaultResource() throws Exception { + testResourceSwitch(new LegacyResourceImpl(), false); + testResourceSwitch(new RampedResourceImpl(), true); + } + + private void testResourceSwitch(@Nonnull ResourceLix instance, boolean expected) throws Exception { + Class resourceLixClass = instance.getClass(); + for (Method method : resourceLixClass.getMethods()) { + if (method.getName().startsWith("test") && !Modifier.isPrivate(method.getModifiers())) { + Class[] parameterTypes = method.getParameterTypes(); + Object[] params = new Object[parameterTypes.length]; + for (int i = 0; i < params.length; i++) { + params[i] = null; + } + assertEquals(method.invoke(instance, params), expected); + } + } + } +} \ No newline at end of file