diff --git a/dao-api/src/main/java/com/linkedin/metadata/dao/GenericLocalDAO.java b/dao-api/src/main/java/com/linkedin/metadata/dao/GenericLocalDAO.java index 3eb980a2a..0f62fe33c 100644 --- a/dao-api/src/main/java/com/linkedin/metadata/dao/GenericLocalDAO.java +++ b/dao-api/src/main/java/com/linkedin/metadata/dao/GenericLocalDAO.java @@ -54,4 +54,13 @@ void save(@Nonnull Urn urn, @Nonnull Class aspectClass, @Nonnull String metadata */ Map, Optional>> backfill(@Nonnull BackfillMode mode, @Nonnull Map>> urnToAspect); + + /** + * Delete the metadata from database. + * + * @param urn The identifier of the entity which the metadata is associated with. + * @param aspectClass The aspect class for the metadata. + * @param auditStamp audit stamp containing information on who and when the metadata is deleted. + */ + void delete(@Nonnull Urn urn, @Nonnull Class aspectClass, @Nonnull AuditStamp auditStamp); } diff --git a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanGenericLocalDAO.java b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanGenericLocalDAO.java index 9e26971ae..cb5074ae8 100644 --- a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanGenericLocalDAO.java +++ b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanGenericLocalDAO.java @@ -87,13 +87,25 @@ public void setEqualityTesters(Map equalityTesters */ public void save(@Nonnull Urn urn, @Nonnull Class aspectClass, @Nonnull String metadata, @Nonnull AuditStamp auditStamp, @Nullable IngestionTrackingContext trackingContext, @Nullable IngestionMode ingestionMode) { + saveCommon(urn, aspectClass, metadata, auditStamp, trackingContext, ingestionMode); + } + + /* a common helper method for saving metadata */ + void saveCommon(@Nonnull Urn urn, @Nonnull Class aspectClass, @Nullable String metadata, @Nonnull AuditStamp auditStamp, + @Nullable IngestionTrackingContext trackingContext, @Nullable IngestionMode ingestionMode) { runInTransactionWithRetry(() -> { final Optional latest = queryLatest(urn, aspectClass); - RecordTemplate newValue = toRecordTemplate(aspectClass, metadata); + + RecordTemplate newValue = null; + if (metadata != null) { + newValue = toRecordTemplate(aspectClass, metadata); + } if (!latest.isPresent()) { saveLatest(urn, aspectClass, newValue, null, auditStamp, null); - _producer.produceAspectSpecificMetadataAuditEvent(urn, null, newValue, auditStamp, trackingContext, ingestionMode); + if (!shouldSkipMAEUpdate(newValue)) { + _producer.produceAspectSpecificMetadataAuditEvent(urn, null, newValue, auditStamp, trackingContext, ingestionMode); + } } else { RecordTemplate currentValue = toRecordTemplate(aspectClass, latest.get().getAspect()); final AuditStamp oldAuditStamp = latest.get().getExtraInfo() == null ? null : latest.get().getExtraInfo().getAudit(); @@ -107,8 +119,13 @@ public void save(@Nonnull Urn urn, @Nonnull Class aspectClass, @Nonnull String m } // Skip update if current value and new value are equal. - if (!areEqual(currentValue, newValue, _equalityTesters.get(aspectClass))) { - saveLatest(urn, aspectClass, newValue, currentValue, auditStamp, latest.get().getExtraInfo().getAudit()); + // currentValue is always not null in this case + if (newValue != null && areEqual(currentValue, newValue, _equalityTesters.get(aspectClass))) { + return null; + } + saveLatest(urn, aspectClass, newValue, currentValue, auditStamp, latest.get().getExtraInfo().getAudit()); + + if (!shouldSkipMAEUpdate(newValue)) { _producer.produceAspectSpecificMetadataAuditEvent(urn, currentValue, newValue, auditStamp, trackingContext, ingestionMode); } } @@ -116,6 +133,11 @@ public void save(@Nonnull Urn urn, @Nonnull Class aspectClass, @Nonnull String m }, 5); } + private boolean shouldSkipMAEUpdate(@Nullable RecordTemplate newValue) { + // do not send MAE for null new value (deletion), to keep the same behavior as in BaseLocalDao + return newValue == null; + } + /** * Query the latest metadata from database. * @param urn The identifier of the entity which the metadata is associated with. @@ -128,7 +150,7 @@ public Optional queryLatest(@Nonnull Urn final PrimaryKey key = new PrimaryKey(urn.toString(), aspectName, LATEST_VERSION); EbeanMetadataAspect metadata = _server.find(EbeanMetadataAspect.class, key); - if (metadata == null || metadata.getMetadata() == null) { + if (metadata == null || metadata.getMetadata() == null || DELETED_VALUE.equals(metadata.getMetadata())) { return Optional.empty(); } @@ -170,6 +192,11 @@ public Map, Optional metadata = _genericLocalDAO.queryLatest(fooUrn, AspectFoo.class); + + // {"value":"foo"} is inserted later so it is the latest metadata. + assertTrue(metadata.isPresent()); + assertEquals(metadata.get().getAspect(), RecordUtils.toJsonString(aspectFoo)); + + reset(_producer); + + // Delete the record and verify it is deleted. + _genericLocalDAO.delete(fooUrn, AspectFoo.class, makeAuditStamp("tester")); + + metadata = _genericLocalDAO.queryLatest(fooUrn, AspectFoo.class); + assertFalse(metadata.isPresent()); + + // does not produce MAE for deletion + verify(_producer, times(0)).produceAspectSpecificMetadataAuditEvent(eq(fooUrn), + any(), any(), any(), any(), any()); + verifyNoMoreInteractions(_producer); + } + + @Test + public void testDeleteVoid() throws URISyntaxException { + FooUrn fooUrn = FooUrn.createFromString("urn:li:foo:1"); + + Optional metadata = _genericLocalDAO.queryLatest(fooUrn, AspectFoo.class); + + // no record is returned. + assertFalse(metadata.isPresent()); + + // Delete the record and verify no record is returned. + _genericLocalDAO.delete(fooUrn, AspectFoo.class, makeAuditStamp("tester")); + + metadata = _genericLocalDAO.queryLatest(fooUrn, AspectFoo.class); + assertFalse(metadata.isPresent()); + + // does not produce MAE for deletion + verify(_producer, times(0)).produceAspectSpecificMetadataAuditEvent(eq(fooUrn), + any(), any(), any(), any(), any()); + verifyNoMoreInteractions(_producer); + } + + @Test + public void testBackfillAfterDelete() throws URISyntaxException { + FooUrn fooUrn = FooUrn.createFromString("urn:li:foo:1"); + AspectFoo aspectFoo = new AspectFoo().setValue("foo"); + + _genericLocalDAO.save(fooUrn, AspectFoo.class, RecordUtils.toJsonString(aspectFoo), + makeAuditStamp("tester"), null, null); + + Map>> aspects = Collections.singletonMap(fooUrn, Collections.singleton(AspectFoo.class)); + + Map, Optional>> backfillResults + = _genericLocalDAO.backfill(BackfillMode.BACKFILL_ALL, aspects); + + assertEquals(backfillResults.size(), 1); + assertEquals(backfillResults.get(fooUrn).get(AspectFoo.class).get(), aspectFoo); + + + // verify no aspect will be backfilled after deletion + _genericLocalDAO.delete(fooUrn, AspectFoo.class, makeAuditStamp("tester")); + + backfillResults = _genericLocalDAO.backfill(BackfillMode.BACKFILL_ALL, aspects); + assertEquals(backfillResults.size(), 1); + assertEquals(backfillResults.get(fooUrn).size(), 0); + } } diff --git a/restli-resources/src/main/java/com/linkedin/metadata/restli/BaseEntityAgnosticAspectResource.java b/restli-resources/src/main/java/com/linkedin/metadata/restli/BaseEntityAgnosticAspectResource.java index 8ddc2164a..b7339252d 100644 --- a/restli-resources/src/main/java/com/linkedin/metadata/restli/BaseEntityAgnosticAspectResource.java +++ b/restli-resources/src/main/java/com/linkedin/metadata/restli/BaseEntityAgnosticAspectResource.java @@ -131,4 +131,22 @@ public Task backfill( throw new RestLiServiceException(HttpStatus.S_400_BAD_REQUEST, String.format("Urn %s is malformed.", urn)); } } + + @Action(name = ACTION_DELETE) + @Nonnull + public Task delete( + @ActionParam(PARAM_URN) @Nonnull String urn, + @ActionParam(PARAM_ASPECT_CLASS) @Nonnull String aspectClass) { + final AuditStamp auditStamp = getAuditor().requestAuditStamp(getContext().getRawRequestContext()); + + try { + Class clazz = this.getClass().getClassLoader().loadClass(aspectClass); + genericLocalDAO().delete(Urn.createFromCharSequence(urn), clazz, auditStamp); + return Task.value(null); + } catch (ClassNotFoundException e) { + throw new RestLiServiceException(HttpStatus.S_400_BAD_REQUEST, String.format("No such class %s.", aspectClass)); + } catch (URISyntaxException e) { + throw new RestLiServiceException(HttpStatus.S_400_BAD_REQUEST, String.format("Urn %s is malformed.", urn)); + } + } } \ No newline at end of file diff --git a/restli-resources/src/main/java/com/linkedin/metadata/restli/RestliConstants.java b/restli-resources/src/main/java/com/linkedin/metadata/restli/RestliConstants.java index 42622c5d2..cdf865947 100644 --- a/restli-resources/src/main/java/com/linkedin/metadata/restli/RestliConstants.java +++ b/restli-resources/src/main/java/com/linkedin/metadata/restli/RestliConstants.java @@ -31,6 +31,7 @@ private RestliConstants() { } public static final String ACTION_RAW_INGEST_ASSET = "rawIngestAsset"; public static final String ACTION_LIST_URNS_FROM_INDEX = "listUrnsFromIndex"; public static final String ACTION_LIST_URNS = "listUrns"; + public static final String ACTION_DELETE = "delete"; public static final String PARAM_INPUT = "input"; public static final String PARAM_ASPECTS = "aspects"; public static final String PARAM_ASPECT = "aspect"; diff --git a/restli-resources/src/test/java/com/linkedin/metadata/restli/BaseEntityAgnosticAspectResourceTest.java b/restli-resources/src/test/java/com/linkedin/metadata/restli/BaseEntityAgnosticAspectResourceTest.java index 99a981deb..04a46e870 100644 --- a/restli-resources/src/test/java/com/linkedin/metadata/restli/BaseEntityAgnosticAspectResourceTest.java +++ b/restli-resources/src/test/java/com/linkedin/metadata/restli/BaseEntityAgnosticAspectResourceTest.java @@ -86,4 +86,13 @@ public void testBackfill() { verifyNoMoreInteractions(_mockLocalDAO); } + + @Test + public void testDelete() { + runAndWait(_resource.delete(ENTITY_URN.toString(), AspectFoo.class.getCanonicalName())); + + verify(_mockLocalDAO, times(1)).delete(eq(ENTITY_URN), eq(AspectFoo.class), any(AuditStamp.class)); + + verifyNoMoreInteractions(_mockLocalDAO); + } }