diff --git a/dao-api/src/main/java/com/linkedin/metadata/dao/SchemaEvolutionManager.java b/dao-api/src/main/java/com/linkedin/metadata/dao/SchemaEvolutionManager.java index 367058817..539919f75 100644 --- a/dao-api/src/main/java/com/linkedin/metadata/dao/SchemaEvolutionManager.java +++ b/dao-api/src/main/java/com/linkedin/metadata/dao/SchemaEvolutionManager.java @@ -27,5 +27,6 @@ class Config { String connectionUrl; String password; String username; + String serviceIdentifier; } } diff --git a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalAccess.java b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalAccess.java index 6137c5dbd..4f3aa1e70 100644 --- a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalAccess.java +++ b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalAccess.java @@ -68,6 +68,7 @@ public class EbeanLocalAccess implements IEbeanLocalAccess private static final int DEFAULT_PAGE_SIZE = 1000; private static final String ASPECT_JSON_PLACEHOLDER = "__PLACEHOLDER__"; private static final String DEFAULT_ACTOR = "urn:li:principal:UNKNOWN"; + private static final String SERVICE_IDENTIFIER = "SERVICE_IDENTIFIER"; // key: table_name, // value: Set(column1, column2, column3 ...) @@ -494,10 +495,14 @@ private String toJsonString(@Nonnull URN urn) { @Nonnull private SchemaEvolutionManager createSchemaEvolutionManager(@Nonnull ServerConfig serverConfig) { + String identifier = serverConfig.getDataSourceConfig().getCustomProperties() != null + ? serverConfig.getDataSourceConfig().getCustomProperties().getOrDefault(SERVICE_IDENTIFIER, null) + : null; SchemaEvolutionManager.Config config = new SchemaEvolutionManager.Config( serverConfig.getDataSourceConfig().getUrl(), serverConfig.getDataSourceConfig().getPassword(), - serverConfig.getDataSourceConfig().getUsername()); + serverConfig.getDataSourceConfig().getUsername(), + identifier); return new FlywaySchemaEvolutionManager(config); } diff --git a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/FlywaySchemaEvolutionManager.java b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/FlywaySchemaEvolutionManager.java index 08c91107b..1d00741bb 100644 --- a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/FlywaySchemaEvolutionManager.java +++ b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/FlywaySchemaEvolutionManager.java @@ -5,19 +5,23 @@ import java.util.Properties; import org.flywaydb.core.Flyway; + public class FlywaySchemaEvolutionManager implements SchemaEvolutionManager { private static final String EVOLUTION_SCRIPTS_LOCATION = "script_directory"; private static final String VERSION_TABLE = "version_table"; private static final String CONFIG_FILE_TEMPLATE = "%s.conf"; + private static final String CONFIG_FILE_TEMPLATE2 = "%s-%s.conf"; private static final String DISABLE_CLEAN = "disable_clean"; private final Flyway _flyway; public FlywaySchemaEvolutionManager(Config config) { String databaseName = getDatabaseName(config); - InputStream configFile = getClass().getClassLoader().getResourceAsStream(String.format(CONFIG_FILE_TEMPLATE, databaseName)); + String serviceIdentifier = config.getServiceIdentifier(); + String configFileName = serviceIdentifier == null + ? String.format(CONFIG_FILE_TEMPLATE, databaseName) : String.format(CONFIG_FILE_TEMPLATE2, serviceIdentifier, databaseName); Properties configProp = new Properties(); - try { + try (InputStream configFile = getClass().getClassLoader().getResourceAsStream(configFileName)) { configProp.load(configFile); if (!configProp.containsKey(EVOLUTION_SCRIPTS_LOCATION) || !configProp.containsKey(VERSION_TABLE)) { @@ -42,7 +46,7 @@ public FlywaySchemaEvolutionManager(Config config) { @Override public void ensureSchemaUpToDate() { - _flyway.migrate(); + _flyway.migrate(); } @Override diff --git a/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/EbeanLocalAccessTestWithoutServiceIdentifier.java b/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/EbeanLocalAccessTestWithoutServiceIdentifier.java new file mode 100644 index 000000000..1811983b0 --- /dev/null +++ b/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/EbeanLocalAccessTestWithoutServiceIdentifier.java @@ -0,0 +1,420 @@ +package com.linkedin.metadata.dao; + +import com.google.common.io.Resources; +import com.linkedin.common.AuditStamp; +import com.linkedin.metadata.dao.urnpath.EmptyPathExtractor; +import com.linkedin.metadata.dao.utils.EmbeddedMariaInstance; +import com.linkedin.metadata.dao.utils.FooUrnPathExtractor; +import com.linkedin.metadata.dao.utils.RecordUtils; +import com.linkedin.metadata.dao.utils.SQLIndexFilterUtils; +import com.linkedin.metadata.query.Condition; +import com.linkedin.metadata.query.IndexCriterion; +import com.linkedin.metadata.query.IndexCriterionArray; +import com.linkedin.metadata.query.IndexFilter; +import com.linkedin.metadata.query.IndexGroupByCriterion; +import com.linkedin.metadata.query.IndexSortCriterion; +import com.linkedin.metadata.query.IndexValue; +import com.linkedin.metadata.query.SortOrder; +import com.linkedin.testing.AspectFoo; +import com.linkedin.testing.urn.BurgerUrn; +import com.linkedin.testing.urn.FooUrn; +import io.ebean.Ebean; +import io.ebean.EbeanServer; +import io.ebean.SqlRow; +import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.sql.Timestamp; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Factory; +import org.testng.annotations.Test; + +import static com.linkedin.common.AuditStamps.*; +import static com.linkedin.testing.TestUtils.*; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.AssertJUnit.assertFalse; +import static org.testng.AssertJUnit.assertNull; +import static org.testng.AssertJUnit.assertTrue; + + +/** + * This class tests have excatly the same content as EbeanLocalAccessTest, except with a different DB server config. + * It is expected to read default EbeanLocalAccessTest.conf file rather than test-EbeanLocalAccessTest.conf. + */ + +public class EbeanLocalAccessTestWithoutServiceIdentifier { + private static EbeanServer _server; + private static EbeanLocalAccess _ebeanLocalAccessFoo; + private static IEbeanLocalAccess _ebeanLocalAccessBurger; + private static long _now; + private final EBeanDAOConfig _ebeanConfig = new EBeanDAOConfig(); + + @Factory(dataProvider = "inputList") + public EbeanLocalAccessTestWithoutServiceIdentifier(boolean nonDollarVirtualColumnsEnabled) { + _ebeanConfig.setNonDollarVirtualColumnsEnabled(nonDollarVirtualColumnsEnabled); + } + + @DataProvider(name = "inputList") + public static Object[][] inputList() { + return new Object[][] { + { true }, + { false } + }; + } + + + @BeforeClass + public void init() { + _server = EmbeddedMariaInstance.getServerWithoutServiceIdentifier(EbeanLocalAccessTest.class.getSimpleName()); + _ebeanLocalAccessFoo = new EbeanLocalAccess<>(_server, EmbeddedMariaInstance.SERVER_CONFIG_MAP.get(_server.getName()), + FooUrn.class, new FooUrnPathExtractor(), _ebeanConfig.isNonDollarVirtualColumnsEnabled()); + _ebeanLocalAccessBurger = new EbeanLocalAccess<>(_server, EmbeddedMariaInstance.SERVER_CONFIG_MAP.get(_server.getName()), + BurgerUrn.class, new EmptyPathExtractor<>(), _ebeanConfig.isNonDollarVirtualColumnsEnabled()); + _now = System.currentTimeMillis(); + } + + @BeforeMethod + public void setupTest() throws IOException { + if (!_ebeanConfig.isNonDollarVirtualColumnsEnabled()) { + _server.execute(Ebean.createSqlUpdate( + Resources.toString(Resources.getResource("ebean-local-access-create-all.sql"), StandardCharsets.UTF_8))); + } else { + _server.execute(Ebean.createSqlUpdate(Resources.toString( + Resources.getResource("ebean-local-access-create-all-with-non-dollar-virtual-column-names.sql"), + StandardCharsets.UTF_8))); + } + // initialize data with metadata_entity_foo table with fooUrns from 0 ~ 99 + int numOfRecords = 100; + for (int i = 0; i < numOfRecords; i++) { + FooUrn fooUrn = makeFooUrn(i); + AspectFoo aspectFoo = new AspectFoo(); + aspectFoo.setValue(String.valueOf(i)); + AuditStamp auditStamp = makeAuditStamp("foo", System.currentTimeMillis()); + _ebeanLocalAccessFoo.add(fooUrn, aspectFoo, AspectFoo.class, auditStamp, null, false); + } + } + + @Test + public void testGetAspect() { + + // Given: metadata_entity_foo table with fooUrns from 0 ~ 99 + + FooUrn fooUrn = makeFooUrn(0); + AspectKey aspectKey = new AspectKey(AspectFoo.class, fooUrn, 0L); + + // When get AspectFoo from urn:li:foo:0 + List ebeanMetadataAspectList = + _ebeanLocalAccessFoo.batchGetUnion(Collections.singletonList(aspectKey), 1000, 0, false, false); + assertEquals(1, ebeanMetadataAspectList.size()); + + EbeanMetadataAspect ebeanMetadataAspect = ebeanMetadataAspectList.get(0); + + // Expect: the content of aspect foo is returned + assertEquals(AspectFoo.class.getCanonicalName(), ebeanMetadataAspect.getKey().getAspect()); + assertEquals(fooUrn.toString(), ebeanMetadataAspect.getKey().getUrn()); + assertEquals("{\"value\":\"0\"}", ebeanMetadataAspect.getMetadata()); + assertEquals("urn:li:testActor:foo", ebeanMetadataAspect.getCreatedBy()); + + // Make sure json can be deserialized to Aspect. + assertNotNull(RecordUtils.toRecordTemplate(AspectFoo.class, ebeanMetadataAspect.getMetadata())); + + // When get AspectFoo from urn:li:foo:9999 (does not exist) + FooUrn nonExistFooUrn = makeFooUrn(9999); + AspectKey nonExistKey = new AspectKey(AspectFoo.class, nonExistFooUrn, 0L); + ebeanMetadataAspectList = _ebeanLocalAccessFoo.batchGetUnion(Collections.singletonList(nonExistKey), 1000, 0, false, false); + + // Expect: get AspectFoo from urn:li:foo:9999 returns empty result + assertTrue(ebeanMetadataAspectList.isEmpty()); + } + + @Test + public void testListUrnsWithOffset() { + + // Given: metadata_entity_foo table with fooUrns from 0 ~ 99 + // When: finding urns where ids >= 25 and id < 50 sorting by ASC + + IndexFilter indexFilter = new IndexFilter(); + IndexCriterionArray indexCriterionArray = new IndexCriterionArray(); + + IndexCriterion indexCriterion1 = + SQLIndexFilterUtils.createIndexCriterion(AspectFoo.class, "value", Condition.GREATER_THAN_OR_EQUAL_TO, + IndexValue.create(25)); + IndexCriterion indexCriterion2 = + SQLIndexFilterUtils.createIndexCriterion(AspectFoo.class, "value", Condition.LESS_THAN, IndexValue.create(50)); + + indexCriterionArray.add(indexCriterion1); + indexCriterionArray.add(indexCriterion2); + indexFilter.setCriteria(indexCriterionArray); + + IndexSortCriterion indexSortCriterion = + SQLIndexFilterUtils.createIndexSortCriterion(AspectFoo.class, "value", SortOrder.ASCENDING); + + // When: list out results with start = 5 and pageSize = 5 + + ListResult listUrns = _ebeanLocalAccessFoo.listUrns(indexFilter, indexSortCriterion, 5, 5); + + assertEquals(5, listUrns.getValues().size()); + assertEquals(5, listUrns.getPageSize()); + assertEquals(10, listUrns.getNextStart()); + assertEquals(25, listUrns.getTotalCount()); + assertEquals(5, listUrns.getTotalPageCount()); + } + + @Test + public void testListUrnsWithLastUrn() throws URISyntaxException { + + // Given: metadata_entity_foo table with fooUrns from 0 ~ 99 + // When: finding urns where ids >= 25 and id < 50 sorting by ASC + + IndexFilter indexFilter = new IndexFilter(); + IndexCriterionArray indexCriterionArray = new IndexCriterionArray(); + + IndexCriterion indexCriterion1 = + SQLIndexFilterUtils.createIndexCriterion(AspectFoo.class, "value", Condition.GREATER_THAN_OR_EQUAL_TO, + IndexValue.create(25)); + IndexCriterion indexCriterion2 = + SQLIndexFilterUtils.createIndexCriterion(AspectFoo.class, "value", Condition.LESS_THAN, IndexValue.create(50)); + + indexCriterionArray.add(indexCriterion1); + indexCriterionArray.add(indexCriterion2); + indexFilter.setCriteria(indexCriterionArray); + + IndexSortCriterion indexSortCriterion = + SQLIndexFilterUtils.createIndexSortCriterion(AspectFoo.class, "value", SortOrder.ASCENDING); + + FooUrn lastUrn = new FooUrn(29); + + // When: list out results with lastUrn = 'urn:li:foo:29' and pageSize = 5 + List result1 = _ebeanLocalAccessFoo.listUrns(indexFilter, indexSortCriterion, lastUrn, 5); + + // Expect: 5 rows are returns (30~34) and the first element is 'urn:li:foo:30' + assertEquals(5, result1.size()); + assertEquals("30", result1.get(0).getId()); + + lastUrn = result1.get(result1.size() - 1); + + // When: list out results with lastUrn = 'urn:li:foo:34' and pageSize = 5, but with only a filter on the aspect + IndexCriterion indexCriterion3 = new IndexCriterion().setAspect(FooUrn.class.getCanonicalName()); + indexCriterionArray = new IndexCriterionArray(Collections.singleton(indexCriterion3)); + IndexFilter filter = new IndexFilter().setCriteria(indexCriterionArray); + List result2 = _ebeanLocalAccessFoo.listUrns(filter, indexSortCriterion, lastUrn, 5); + + // Expect: 5 rows are returns (35~39) and the first element is 'urn:li:foo:35' + assertEquals(5, result2.size()); + assertEquals("35", result2.get(0).getId()); + + // When: list urns with no filter, no sorting criterion, no last urn. + List result3 = _ebeanLocalAccessFoo.listUrns(null, null, null, 10); + + // 0, 1, 10, 11, 12, 13, 14, 15, 16, 17 + assertEquals(result3.size(), 10); + assertEquals(result3.get(0).getId(), "0"); + assertEquals(result3.get(9).getId(), "17"); + + // When: list urns with no filter, no sorting criterion + List result4 = _ebeanLocalAccessFoo.listUrns(null, null, new FooUrn(17), 10); + + // 18, 19, 2, 20, 21, 22, 23, 24, 25, 26 + assertEquals(result4.size(), 10); + assertEquals(result4.get(0).getId(), "18"); + assertEquals(result4.get(9).getId(), "26"); + } + + @Test + public void testExists() throws URISyntaxException { + // Given: metadata_entity_foo table with fooUrns from 0 ~ 99 + + // When: check whether urn:li:foo:0 exist + FooUrn foo0 = new FooUrn(0); + + // Expect: urn:li:foo:0 exists + assertTrue(_ebeanLocalAccessFoo.exists(foo0)); + + // When: check whether urn:li:foo:9999 exist + FooUrn foo9999 = new FooUrn(9999); + + // Expect: urn:li:foo:9999 does not exists + assertFalse(_ebeanLocalAccessFoo.exists(foo9999)); + } + + @Test + public void testListUrns() throws URISyntaxException { + // Given: metadata_entity_foo table with fooUrns from 0 ~ 99 + + // When: list urns from the 1st record, with 50 page size + ListResult fooUrnListResult = _ebeanLocalAccessFoo.listUrns(AspectFoo.class, 0, 50); + + // Expect: 50 results is returned and 100 total records + assertEquals(50, fooUrnListResult.getValues().size()); + assertEquals(100, fooUrnListResult.getTotalCount()); + + // When: list urns from the 55th record, with 50 page size + fooUrnListResult = _ebeanLocalAccessFoo.listUrns(AspectFoo.class, 55, 50); + + // Expect: 45 results is returned and 100 total records + assertEquals(45, fooUrnListResult.getValues().size()); + assertEquals(100, fooUrnListResult.getTotalCount()); + + // When: list urns from the 101th record, with 50 page size + fooUrnListResult = _ebeanLocalAccessFoo.listUrns(AspectFoo.class, 101, 50); + + // Expect: 0 results is returned and 100 total records + assertEquals(0, fooUrnListResult.getValues().size()); + assertEquals(100, fooUrnListResult.getTotalCount()); + } + + @Test + public void testCountAggregate() { + // Given: metadata_entity_foo table with fooUrns from 0 ~ 99 + + // When: count aggregate with filter value = 25 + IndexFilter indexFilter = new IndexFilter(); + IndexCriterionArray indexCriterionArray = new IndexCriterionArray(); + + IndexCriterion indexCriterion1 = + SQLIndexFilterUtils.createIndexCriterion(AspectFoo.class, "value", Condition.EQUAL, IndexValue.create(25)); + + indexCriterionArray.add(indexCriterion1); + indexFilter.setCriteria(indexCriterionArray); + + IndexGroupByCriterion indexGroupByCriterion = new IndexGroupByCriterion(); + indexGroupByCriterion.setPath("/value"); + indexGroupByCriterion.setAspect(AspectFoo.class.getCanonicalName()); + Map countMap = _ebeanLocalAccessFoo.countAggregate(indexFilter, indexGroupByCriterion); + + // Expect: there is 1 count for value 25 + assertEquals(countMap.get("25"), Long.valueOf(1)); + + // When: change foo:26's value to be 25 + + FooUrn fooUrn = makeFooUrn(26); + AspectFoo aspectFoo = new AspectFoo(); + aspectFoo.setValue(String.valueOf(25)); + AuditStamp auditStamp = makeAuditStamp("foo", System.currentTimeMillis()); + _ebeanLocalAccessFoo.add(fooUrn, aspectFoo, AspectFoo.class, auditStamp, null, false); + countMap = _ebeanLocalAccessFoo.countAggregate(indexFilter, indexGroupByCriterion); + + // Expect: there are 2 counts for value 25 + assertEquals(countMap.get("25"), Long.valueOf(2)); + } + + @Test + public void testEscapeSpecialCharInUrn() { + AspectFoo aspectFoo = new AspectFoo().setValue("test"); + AuditStamp auditStamp = makeAuditStamp("foo", System.currentTimeMillis()); + + // Single quote is a special char in SQL. + BurgerUrn johnsBurgerUrn1 = makeBurgerUrn("urn:li:burger:John's burger"); + _ebeanLocalAccessBurger.add(johnsBurgerUrn1, aspectFoo, AspectFoo.class, auditStamp, null, false); + + AspectKey aspectKey1 = new AspectKey(AspectFoo.class, johnsBurgerUrn1, 0L); + List ebeanMetadataAspectList = _ebeanLocalAccessFoo.batchGetUnion(Collections.singletonList(aspectKey1), 1, 0, false, false); + assertEquals(1, ebeanMetadataAspectList.size()); + assertEquals(ebeanMetadataAspectList.get(0).getKey().getUrn(), johnsBurgerUrn1.toString()); + + // Double quote is a special char in SQL. + BurgerUrn johnsBurgerUrn2 = makeBurgerUrn("urn:li:burger:John\"s burger"); + _ebeanLocalAccessBurger.add(johnsBurgerUrn2, aspectFoo, AspectFoo.class, auditStamp, null, false); + + AspectKey aspectKey2 = new AspectKey(AspectFoo.class, johnsBurgerUrn2, 0L); + ebeanMetadataAspectList = _ebeanLocalAccessFoo.batchGetUnion(Collections.singletonList(aspectKey2), 1, 0, false, false); + assertEquals(1, ebeanMetadataAspectList.size()); + assertEquals(ebeanMetadataAspectList.get(0).getKey().getUrn(), johnsBurgerUrn2.toString()); + + // Backslash is a special char in SQL. + BurgerUrn johnsBurgerUrn3 = makeBurgerUrn("urn:li:burger:John\\s burger"); + _ebeanLocalAccessBurger.add(johnsBurgerUrn3, aspectFoo, AspectFoo.class, auditStamp, null, false); + + AspectKey aspectKey3 = new AspectKey(AspectFoo.class, johnsBurgerUrn3, 0L); + ebeanMetadataAspectList = _ebeanLocalAccessFoo.batchGetUnion(Collections.singletonList(aspectKey3), 1, 0, false, false); + assertEquals(1, ebeanMetadataAspectList.size()); + assertEquals(ebeanMetadataAspectList.get(0).getKey().getUrn(), johnsBurgerUrn3.toString()); + } + + @Test + public void testUrnExtraction() { + FooUrn urn1 = makeFooUrn(1); + AspectFoo foo1 = new AspectFoo().setValue("foo"); + _ebeanLocalAccessFoo.add(urn1, foo1, AspectFoo.class, makeAuditStamp("actor", _now), null, false); + + List results; + // get content of virtual column + if (_ebeanConfig.isNonDollarVirtualColumnsEnabled()) { + results = _server.createSqlQuery("SELECT i_urn0fooId as id FROM metadata_entity_foo").findList(); + } else { + results = _server.createSqlQuery("SELECT i_urn$fooId as id FROM metadata_entity_foo").findList(); + } + assertEquals(100, results.size()); + + // ensure content is as expected + SqlRow firstResult = results.get(0); + assertEquals("0", firstResult.getString("id")); + } + + @Test + public void test() throws URISyntaxException { + FooUrn foo0 = new FooUrn(0); + // Expect: urn:li:foo:0 exists + assertTrue(_ebeanLocalAccessFoo.exists(foo0)); + } + + @Test + public void testFindLatestMetadataAspect() throws URISyntaxException { + // Given: metadata_aspect table has a record of foo0 + + FooUrn foo0 = new FooUrn(0); + AspectFoo f = new AspectFoo(); + f.setValue("foo"); + EbeanMetadataAspect ebeanMetadataAspect = new EbeanMetadataAspect(); + ebeanMetadataAspect.setKey(new EbeanMetadataAspect.PrimaryKey(foo0.toString(), f.getClass().getCanonicalName(), 0)); + ebeanMetadataAspect.setCreatedOn(new Timestamp(System.currentTimeMillis())); + ebeanMetadataAspect.setMetadata(f.toString()); + ebeanMetadataAspect.setCreatedBy("yanyang"); + _server.save(ebeanMetadataAspect); + + // When: check whether urn:li:foo:0 exist + // Expect: urn:li:foo:0 exists + ebeanMetadataAspect = EbeanLocalAccess.findLatestMetadataAspect(_server, foo0, AspectFoo.class); + assertNotNull(ebeanMetadataAspect); + assertEquals(ebeanMetadataAspect.getKey().getUrn(), foo0.toString()); + + // When: check whether urn:li:foo:9999 exist + FooUrn foo9999 = new FooUrn(9999); + + // Expect: urn:li:foo:9999 does not exists + assertNull(EbeanLocalAccess.findLatestMetadataAspect(_server, foo9999, AspectFoo.class)); + } + + @Test + public void testGetAspectNoSoftDeleteCheck() { + FooUrn fooUrn = makeFooUrn(0); + _ebeanLocalAccessFoo.add(fooUrn, null, AspectFoo.class, makeAuditStamp("foo", System.currentTimeMillis()), null, false); + AspectKey aspectKey = new AspectKey(AspectFoo.class, fooUrn, 0L); + List ebeanMetadataAspectList = + _ebeanLocalAccessFoo.batchGetUnion(Collections.singletonList(aspectKey), 1000, 0, false, false); + assertEquals(0, ebeanMetadataAspectList.size()); + + ebeanMetadataAspectList = + _ebeanLocalAccessFoo.batchGetUnion(Collections.singletonList(aspectKey), 1000, 0, true, false); + assertFalse(ebeanMetadataAspectList.isEmpty()); + assertEquals(fooUrn.toString(), ebeanMetadataAspectList.get(0).getKey().getUrn()); + } + + @Test + public void testCheckColumnExists() { + assertTrue(_ebeanLocalAccessFoo.checkColumnExists("metadata_entity_foo", "a_aspectfoo")); + assertFalse(_ebeanLocalAccessFoo.checkColumnExists("metadata_entity_foo", "a_aspect_not_exist")); + assertFalse(_ebeanLocalAccessFoo.checkColumnExists("metadata_entity_notexist", "a_aspectfoo")); + if (!_ebeanConfig.isNonDollarVirtualColumnsEnabled()) { + assertTrue(_ebeanLocalAccessFoo.checkColumnExists("metadata_entity_foo", "i_aspectfoo$value")); + } else { + assertTrue(_ebeanLocalAccessFoo.checkColumnExists("metadata_entity_foo", "i_aspectfoo0value")); + } + } +} \ No newline at end of file diff --git a/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/FlywaySchemaEvolutionManagerTest.java b/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/FlywaySchemaEvolutionManagerTest.java index aee9cd3a0..f7010c590 100644 --- a/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/FlywaySchemaEvolutionManagerTest.java +++ b/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/FlywaySchemaEvolutionManagerTest.java @@ -26,7 +26,8 @@ public void init() throws IOException { SchemaEvolutionManager.Config config = new SchemaEvolutionManager.Config( EmbeddedMariaInstance.SERVER_CONFIG_MAP.get(_server.getName()).getDataSourceConfig().getUrl(), EmbeddedMariaInstance.SERVER_CONFIG_MAP.get(_server.getName()).getDataSourceConfig().getPassword(), - EmbeddedMariaInstance.SERVER_CONFIG_MAP.get(_server.getName()).getDataSourceConfig().getUsername() + EmbeddedMariaInstance.SERVER_CONFIG_MAP.get(_server.getName()).getDataSourceConfig().getUsername(), + EmbeddedMariaInstance.SERVER_CONFIG_MAP.get(_server.getName()).getDataSourceConfig().getCustomProperties().get("SERVICE_IDENTIFIER") ); _schemaEvolutionManager = new FlywaySchemaEvolutionManager(config); @@ -68,18 +69,18 @@ public void testGetDatabaseName() throws NoSuchMethodException, InvocationTarget // Case 1: invalid database connection URL - whole string String databaseUrl = "asdfqwerty"; - SchemaEvolutionManager.Config config1 = new SchemaEvolutionManager.Config(databaseUrl, "pw", "user"); + SchemaEvolutionManager.Config config1 = new SchemaEvolutionManager.Config(databaseUrl, "pw", "user", "case1"); assertThrows(IllegalArgumentException.class, () -> { try { method.invoke(_schemaEvolutionManager, config1); } catch (InvocationTargetException e) { - throw e.getCause(); + throw e.getCause(); } }); // Case 2: invalid database connection URL - missing database name databaseUrl = "jdbc:mysql://example.linkedin.com:1234"; - SchemaEvolutionManager.Config config2 = new SchemaEvolutionManager.Config(databaseUrl, "pw", "user"); + SchemaEvolutionManager.Config config2 = new SchemaEvolutionManager.Config(databaseUrl, "pw", "user", "case2"); assertThrows(IllegalArgumentException.class, () -> { try { method.invoke(_schemaEvolutionManager, config2); @@ -90,12 +91,12 @@ public void testGetDatabaseName() throws NoSuchMethodException, InvocationTarget // Case 3: valid database connection URL with no options databaseUrl = "jdbc:mysql://example.linkedin.com:1234/my_first_db"; - SchemaEvolutionManager.Config config3 = new SchemaEvolutionManager.Config(databaseUrl, "pw", "user"); + SchemaEvolutionManager.Config config3 = new SchemaEvolutionManager.Config(databaseUrl, "pw", "user", "case3"); assertEquals(method.invoke(_schemaEvolutionManager, config3), "my_first_db"); // Case 4: valid database connection URL with options databaseUrl = "jdbc:mysql://example.linkedin.com:1234/my_first_db?autoReconnect=true&useSSL=false"; - SchemaEvolutionManager.Config config4 = new SchemaEvolutionManager.Config(databaseUrl, "pw", "user"); + SchemaEvolutionManager.Config config4 = new SchemaEvolutionManager.Config(databaseUrl, "pw", "user", "case4"); assertEquals(method.invoke(_schemaEvolutionManager, config4), "my_first_db"); } } diff --git a/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/FlywaySchemaEvolutionManagerTestWithoutServiceIdentifier.java b/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/FlywaySchemaEvolutionManagerTestWithoutServiceIdentifier.java new file mode 100644 index 000000000..76fa119f1 --- /dev/null +++ b/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/FlywaySchemaEvolutionManagerTestWithoutServiceIdentifier.java @@ -0,0 +1,65 @@ +package com.linkedin.metadata.dao; + +import com.google.common.io.Resources; +import com.linkedin.metadata.dao.utils.EmbeddedMariaInstance; +import io.ebean.Ebean; +import io.ebean.EbeanServer; +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import static org.testng.Assert.*; + + +/** + * This test is to ensure without service identifier, the schema evolution manager can bring the schema up-to-date from + * the default config file. + */ +public class FlywaySchemaEvolutionManagerTestWithoutServiceIdentifier { + private FlywaySchemaEvolutionManager _schemaEvolutionManager; + private EbeanServer _server; + + @BeforeClass + public void init() throws IOException { + //reuse the db instance + //but with a different server config + _server = EmbeddedMariaInstance.getServerWithoutServiceIdentifier(FlywaySchemaEvolutionManagerTest.class.getSimpleName()); + _server.execute(Ebean.createSqlUpdate( + Resources.toString(Resources.getResource("schema-evolution-create-all.sql"), StandardCharsets.UTF_8))); + SchemaEvolutionManager.Config config = new SchemaEvolutionManager.Config( + EmbeddedMariaInstance.SERVER_CONFIG_MAP.get(_server.getName()).getDataSourceConfig().getUrl(), + EmbeddedMariaInstance.SERVER_CONFIG_MAP.get(_server.getName()).getDataSourceConfig().getPassword(), + EmbeddedMariaInstance.SERVER_CONFIG_MAP.get(_server.getName()).getDataSourceConfig().getUsername(), + null + ); + + _schemaEvolutionManager = new FlywaySchemaEvolutionManager(config); + } + + @Test + public void testSchemaUpToDate() { + _schemaEvolutionManager.clean(); + + // make sure table did not exists + assertFalse(checkTableExists("metadata_entity_foobaz")); + assertFalse(checkTableExists("my_another_version_table")); + + // Execute the evolution scripts to bring schema up-to-date. + _schemaEvolutionManager.ensureSchemaUpToDate(); + + // V1__create_foobaz_entity_table.sql create metadata_entity_foobaz table. + assertTrue(checkTableExists("metadata_entity_foobaz")); + + // Make sure version table is created. + assertTrue(checkTableExists("my_another_version_table")); + _server.createSqlUpdate("DROP TABLE my_another_version_table").execute(); + } + + private boolean checkTableExists(String tableName) { + String checkTableExistsSql = String.format("SELECT count(*) as count FROM information_schema.TABLES WHERE TABLE_SCHEMA = '%s' AND" + + " TABLE_NAME = '%s'", _server.getName(), tableName); + return _server.createSqlQuery(checkTableExistsSql).findOne().getInteger("count") == 1; + } +} diff --git a/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/utils/EmbeddedMariaInstance.java b/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/utils/EmbeddedMariaInstance.java index c37528272..9ca46f49f 100644 --- a/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/utils/EmbeddedMariaInstance.java +++ b/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/utils/EmbeddedMariaInstance.java @@ -13,6 +13,8 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.Comparator; +import java.util.HashMap; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -39,6 +41,33 @@ public static synchronized EbeanServer getServer(String dbSchema) { throw new RuntimeException(e); } + DataSourceConfig dataSourceConfig = new DataSourceConfig(); + dataSourceConfig.setUsername(DB_USER); + dataSourceConfig.setPassword(DB_PASS); + dataSourceConfig.setUrl(String.format("jdbc:mysql://localhost:%s/%s?allowMultiQueries=true", PORT, dbSchema)); + dataSourceConfig.setDriver("com.mysql.cj.jdbc.Driver"); + Map customProperties = new HashMap<>(); + customProperties.put("SERVICE_IDENTIFIER", "test"); + dataSourceConfig.setCustomProperties(customProperties); + + ServerConfig serverConfig = new ServerConfig(); + serverConfig.setName(dbSchema); + serverConfig.setDataSourceConfig(dataSourceConfig); + serverConfig.setDdlGenerate(false); + serverConfig.setDdlRun(false); + SERVER_CONFIG_MAP.put(serverConfig.getName(), serverConfig); + return EbeanServerFactory.create(serverConfig); + } + + public static synchronized EbeanServer getServerWithoutServiceIdentifier(String dbSchema) { + initDB(); // initDB is idempotent + + try { + db.createDB(dbSchema); + } catch (ManagedProcessException e) { + throw new RuntimeException(e); + } + DataSourceConfig dataSourceConfig = new DataSourceConfig(); dataSourceConfig.setUsername(DB_USER); dataSourceConfig.setPassword(DB_PASS); diff --git a/dao-impl/ebean-dao/src/test/resources/FlywaySchemaEvolutionManagerTest.conf b/dao-impl/ebean-dao/src/test/resources/FlywaySchemaEvolutionManagerTest.conf index f7f6edd4b..ace37ceb0 100644 --- a/dao-impl/ebean-dao/src/test/resources/FlywaySchemaEvolutionManagerTest.conf +++ b/dao-impl/ebean-dao/src/test/resources/FlywaySchemaEvolutionManagerTest.conf @@ -1,3 +1,3 @@ -version_table=my_version_table -script_directory=db/foobar +version_table=my_another_version_table +script_directory=db/foobaz disable_clean=False \ No newline at end of file diff --git a/dao-impl/ebean-dao/src/test/resources/db/foobaz/V1__create_foobaz_entity_table.sql b/dao-impl/ebean-dao/src/test/resources/db/foobaz/V1__create_foobaz_entity_table.sql new file mode 100644 index 000000000..00fea6dd1 --- /dev/null +++ b/dao-impl/ebean-dao/src/test/resources/db/foobaz/V1__create_foobaz_entity_table.sql @@ -0,0 +1,7 @@ +CREATE TABLE IF NOT EXISTS metadata_entity_foobaz ( + urn VARCHAR(100) NOT NULL, + lastmodifiedon DATETIME(6) NOT NULL, + lastmodifiedby VARCHAR(255) NOT NULL, + createdfor VARCHAR(255), + CONSTRAINT pk_metadata_aspect PRIMARY KEY (urn) +); \ No newline at end of file diff --git a/dao-impl/ebean-dao/src/test/resources/schema-evolution-create-all.sql b/dao-impl/ebean-dao/src/test/resources/schema-evolution-create-all.sql index 2d25f02f9..aefbba8d5 100644 --- a/dao-impl/ebean-dao/src/test/resources/schema-evolution-create-all.sql +++ b/dao-impl/ebean-dao/src/test/resources/schema-evolution-create-all.sql @@ -1,3 +1,4 @@ +DROP TABLE IF EXISTS metadata_entity_foobaz; DROP TABLE IF EXISTS metadata_entity_foo; DROP TABLE IF EXISTS metadata_entity_bar; DROP TABLE IF EXISTS metadata_aspect; diff --git a/dao-impl/ebean-dao/src/test/resources/EBeanDAOUtilsTest.conf b/dao-impl/ebean-dao/src/test/resources/test-EBeanDAOUtilsTest.conf similarity index 100% rename from dao-impl/ebean-dao/src/test/resources/EBeanDAOUtilsTest.conf rename to dao-impl/ebean-dao/src/test/resources/test-EBeanDAOUtilsTest.conf diff --git a/dao-impl/ebean-dao/src/test/resources/EbeanLocalDAOTest.conf b/dao-impl/ebean-dao/src/test/resources/test-EbeanLocalAccessTest.conf similarity index 100% rename from dao-impl/ebean-dao/src/test/resources/EbeanLocalDAOTest.conf rename to dao-impl/ebean-dao/src/test/resources/test-EbeanLocalAccessTest.conf diff --git a/dao-impl/ebean-dao/src/test/resources/EbeanLocalRelationshipQueryDAOTest.conf b/dao-impl/ebean-dao/src/test/resources/test-EbeanLocalDAOTest.conf similarity index 100% rename from dao-impl/ebean-dao/src/test/resources/EbeanLocalRelationshipQueryDAOTest.conf rename to dao-impl/ebean-dao/src/test/resources/test-EbeanLocalDAOTest.conf diff --git a/dao-impl/ebean-dao/src/test/resources/EbeanLocalRelationshipWriterDAOTest.conf b/dao-impl/ebean-dao/src/test/resources/test-EbeanLocalRelationshipQueryDAOTest.conf similarity index 100% rename from dao-impl/ebean-dao/src/test/resources/EbeanLocalRelationshipWriterDAOTest.conf rename to dao-impl/ebean-dao/src/test/resources/test-EbeanLocalRelationshipQueryDAOTest.conf diff --git a/dao-impl/ebean-dao/src/test/resources/test-EbeanLocalRelationshipWriterDAOTest.conf b/dao-impl/ebean-dao/src/test/resources/test-EbeanLocalRelationshipWriterDAOTest.conf new file mode 100644 index 000000000..f7f6edd4b --- /dev/null +++ b/dao-impl/ebean-dao/src/test/resources/test-EbeanLocalRelationshipWriterDAOTest.conf @@ -0,0 +1,3 @@ +version_table=my_version_table +script_directory=db/foobar +disable_clean=False \ No newline at end of file diff --git a/dao-impl/ebean-dao/src/test/resources/test-FlywaySchemaEvolutionManagerTest.conf b/dao-impl/ebean-dao/src/test/resources/test-FlywaySchemaEvolutionManagerTest.conf new file mode 100644 index 000000000..f7f6edd4b --- /dev/null +++ b/dao-impl/ebean-dao/src/test/resources/test-FlywaySchemaEvolutionManagerTest.conf @@ -0,0 +1,3 @@ +version_table=my_version_table +script_directory=db/foobar +disable_clean=False \ No newline at end of file