From 3f21e7c235bfbd52bd1a85bd74c8b1fb7f6566a6 Mon Sep 17 00:00:00 2001 From: Yang Yang Date: Thu, 26 Sep 2024 14:32:44 -0700 Subject: [PATCH] Read column from aspect-alias instead of column annotation - part 1 (#433) * Read column from aspect-alias instead of column annotation * address comments --------- Co-authored-by: Yang Yang --- .../linkedin/metadata/query/AspectField.pdl | 8 ++++ .../metadata/dao/EbeanLocalAccess.java | 27 ++++++------- .../dao/EbeanLocalRelationshipQueryDAO.java | 3 +- .../metadata/dao/utils/EBeanDAOUtils.java | 11 ++++- .../dao/utils/SQLIndexFilterUtils.java | 14 ++++--- .../metadata/dao/utils/SQLSchemaUtils.java | 38 +++++++++++++----- .../metadata/dao/utils/SQLStatementUtils.java | 40 +++++++++++-------- .../metadata/dao/EbeanLocalDAOTest.java | 10 ++--- .../dao/utils/SQLIndexFilterUtilsTest.java | 23 ++++++----- .../dao/utils/SQLSchemaUtilsTest.java | 11 +++-- .../dao/utils/SQLStatementUtilsTest.java | 13 +++--- .../pegasus/com/linkedin/testing/BarAsset | 19 +++++++++ 12 files changed, 145 insertions(+), 72 deletions(-) create mode 100644 testing/test-models/src/main/pegasus/com/linkedin/testing/BarAsset diff --git a/dao-api/src/main/pegasus/com/linkedin/metadata/query/AspectField.pdl b/dao-api/src/main/pegasus/com/linkedin/metadata/query/AspectField.pdl index 67c531eb2..1fbe60061 100644 --- a/dao-api/src/main/pegasus/com/linkedin/metadata/query/AspectField.pdl +++ b/dao-api/src/main/pegasus/com/linkedin/metadata/query/AspectField.pdl @@ -2,6 +2,14 @@ namespace com.linkedin.metadata.query record AspectField { + /** + * Asset type (entityType) this aspect belongs to. When running a query filtered by an aspect field (aspect alias), + * depends on which asset this aspect belongs to, the field (aspect alias) can be different. For more context, check + * the decision on aspect alias: go/mg/aspect-alias-decision. The underlying logic requires asset type information + * to query on the right target. e.g. DB column name, which is from the aspect-alias defined on Asset. + */ + asset: string + /** * FQCN of the aspect class e.g. com.linkedin.common.Status */ diff --git a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalAccess.java b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalAccess.java index 4f3aa1e70..0092d8633 100644 --- a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalAccess.java +++ b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalAccess.java @@ -176,7 +176,7 @@ public List batchGetUnion( final Urn entityUrn = aspectKeys.get(index).getUrn(); final Class aspectClass = (Class) aspectKeys.get(index).getAspectClass(); if (checkColumnExists(isTestMode ? getTestTableName(entityUrn) : getTableName(entityUrn), - getAspectColumnName(aspectClass))) { + getAspectColumnName(entityUrn.getEntityType(), aspectClass))) { keysToQueryMap.computeIfAbsent(aspectClass, unused -> new HashSet<>()).add(entityUrn); } } @@ -262,7 +262,7 @@ public ListResult list(@Nonnull Class ListResult list(@Nonnull Class ListResult list(@Nonnull Class aspectClass, int start, int pageSize) { - final String tableName = SQLSchemaUtils.getTableName(_entityType); - final String listAspectSql = SQLStatementUtils.createListAspectWithPaginationSql(aspectClass, tableName, false, start, pageSize); + + final String listAspectSql = SQLStatementUtils.createListAspectWithPaginationSql(aspectClass, _entityType, false, start, pageSize); final SqlQuery sqlQuery = _server.createSqlQuery(listAspectSql); final List sqlRows = sqlQuery.findList(); if (sqlRows.isEmpty()) { @@ -298,7 +298,7 @@ public ListResult list(@Nonnull Class ListResult list(@Nonnull Class countAggregate(@Nullable IndexFilter indexFilter, @Nonnull IndexGroupByCriterion indexGroupByCriterion) { final String tableName = SQLSchemaUtils.getTableName(_entityType); - final String groupByColumn = getGeneratedColumnName(indexGroupByCriterion.getAspect(), indexGroupByCriterion.getPath(), _nonDollarVirtualColumnsEnabled); + final String groupByColumn = + getGeneratedColumnName(_entityType, indexGroupByCriterion.getAspect(), indexGroupByCriterion.getPath(), + _nonDollarVirtualColumnsEnabled); // first, check for existence of the column we want to GROUP BY if (!checkColumnExists(tableName, groupByColumn)) { // if we are trying to GROUP BY the results on a column that does not exist, just return an empty map @@ -317,7 +319,7 @@ public Map countAggregate(@Nullable IndexFilter indexFilter, } // now run the actual GROUP BY query - final String groupBySql = SQLStatementUtils.createGroupBySql(tableName, indexFilter, indexGroupByCriterion, _nonDollarVirtualColumnsEnabled); + final String groupBySql = SQLStatementUtils.createGroupBySql(_entityType, indexFilter, indexGroupByCriterion, _nonDollarVirtualColumnsEnabled); final SqlQuery sqlQuery = _server.createSqlQuery(groupBySql); final List sqlRows = sqlQuery.findList(); Map resultMap = new HashMap<>(); @@ -343,12 +345,10 @@ public Map countAggregate(@Nullable IndexFilter indexFilter, */ private SqlQuery createFilterSqlQuery(@Nullable IndexFilter indexFilter, @Nullable IndexSortCriterion indexSortCriterion, int offset, int pageSize) { - - final String tableName = SQLSchemaUtils.getTableName(_entityType); StringBuilder filterSql = new StringBuilder(); - filterSql.append(SQLStatementUtils.createFilterSql(tableName, indexFilter, true, _nonDollarVirtualColumnsEnabled)); + filterSql.append(SQLStatementUtils.createFilterSql(_entityType, indexFilter, true, _nonDollarVirtualColumnsEnabled)); filterSql.append("\n"); - filterSql.append(parseSortCriteria(indexSortCriterion, _nonDollarVirtualColumnsEnabled)); + filterSql.append(parseSortCriteria(_entityType, indexSortCriterion, _nonDollarVirtualColumnsEnabled)); filterSql.append(String.format(" LIMIT %d", Math.max(pageSize, 0))); filterSql.append(String.format(" OFFSET %d", Math.max(offset, 0))); return _server.createSqlQuery(filterSql.toString()); @@ -360,8 +360,7 @@ private SqlQuery createFilterSqlQuery(@Nullable IndexFilter indexFilter, private SqlQuery createFilterSqlQuery(@Nullable IndexFilter indexFilter, @Nullable IndexSortCriterion indexSortCriterion, @Nullable URN lastUrn, int pageSize) { StringBuilder filterSql = new StringBuilder(); - final String tableName = SQLSchemaUtils.getTableName(_entityType); - filterSql.append(SQLStatementUtils.createFilterSql(tableName, indexFilter, false, _nonDollarVirtualColumnsEnabled)); + filterSql.append(SQLStatementUtils.createFilterSql(_entityType, indexFilter, false, _nonDollarVirtualColumnsEnabled)); if (lastUrn != null) { // because createFilterSql will only include a WHERE clause if there are non-urn filters, we need to make sure @@ -378,7 +377,7 @@ private SqlQuery createFilterSqlQuery(@Nullable IndexFilter indexFilter, } filterSql.append("\n"); - filterSql.append(parseSortCriteria(indexSortCriterion, _nonDollarVirtualColumnsEnabled)); + filterSql.append(parseSortCriteria(_entityType, indexSortCriterion, _nonDollarVirtualColumnsEnabled)); filterSql.append(String.format(" LIMIT %d", Math.max(pageSize, 0))); return _server.createSqlQuery(filterSql.toString()); } diff --git a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalRelationshipQueryDAO.java b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalRelationshipQueryDAO.java index 80a909b23..b60d57bf6 100644 --- a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalRelationshipQueryDAO.java +++ b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalRelationshipQueryDAO.java @@ -371,7 +371,8 @@ private SNAPSHOT constructSnapshot(@Nonnull fi final List aspects = new ArrayList<>(); for (String aspectCanonicalName : ModelUtils.getAspectClassNames(unionTemplateClass)) { - String colName = SQLSchemaUtils.getAspectColumnName(aspectCanonicalName); + String colName = + SQLSchemaUtils.getAspectColumnName(ModelUtils.getUrnTypeFromSnapshot(snapshotClass), aspectCanonicalName); String auditedAspectStr = sqlRow.getString(colName); if (auditedAspectStr != null) { diff --git a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/utils/EBeanDAOUtils.java b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/utils/EBeanDAOUtils.java index 4b57f8611..951f56461 100644 --- a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/utils/EBeanDAOUtils.java +++ b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/utils/EBeanDAOUtils.java @@ -1,5 +1,6 @@ package com.linkedin.metadata.dao.utils; +import com.linkedin.common.urn.Urn; import com.linkedin.data.schema.RecordDataSchema; import com.linkedin.data.template.DataTemplateUtil; import com.linkedin.data.template.RecordTemplate; @@ -18,6 +19,7 @@ import io.ebean.SqlRow; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.net.URISyntaxException; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -235,11 +237,16 @@ public static List readSqlR */ private static EbeanMetadataAspect readSqlRow(SqlRow sqlRow, Class aspectClass) { - final String columnName = SQLSchemaUtils.getAspectColumnName(aspectClass); + final EbeanMetadataAspect ebeanMetadataAspect = new EbeanMetadataAspect(); final String urn = sqlRow.getString("urn"); EbeanMetadataAspect.PrimaryKey primaryKey; - + final String columnName; + try { + columnName = SQLSchemaUtils.getAspectColumnName(Urn.createFromString(urn).getEntityType(), aspectClass); + } catch (URISyntaxException e) { + throw new RuntimeException("Invalid urn format: " + urn, e); + } if (isSoftDeletedAspect(sqlRow, columnName)) { primaryKey = new EbeanMetadataAspect.PrimaryKey(urn, aspectClass.getCanonicalName(), LATEST_VERSION); ebeanMetadataAspect.setCreatedBy(sqlRow.getString("lastmodifiedby")); diff --git a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/utils/SQLIndexFilterUtils.java b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/utils/SQLIndexFilterUtils.java index 7dc864783..d9e710dbe 100644 --- a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/utils/SQLIndexFilterUtils.java +++ b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/utils/SQLIndexFilterUtils.java @@ -65,17 +65,19 @@ private static String parseIndexValue(@Nullable IndexValue indexValue) { /** * Parse {@link IndexSortCriterion} into SQL syntax. + * @param entityType entity type from the Urn * @param indexSortCriterion filter sorting criterion * @param nonDollarVirtualColumnsEnabled true if virtual column does not contain $, false otherwise * @return SQL statement of sorting, e.g. ORDER BY ... DESC ..etc. */ - public static String parseSortCriteria(@Nullable IndexSortCriterion indexSortCriterion, boolean nonDollarVirtualColumnsEnabled) { + public static String parseSortCriteria(@Nonnull String entityType, @Nullable IndexSortCriterion indexSortCriterion, + boolean nonDollarVirtualColumnsEnabled) { if (indexSortCriterion == null) { // Default to order by urn if user does not provide sort criterion. return "ORDER BY URN"; } final String indexColumn = - SQLSchemaUtils.getGeneratedColumnName(indexSortCriterion.getAspect(), indexSortCriterion.getPath(), + SQLSchemaUtils.getGeneratedColumnName(entityType, indexSortCriterion.getAspect(), indexSortCriterion.getPath(), nonDollarVirtualColumnsEnabled); if (!indexSortCriterion.hasOrder()) { @@ -87,12 +89,12 @@ public static String parseSortCriteria(@Nullable IndexSortCriterion indexSortCri /** * Parse {@link IndexFilter} into MySQL syntax. - * + * @param entityType entity type from the Urn * @param indexFilter index filter * @param nonDollarVirtualColumnsEnabled whether to enable non-dollar virtual columns * @return translated SQL condition expression, e.g. WHERE ... */ - public static String parseIndexFilter(@Nullable IndexFilter indexFilter, boolean nonDollarVirtualColumnsEnabled) { + public static String parseIndexFilter(@Nonnull String entityType, @Nullable IndexFilter indexFilter, boolean nonDollarVirtualColumnsEnabled) { List sqlFilters = new ArrayList<>(); if (indexFilter == null || !indexFilter.hasCriteria()) { @@ -103,7 +105,7 @@ public static String parseIndexFilter(@Nullable IndexFilter indexFilter, boolean final String aspect = indexCriterion.getAspect(); if (!isUrn(aspect)) { // if aspect is not urn, then check aspect is not soft deleted and is not null - final String aspectColumn = getAspectColumnName(indexCriterion.getAspect()); + final String aspectColumn = getAspectColumnName(entityType, indexCriterion.getAspect()); sqlFilters.add(aspectColumn + " IS NOT NULL"); sqlFilters.add(String.format(SOFT_DELETED_CHECK, aspectColumn)); } @@ -112,7 +114,7 @@ public static String parseIndexFilter(@Nullable IndexFilter indexFilter, boolean if (pathParams != null) { validateConditionAndValue(indexCriterion); final Condition condition = pathParams.getCondition(); - final String indexColumn = getGeneratedColumnName(aspect, pathParams.getPath(), nonDollarVirtualColumnsEnabled); + final String indexColumn = getGeneratedColumnName(entityType, aspect, pathParams.getPath(), nonDollarVirtualColumnsEnabled); sqlFilters.add(parseSqlFilter(indexColumn, condition, pathParams.getValue())); } } diff --git a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/utils/SQLSchemaUtils.java b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/utils/SQLSchemaUtils.java index bdc2da3be..426e4d238 100644 --- a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/utils/SQLSchemaUtils.java +++ b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/utils/SQLSchemaUtils.java @@ -8,13 +8,16 @@ import com.linkedin.metadata.dao.exception.MissingAnnotationException; import java.util.Map; import javax.annotation.Nonnull; +import lombok.extern.slf4j.Slf4j; + +import static com.linkedin.metadata.dao.utils.SQLStatementUtils.*; /** * Generate schema related SQL script, such as normalized table / column names ..etc */ +@Slf4j public class SQLSchemaUtils { - private static final String GMA = "gma"; public static final String ENTITY_TABLE_PREFIX = "metadata_entity_"; public static final String RELATIONSHIP_TABLE_PREFIX = "metadata_relationship_"; @@ -24,6 +27,17 @@ public class SQLSchemaUtils { private static final int MYSQL_MAX_COLUMN_NAME_LENGTH = 64 - ASPECT_PREFIX.length(); + /** + * This field is used when asset field in {@link com.linkedin.metadata.query.AspectField} is not provided in the + * legacy implementation. When this is field is set, the getColumnNameFromAnnotation() will retrieve + * "column" from the "column" annotation from the Aspect. However, the going forward way is to retrieve "column" + * information from aspect alias defined in the asset. + * + *

For more context, see: Decision - Using Proto Field Name as Aspect URI + * go/mg/aspect-alias-decision + */ + protected static final String UNKNOWN_ASSET = "UNKNOWN_ASSET"; + private SQLSchemaUtils() { } @@ -113,8 +127,8 @@ public static String getTestRelationshipTa * Get column name from aspect class canonical name. */ @Nonnull - public static String getAspectColumnName(@Nonnull final String aspectCanonicalName) { - return ASPECT_PREFIX + getColumnNameFromAnnotation(aspectCanonicalName); + public static String getAspectColumnName(@Nonnull final String entityType, @Nonnull final String aspectCanonicalName) { + return ASPECT_PREFIX + getColumnNameFromAnnotation(entityType, aspectCanonicalName); } /** @@ -123,21 +137,25 @@ public static String getAspectColumnName(@Nonnull final String aspectCanonicalNa * @param aspect that extends {@link RecordTemplate} * @return aspect column name */ - public static String getAspectColumnName(@Nonnull Class aspectClass) { - return getAspectColumnName(aspectClass.getCanonicalName()); + public static String getAspectColumnName(@Nonnull final String entityType, + @Nonnull Class aspectClass) { + return getAspectColumnName(entityType, aspectClass.getCanonicalName()); } /** * Get generated column name from aspect and path. */ @Nonnull - public static String getGeneratedColumnName(@Nonnull String aspect, @Nonnull String path, boolean nonDollarVirtualColumnsEnabled) { + public static String getGeneratedColumnName(@Nonnull String assetType, @Nonnull String aspect, @Nonnull String path, + boolean nonDollarVirtualColumnsEnabled) { char delimiter = nonDollarVirtualColumnsEnabled ? '0' : '$'; if (isUrn(aspect)) { return INDEX_PREFIX + "urn" + processPath(path, delimiter); } - - return INDEX_PREFIX + getColumnNameFromAnnotation(aspect) + processPath(path, delimiter); + if (UNKNOWN_ASSET.equals(assetType)) { + log.warn("query with unknown asset type. aspect = {}, path ={}, delimiter = {}", aspect, path, delimiter); + } + return INDEX_PREFIX + getColumnNameFromAnnotation(assetType, aspect) + processPath(path, delimiter); } /** @@ -165,11 +183,13 @@ public static String processPath(@Nonnull String path, char delimiter) { /** * Get Column name from aspect canonical name. * + * @param assetType entity type from Urn definition. * @param aspectCanonicalName aspect name in canonical form. * @return aspect column name */ @Nonnull - private static String getColumnNameFromAnnotation(@Nonnull final String aspectCanonicalName) { + private static String getColumnNameFromAnnotation(@Nonnull final String assetType, @Nonnull final String aspectCanonicalName) { + // TODO(yanyang) implement a map of (assetType, aspectClass) --> aspect_alias (column) try { final RecordDataSchema schema = (RecordDataSchema) DataTemplateUtil.getSchema(ClassUtils.loadClass(aspectCanonicalName)); final Map properties = schema.getProperties(); diff --git a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/utils/SQLStatementUtils.java b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/utils/SQLStatementUtils.java index 74410111f..0fc2ee6d1 100644 --- a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/utils/SQLStatementUtils.java +++ b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/utils/SQLStatementUtils.java @@ -160,10 +160,11 @@ public static String createAspectReadSql(@Nonnul if (urns.size() == 0) { throw new IllegalArgumentException("Need at least 1 urn to query."); } - final String columnName = getAspectColumnName(aspectClass); + StringBuilder stringBuilder = new StringBuilder(); List selectStatements = urns.stream().map(urn -> { final String tableName = isTestMode ? getTestTableName(urn) : getTableName(urn); + final String columnName = getAspectColumnName(urn.getEntityType(), aspectClass); final String sqlTemplate = includeSoftDeleted ? SQL_READ_ASPECT_WITH_SOFT_DELETED_TEMPLATE : SQL_READ_ASPECT_TEMPLATE; return String.format(sqlTemplate, columnName, tableName, escapeReservedCharInUrn(urn.toString()), columnName); @@ -182,7 +183,7 @@ public static String createAspectReadSql(@Nonnul */ public static String createListAspectByUrnSql(@Nonnull Class aspectClass, @Nonnull Urn urn, boolean includeSoftDeleted) { - final String columnName = getAspectColumnName(aspectClass); + final String columnName = getAspectColumnName(urn.getEntityType(), aspectClass); final String tableName = getTableName(urn); if (includeSoftDeleted) { return String.format(SQL_LIST_ASPECT_BY_URN_WITH_SOFT_DELETED_TEMPLATE, columnName, tableName, @@ -196,7 +197,7 @@ public static String createListAspectByUrnSql(@N /** * List all the aspects for a given entity type and aspect type. * @param aspectClass aspect type - * @param tableName table name + * @param entityType entity name from Urn * @param includeSoftDeleted whether to include soft deleted aspects * @param start pagination offset * @param pageSize page size @@ -204,8 +205,9 @@ public static String createListAspectByUrnSql(@N * @return a SQL to run listing aspect query with pagination. */ public static String createListAspectWithPaginationSql(@Nonnull Class aspectClass, - String tableName, boolean includeSoftDeleted, int start, int pageSize) { - final String columnName = getAspectColumnName(aspectClass); + String entityType, boolean includeSoftDeleted, int start, int pageSize) { + final String tableName = SQLSchemaUtils.getTableName(entityType); + final String columnName = getAspectColumnName(entityType, aspectClass); if (includeSoftDeleted) { return String.format(SQL_LIST_ASPECT_WITH_PAGINATION_WITH_SOFT_DELETED_TEMPLATE, columnName, tableName, columnName, tableName, columnName, pageSize, start); @@ -226,7 +228,7 @@ public static String createListAspectWithPaginat public static String createAspectUpsertSql(@Nonnull Urn urn, @Nonnull Class aspectClass, boolean urnExtraction, boolean isTestMode) { final String tableName = isTestMode ? getTestTableName(urn) : getTableName(urn); - final String columnName = getAspectColumnName(aspectClass); + final String columnName = getAspectColumnName(urn.getEntityType(), aspectClass); return String.format(urnExtraction ? SQL_UPSERT_ASPECT_WITH_URN_TEMPLATE : SQL_UPSERT_ASPECT_TEMPLATE, tableName, columnName, columnName); } @@ -242,21 +244,22 @@ public static String createAspectUpsertSql(@Nonn public static String createAspectUpdateWithOptimisticLockSql(@Nonnull Urn urn, @Nonnull Class aspectClass, boolean urnExtraction, boolean isTestMode) { final String tableName = isTestMode ? getTestTableName(urn) : getTableName(urn); - final String columnName = getAspectColumnName(aspectClass); + final String columnName = getAspectColumnName(urn.getEntityType(), aspectClass); return String.format(urnExtraction ? SQL_UPDATE_ASPECT_WITH_URN_TEMPLATE : SQL_UPDATE_ASPECT_TEMPLATE, tableName, columnName, columnName, columnName); } /** * Create filter SQL statement. - * @param tableName table name + * @param entityType entity type from urn * @param indexFilter index filter * @param hasTotalCount whether to calculate total count in SQL. * @param nonDollarVirtualColumnsEnabled true if virtual column does not contain $, false otherwise * @return translated SQL where statement */ - public static String createFilterSql(String tableName, @Nullable IndexFilter indexFilter, boolean hasTotalCount, boolean nonDollarVirtualColumnsEnabled) { - String whereClause = parseIndexFilter(indexFilter, nonDollarVirtualColumnsEnabled); + public static String createFilterSql(String entityType, @Nullable IndexFilter indexFilter, boolean hasTotalCount, boolean nonDollarVirtualColumnsEnabled) { + final String tableName = SQLSchemaUtils.getTableName(entityType); + String whereClause = parseIndexFilter(entityType, indexFilter, nonDollarVirtualColumnsEnabled); String totalCountSql = String.format("SELECT COUNT(urn) FROM %s %s", tableName, whereClause); StringBuilder sb = new StringBuilder(); @@ -273,19 +276,22 @@ public static String createFilterSql(String tableName, @Nullable IndexFilter ind /** * Create index group by SQL statement. - * @param tableName table name + * @param entityType entity type * @param indexFilter index filter * @param indexGroupByCriterion group by * @param nonDollarVirtualColumnsEnabled true if virtual column does not contain $, false otherwise * @return translated group by SQL */ - public static String createGroupBySql(String tableName, @Nullable IndexFilter indexFilter, + public static String createGroupBySql(String entityType, @Nullable IndexFilter indexFilter, @Nonnull IndexGroupByCriterion indexGroupByCriterion, boolean nonDollarVirtualColumnsEnabled) { - final String columnName = getGeneratedColumnName(indexGroupByCriterion.getAspect(), indexGroupByCriterion.getPath(), nonDollarVirtualColumnsEnabled); + final String tableName = SQLSchemaUtils.getTableName(entityType); + final String columnName = + getGeneratedColumnName(entityType, indexGroupByCriterion.getAspect(), indexGroupByCriterion.getPath(), + nonDollarVirtualColumnsEnabled); StringBuilder sb = new StringBuilder(); sb.append(String.format(INDEX_GROUP_BY_CRITERION, columnName, tableName)); sb.append("\n"); - sb.append(parseIndexFilter(indexFilter, nonDollarVirtualColumnsEnabled)); + sb.append(parseIndexFilter(entityType, indexFilter, nonDollarVirtualColumnsEnabled)); sb.append("\nGROUP BY "); sb.append(columnName); return sb.toString(); @@ -305,7 +311,7 @@ public static String getAllColumnForTable(String tableName) { public static String createAspectBrowseSql(String entityType, Class aspectClass, int offset, int pageSize) { final String tableName = getTableName(entityType); - final String columnName = getAspectColumnName(aspectClass); + final String columnName = getAspectColumnName(entityType, aspectClass); return String.format(SQL_BROWSE_ASPECT_TEMPLATE, columnName, tableName, tableName, columnName, Math.max(pageSize, 0), Math.max(offset, 0)); } @@ -469,7 +475,9 @@ private static String parseLocalRelationshipField( } if (field.isAspectField()) { - return tablePrefix + SQLSchemaUtils.getGeneratedColumnName(field.getAspectField().getAspect(), + // entity type from Urn definition. + String asset = field.getAspectField().hasAsset() ? field.getAspectField().getAsset() : UNKNOWN_ASSET; + return tablePrefix + SQLSchemaUtils.getGeneratedColumnName(asset, field.getAspectField().getAspect(), field.getAspectField().getPath(), nonDollarVirtualColumnsEnabled); } diff --git a/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/EbeanLocalDAOTest.java b/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/EbeanLocalDAOTest.java index de68db81d..426c118e9 100644 --- a/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/EbeanLocalDAOTest.java +++ b/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/EbeanLocalDAOTest.java @@ -3277,7 +3277,7 @@ private void addMetadataEntityTable(Urn urn, Cla return; } String aspectName = aspectClass.getCanonicalName(); - String columnName = SQLSchemaUtils.getAspectColumnName(aspectName); + String columnName = SQLSchemaUtils.getAspectColumnName(urn.getEntityType(), aspectName); String template = "insert into metadata_entity_%s (urn, %s, lastmodifiedon, lastmodifiedby, createdfor) value" + "('%s', '%s', '%s', '%s', '%s') ON DUPLICATE KEY UPDATE %s = '%s';"; String query = String.format(template, urn.getEntityType(), columnName, urn, createAuditedAspect(metadata, aspectClass, createdOn, createdBy, createdFor), @@ -3334,8 +3334,8 @@ private void addIndex(Urn urn, String aspectName, String pathName, Object val) { urn:2| | "actor" | "{..."longval":5...} | 5 | */ - String aspectColumnName = isUrn(aspectName) ? null : SQLSchemaUtils.getAspectColumnName(aspectName); // e.g. a_aspectfoo; - String fullIndexColumnName = SQLSchemaUtils.getGeneratedColumnName(aspectName, pathName, + String aspectColumnName = isUrn(aspectName) ? null : SQLSchemaUtils.getAspectColumnName(urn.getEntityType(), aspectName); // e.g. a_aspectfoo; + String fullIndexColumnName = SQLSchemaUtils.getGeneratedColumnName(urn.getEntityType(), aspectName, pathName, _eBeanDAOConfig.isNonDollarVirtualColumnsEnabled()); // e.g. i_aspectfoo$path1$value1 String checkColumnExistance = String.format("SELECT * FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = '%s' AND" @@ -3372,7 +3372,7 @@ private void addIndex(Urn urn, String aspectName, String pathName, Object val) { private EbeanMetadataAspect getMetadata(Urn urn, String aspectName, long version) { if (_schemaConfig == SchemaConfig.NEW_SCHEMA_ONLY && version == 0) { - String aspectColumn = getAspectColumnName(aspectName); + String aspectColumn = getAspectColumnName(urn.getEntityType(), aspectName); String template = "select urn, lastmodifiedon, lastmodifiedby, createdfor, %s from metadata_entity_%s"; String query = String.format(template, aspectColumn, urn.getEntityType()); SqlRow result = _server.createSqlQuery(query).findOne(); @@ -3397,7 +3397,7 @@ private EbeanMetadataAspect getMetadata(Urn urn, String aspectName, long version private EbeanMetadataAspect getTestMetadata(Urn urn, String aspectName, long version) { if (_schemaConfig == SchemaConfig.NEW_SCHEMA_ONLY && version == 0) { - String aspectColumn = getAspectColumnName(aspectName); + String aspectColumn = getAspectColumnName(urn.getEntityType(), aspectName); String template = "select urn, lastmodifiedon, lastmodifiedby, createdfor, %s from metadata_entity_%s_test"; String query = String.format(template, aspectColumn, urn.getEntityType()); SqlRow result = _server.createSqlQuery(query).findOne(); diff --git a/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/utils/SQLIndexFilterUtilsTest.java b/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/utils/SQLIndexFilterUtilsTest.java index 6db7a6d44..2ac239462 100644 --- a/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/utils/SQLIndexFilterUtilsTest.java +++ b/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/utils/SQLIndexFilterUtilsTest.java @@ -8,6 +8,8 @@ import com.linkedin.metadata.query.IndexValue; import com.linkedin.metadata.query.SortOrder; import com.linkedin.testing.AspectFoo; +import com.linkedin.testing.urn.FooUrn; +import java.net.URISyntaxException; import org.testng.annotations.Test; import static org.testng.Assert.*; @@ -16,23 +18,24 @@ public class SQLIndexFilterUtilsTest { @Test - public void testParseSortCriteria() { + public void testParseSortCriteria() throws URISyntaxException { + FooUrn fooUrn = new FooUrn(1); IndexSortCriterion indexSortCriterion = SQLIndexFilterUtils.createIndexSortCriterion(AspectFoo.class, "id", SortOrder.ASCENDING); assertEquals(indexSortCriterion.getOrder(), SortOrder.ASCENDING); assertEquals(indexSortCriterion.getAspect(), AspectFoo.class.getCanonicalName()); - String sql1 = SQLIndexFilterUtils.parseSortCriteria(indexSortCriterion, false); + String sql1 = SQLIndexFilterUtils.parseSortCriteria(fooUrn.getEntityType(), indexSortCriterion, false); assertEquals(sql1, "ORDER BY i_aspectfoo$id ASC"); - String sql2 = SQLIndexFilterUtils.parseSortCriteria(indexSortCriterion, true); + String sql2 = SQLIndexFilterUtils.parseSortCriteria(fooUrn.getEntityType(), indexSortCriterion, true); assertEquals(sql2, "ORDER BY i_aspectfoo0id ASC"); indexSortCriterion.setOrder(SortOrder.DESCENDING); - sql1 = SQLIndexFilterUtils.parseSortCriteria(indexSortCriterion, false); + sql1 = SQLIndexFilterUtils.parseSortCriteria(fooUrn.getEntityType(), indexSortCriterion, false); assertEquals(sql1, "ORDER BY i_aspectfoo$id DESC"); - sql2 = SQLIndexFilterUtils.parseSortCriteria(indexSortCriterion, true); + sql2 = SQLIndexFilterUtils.parseSortCriteria(fooUrn.getEntityType(), indexSortCriterion, true); assertEquals(sql2, "ORDER BY i_aspectfoo0id DESC"); } @@ -45,10 +48,12 @@ public void testParseIndexFilter() { indexCriterionArray.add(indexCriterion); indexFilter.setCriteria(indexCriterionArray); - String sql = SQLIndexFilterUtils.parseIndexFilter(indexFilter, false); - assertEquals(sql, "WHERE a_aspectfoo IS NOT NULL\nAND JSON_EXTRACT(a_aspectfoo, '$.gma_deleted') IS NULL\nAND i_aspectfoo$id < 12"); + String sql = SQLIndexFilterUtils.parseIndexFilter(FooUrn.ENTITY_TYPE, indexFilter, false); + assertEquals(sql, + "WHERE a_aspectfoo IS NOT NULL\nAND JSON_EXTRACT(a_aspectfoo, '$.gma_deleted') IS NULL\nAND i_aspectfoo$id < 12"); - sql = SQLIndexFilterUtils.parseIndexFilter(indexFilter, true); - assertEquals(sql, "WHERE a_aspectfoo IS NOT NULL\nAND JSON_EXTRACT(a_aspectfoo, '$.gma_deleted') IS NULL\nAND i_aspectfoo0id < 12"); + sql = SQLIndexFilterUtils.parseIndexFilter(FooUrn.ENTITY_TYPE, indexFilter, true); + assertEquals(sql, + "WHERE a_aspectfoo IS NOT NULL\nAND JSON_EXTRACT(a_aspectfoo, '$.gma_deleted') IS NULL\nAND i_aspectfoo0id < 12"); } } \ No newline at end of file diff --git a/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/utils/SQLSchemaUtilsTest.java b/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/utils/SQLSchemaUtilsTest.java index b874303b6..a37de3b4e 100644 --- a/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/utils/SQLSchemaUtilsTest.java +++ b/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/utils/SQLSchemaUtilsTest.java @@ -1,6 +1,8 @@ package com.linkedin.metadata.dao.utils; import com.linkedin.testing.AspectFoo; +import com.linkedin.testing.urn.BarUrn; +import com.linkedin.testing.urn.FooUrn; import org.testng.annotations.Test; import static com.linkedin.metadata.dao.utils.SQLSchemaUtils.*; @@ -11,15 +13,18 @@ public class SQLSchemaUtilsTest { @Test public void testGetGeneratedColumnName() { - String generatedColumnName = getGeneratedColumnName(AspectFoo.class.getCanonicalName(), "/value", false); + String generatedColumnName = + getGeneratedColumnName(FooUrn.ENTITY_TYPE, AspectFoo.class.getCanonicalName(), "/value", false); assertEquals(generatedColumnName, "i_aspectfoo$value"); - generatedColumnName = getGeneratedColumnName(AspectFoo.class.getCanonicalName(), "/value", true); + generatedColumnName = + getGeneratedColumnName(FooUrn.ENTITY_TYPE, AspectFoo.class.getCanonicalName(), "/value", true); assertEquals(generatedColumnName, "i_aspectfoo0value"); } @Test public void testGetAspectColumnName() { - assertEquals("a_aspectbar", SQLSchemaUtils.getAspectColumnName("com.linkedin.testing.AspectBar")); + assertEquals("a_aspectbar", + SQLSchemaUtils.getAspectColumnName(BarUrn.ENTITY_TYPE, "com.linkedin.testing.AspectBar")); } } \ No newline at end of file diff --git a/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/utils/SQLStatementUtilsTest.java b/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/utils/SQLStatementUtilsTest.java index 68b1e10a3..571d919c5 100644 --- a/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/utils/SQLStatementUtilsTest.java +++ b/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/utils/SQLStatementUtilsTest.java @@ -78,7 +78,7 @@ public void testCreateFilterSql() { indexCriterionArray.add(indexCriterion2); indexFilter.setCriteria(indexCriterionArray); - String sql1 = SQLStatementUtils.createFilterSql("metadata_entity_foo", indexFilter, true, false); + String sql1 = SQLStatementUtils.createFilterSql("foo", indexFilter, true, false); String expectedSql1 = "SELECT *, (SELECT COUNT(urn) FROM metadata_entity_foo WHERE a_aspectfoo IS NOT NULL\n" + "AND JSON_EXTRACT(a_aspectfoo, '$.gma_deleted') IS NULL\n" + "AND i_aspectfoo$value >= 25\n" + "AND a_aspectfoo IS NOT NULL\n" + "AND JSON_EXTRACT(a_aspectfoo, '$.gma_deleted') IS NULL\n" @@ -89,7 +89,7 @@ public void testCreateFilterSql() { assertEquals(sql1, expectedSql1); - String sql2 = SQLStatementUtils.createFilterSql("metadata_entity_foo", indexFilter, true, true); + String sql2 = SQLStatementUtils.createFilterSql("foo", indexFilter, true, true); String expectedSql2 = "SELECT *, (SELECT COUNT(urn) FROM metadata_entity_foo WHERE a_aspectfoo IS NOT NULL\n" + "AND JSON_EXTRACT(a_aspectfoo, '$.gma_deleted') IS NULL\n" + "AND i_aspectfoo0value >= 25\n" + "AND a_aspectfoo IS NOT NULL\n" + "AND JSON_EXTRACT(a_aspectfoo, '$.gma_deleted') IS NULL\n" @@ -120,14 +120,14 @@ public void testCreateGroupBySql() { indexGroupByCriterion.setAspect(AspectFoo.class.getCanonicalName()); indexGroupByCriterion.setPath("/value"); - String sql1 = SQLStatementUtils.createGroupBySql("metadata_entity_foo", indexFilter, indexGroupByCriterion, false); + String sql1 = SQLStatementUtils.createGroupBySql("foo", indexFilter, indexGroupByCriterion, false); assertEquals(sql1, "SELECT count(*) as COUNT, i_aspectfoo$value FROM metadata_entity_foo\n" + "WHERE a_aspectfoo IS NOT NULL\n" + "AND JSON_EXTRACT(a_aspectfoo, '$.gma_deleted') IS NULL\n" + "AND i_aspectfoo$value >= 25\n" + "AND a_aspectfoo IS NOT NULL\n" + "AND JSON_EXTRACT(a_aspectfoo, '$.gma_deleted') IS NULL\n" + "AND i_aspectfoo$value < 50\n" + "GROUP BY i_aspectfoo$value"); - String sql2 = SQLStatementUtils.createGroupBySql("metadata_entity_foo", indexFilter, indexGroupByCriterion, true); + String sql2 = SQLStatementUtils.createGroupBySql("foo", indexFilter, indexGroupByCriterion, true); assertEquals(sql2, "SELECT count(*) as COUNT, i_aspectfoo0value FROM metadata_entity_foo\n" + "WHERE a_aspectfoo IS NOT NULL\n" + "AND JSON_EXTRACT(a_aspectfoo, '$.gma_deleted') IS NULL\n" + "AND i_aspectfoo0value >= 25\n" + "AND a_aspectfoo IS NOT NULL\n" @@ -348,14 +348,13 @@ public void testCreateListAspectByUrnSql() throws URISyntaxException { @Test public void testCreateListAspectSql() throws URISyntaxException { FooUrn fooUrn = new FooUrn(1); - String tableName = SQLSchemaUtils.getTableName(fooUrn.getEntityType()); assertEquals( - SQLStatementUtils.createListAspectWithPaginationSql(AspectFoo.class, tableName, true, 0, 5), + SQLStatementUtils.createListAspectWithPaginationSql(AspectFoo.class, fooUrn.getEntityType(), true, 0, 5), "SELECT urn, a_aspectfoo, lastmodifiedon, lastmodifiedby, createdfor, (SELECT COUNT(urn) FROM " + "metadata_entity_foo WHERE a_aspectfoo IS NOT NULL) as _total_count FROM metadata_entity_foo " + "WHERE a_aspectfoo IS NOT NULL LIMIT 5 OFFSET 0"); assertEquals( - SQLStatementUtils.createListAspectWithPaginationSql(AspectFoo.class, tableName, false, 0, 5), + SQLStatementUtils.createListAspectWithPaginationSql(AspectFoo.class, fooUrn.getEntityType(), false, 0, 5), "SELECT urn, a_aspectfoo, lastmodifiedon, lastmodifiedby, createdfor, (SELECT COUNT(urn) FROM " + "metadata_entity_foo WHERE a_aspectfoo IS NOT NULL AND JSON_EXTRACT(a_aspectfoo, '$.gma_deleted') IS NULL) " + "as _total_count FROM metadata_entity_foo WHERE a_aspectfoo IS NOT NULL AND " diff --git a/testing/test-models/src/main/pegasus/com/linkedin/testing/BarAsset b/testing/test-models/src/main/pegasus/com/linkedin/testing/BarAsset new file mode 100644 index 000000000..d24a66b9d --- /dev/null +++ b/testing/test-models/src/main/pegasus/com/linkedin/testing/BarAsset @@ -0,0 +1,19 @@ +namespace com.linkedin.testing + +import com.linkedin.common.Urn +import com.linkedin.testing.localrelationship.AspectFooBar + +/** + * For unit tests + */ +record BarAsset { + /** + * For unit tests + */ + urn: optional BarUrn + + /** + * For unit tests + */ + aspectbar: optional AspectBar, +} \ No newline at end of file