Skip to content

Commit

Permalink
Merge - Sync with master
Browse files Browse the repository at this point in the history
  • Loading branch information
rtekal committed Jun 21, 2023
2 parents 5ff65c5 + af6973e commit 03319c1
Show file tree
Hide file tree
Showing 190 changed files with 13,177 additions and 5,977 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ public class Constants {
public static final String STEPS_SCHEMA_FILE = "step.graphql";
public static final String LINEAGE_SCHEMA_FILE = "lineage.graphql";
public static final String BROWSE_PATH_DELIMITER = "/";
public static final String BROWSE_PATH_V2_DELIMITER = "␟";
public static final String VERSION_STAMP_FIELD_NAME = "versionStamp";

public static final String ENTITY_FILTER_NAME = "_entityType";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@
import com.linkedin.datahub.graphql.generated.Assertion;
import com.linkedin.datahub.graphql.generated.AutoCompleteResultForEntity;
import com.linkedin.datahub.graphql.generated.AutoCompleteResults;
import com.linkedin.datahub.graphql.generated.BrowsePathEntry;
import com.linkedin.datahub.graphql.generated.BrowseResultGroupV2;
import com.linkedin.datahub.graphql.generated.BrowseResults;
import com.linkedin.datahub.graphql.generated.Chart;
import com.linkedin.datahub.graphql.generated.ChartInfo;
Expand Down Expand Up @@ -105,6 +107,7 @@
import com.linkedin.datahub.graphql.resolvers.browse.BrowsePathsResolver;
import com.linkedin.datahub.graphql.resolvers.browse.BrowseResolver;
import com.linkedin.datahub.graphql.resolvers.browse.EntityBrowsePathsResolver;
import com.linkedin.datahub.graphql.resolvers.chart.BrowseV2Resolver;
import com.linkedin.datahub.graphql.resolvers.chart.ChartStatsSummaryResolver;
import com.linkedin.datahub.graphql.resolvers.config.AppConfigResolver;
import com.linkedin.datahub.graphql.resolvers.container.ContainerEntitiesResolver;
Expand Down Expand Up @@ -793,6 +796,7 @@ private void configureQueryResolvers(final RuntimeWiring.Builder builder) {
.dataFetcher("dataProduct", getResolver(dataProductType))
.dataFetcher("listDataProductAssets", new ListDataProductAssetsResolver(this.entityClient))
.dataFetcher("listOwnershipTypes", new ListOwnershipTypesResolver(this.entityClient))
.dataFetcher("browseV2", new BrowseV2Resolver(this.entityClient, this.viewService))
);
}

Expand Down Expand Up @@ -970,6 +974,14 @@ private void configureGenericEntityResolvers(final RuntimeWiring.Builder builder
.dataFetcher("entity", new EntityTypeResolver(entityTypes,
(env) -> ((EntityRelationship) env.getSource()).getEntity()))
)
.type("BrowseResultGroupV2", typeWiring -> typeWiring
.dataFetcher("entity", new EntityTypeResolver(entityTypes,
(env) -> ((BrowseResultGroupV2) env.getSource()).getEntity()))
)
.type("BrowsePathEntry", typeWiring -> typeWiring
.dataFetcher("entity", new EntityTypeResolver(entityTypes,
(env) -> ((BrowsePathEntry) env.getSource()).getEntity()))
)
.type("LineageRelationship", typeWiring -> typeWiring
.dataFetcher("entity", new EntityTypeResolver(entityTypes,
(env) -> ((LineageRelationship) env.getSource()).getEntity()))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,5 +11,7 @@ public class FeatureFlags {
private boolean pointInTimeCreationEnabled = false;
private boolean alwaysEmitChangeLog = false;
private boolean readOnlyModeEnabled = false;
private boolean showSearchFiltersV2 = false;
private boolean showBrowseV2 = false;
private PreProcessHooks preProcessHooks;
}
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
import com.linkedin.metadata.Constants;
import com.linkedin.metadata.authorization.PoliciesConfig;
import com.linkedin.metadata.entity.EntityService;
import com.linkedin.r2.RemoteInvocationException;
import graphql.schema.DataFetcher;
import graphql.schema.DataFetchingEnvironment;
import java.util.concurrent.CompletableFuture;
Expand Down Expand Up @@ -54,7 +53,7 @@ public CompletableFuture<Boolean> get(final DataFetchingEnvironment environment)
CompletableFuture.runAsync(() -> {
try {
_entityClient.deleteEntityReferences(assertionUrn, context.getAuthentication());
} catch (RemoteInvocationException e) {
} catch (Exception e) {
log.error(String.format("Caught exception while attempting to clear all entity references for assertion with urn %s", assertionUrn), e);
}
});
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
package com.linkedin.datahub.graphql.resolvers.chart;

import com.linkedin.common.urn.UrnUtils;
import com.linkedin.datahub.graphql.QueryContext;
import com.linkedin.datahub.graphql.generated.BrowseResultGroupV2;
import com.linkedin.datahub.graphql.generated.BrowseResultMetadata;
import com.linkedin.datahub.graphql.generated.BrowseResultsV2;
import com.linkedin.datahub.graphql.generated.BrowseV2Input;
import com.linkedin.datahub.graphql.resolvers.EntityTypeMapper;
import com.linkedin.datahub.graphql.resolvers.ResolverUtils;
import com.linkedin.datahub.graphql.resolvers.search.SearchUtils;
import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper;
import com.linkedin.entity.client.EntityClient;
import com.linkedin.metadata.browse.BrowseResultV2;
import com.linkedin.metadata.query.filter.Filter;
import com.linkedin.metadata.service.ViewService;
import com.linkedin.view.DataHubViewInfo;
import graphql.schema.DataFetcher;
import graphql.schema.DataFetchingEnvironment;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;

import static com.linkedin.datahub.graphql.Constants.BROWSE_PATH_V2_DELIMITER;
import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument;
import static com.linkedin.datahub.graphql.resolvers.search.SearchUtils.resolveView;

@Slf4j
@RequiredArgsConstructor
public class BrowseV2Resolver implements DataFetcher<CompletableFuture<BrowseResultsV2>> {

private final EntityClient _entityClient;
private final ViewService _viewService;

private static final int DEFAULT_START = 0;
private static final int DEFAULT_COUNT = 10;

@Override
public CompletableFuture<BrowseResultsV2> get(DataFetchingEnvironment environment) {
final QueryContext context = environment.getContext();
final BrowseV2Input input = bindArgument(environment.getArgument("input"), BrowseV2Input.class);
final String entityName = EntityTypeMapper.getName(input.getType());

final int start = input.getStart() != null ? input.getStart() : DEFAULT_START;
final int count = input.getCount() != null ? input.getCount() : DEFAULT_COUNT;
final String query = input.getQuery() != null ? input.getQuery() : "*";

return CompletableFuture.supplyAsync(() -> {
try {
final DataHubViewInfo maybeResolvedView = (input.getViewUrn() != null)
? resolveView(_viewService, UrnUtils.getUrn(input.getViewUrn()), context.getAuthentication())
: null;
final String pathStr = input.getPath().size() > 0 ? BROWSE_PATH_V2_DELIMITER + String.join(BROWSE_PATH_V2_DELIMITER, input.getPath()) : "";
final Filter filter = ResolverUtils.buildFilter(null, input.getOrFilters());

BrowseResultV2 browseResults = _entityClient.browseV2(
entityName,
pathStr,
maybeResolvedView != null
? SearchUtils.combineFilters(filter, maybeResolvedView.getDefinition().getFilter())
: filter,
query,
start,
count,
context.getAuthentication()
);
return mapBrowseResults(browseResults);
} catch (Exception e) {
throw new RuntimeException("Failed to execute browse V2", e);
}
});
}

private BrowseResultsV2 mapBrowseResults(BrowseResultV2 browseResults) {
BrowseResultsV2 results = new BrowseResultsV2();
results.setTotal(browseResults.getNumGroups());
results.setStart(browseResults.getFrom());
results.setCount(browseResults.getPageSize());

List<BrowseResultGroupV2> groups = new ArrayList<>();
browseResults.getGroups().forEach(group -> {
BrowseResultGroupV2 browseGroup = new BrowseResultGroupV2();
browseGroup.setName(group.getName());
browseGroup.setCount(group.getCount());
browseGroup.setHasSubGroups(group.isHasSubGroups());
if (group.hasUrn() && group.getUrn() != null) {
browseGroup.setEntity(UrnToEntityMapper.map(group.getUrn()));
}
groups.add(browseGroup);
});
results.setGroups(groups);

BrowseResultMetadata resultMetadata = new BrowseResultMetadata();
resultMetadata.setPath(Arrays.stream(browseResults.getMetadata().getPath()
.split(BROWSE_PATH_V2_DELIMITER))
.filter(pathComponent -> !"".equals(pathComponent))
.collect(Collectors.toList())
);
resultMetadata.setTotalNumEntities(browseResults.getMetadata().getTotalNumEntities());
results.setMetadata(resultMetadata);

return results;
}
}

Original file line number Diff line number Diff line change
Expand Up @@ -147,8 +147,12 @@ public CompletableFuture<AppConfig> get(final DataFetchingEnvironment environmen
viewsConfig.setEnabled(_viewsConfiguration.isEnabled());
appConfig.setViewsConfig(viewsConfig);

final FeatureFlagsConfig featureFlagsConfig = new FeatureFlagsConfig();
featureFlagsConfig.setReadOnlyModeEnabled(_featureFlags.isReadOnlyModeEnabled());
final FeatureFlagsConfig featureFlagsConfig = FeatureFlagsConfig.builder()
.setShowSearchFiltersV2(_featureFlags.isShowSearchFiltersV2())
.setReadOnlyModeEnabled(_featureFlags.isReadOnlyModeEnabled())
.setShowBrowseV2(_featureFlags.isShowBrowseV2())
.build();

appConfig.setFeatureFlags(featureFlagsConfig);

return CompletableFuture.completedFuture(appConfig);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,8 @@ public class ChartType implements SearchableEntityType<Chart, String>, Browsable
DATA_PLATFORM_INSTANCE_ASPECT_NAME,
INPUT_FIELDS_ASPECT_NAME,
EMBED_ASPECT_NAME,
DATA_PRODUCTS_ASPECT_NAME
DATA_PRODUCTS_ASPECT_NAME,
BROWSE_PATHS_V2_ASPECT_NAME
);
private static final Set<String> FACET_FIELDS = ImmutableSet.of("access", "queryType", "tool", "type");

Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package com.linkedin.datahub.graphql.types.chart.mappers;

import com.linkedin.chart.EditableChartProperties;
import com.linkedin.common.BrowsePathsV2;
import com.linkedin.common.DataPlatformInstance;
import com.linkedin.common.Deprecation;
import com.linkedin.common.Embed;
Expand All @@ -25,6 +26,7 @@
import com.linkedin.datahub.graphql.generated.Dataset;
import com.linkedin.datahub.graphql.generated.EntityType;
import com.linkedin.datahub.graphql.types.common.mappers.AuditStampMapper;
import com.linkedin.datahub.graphql.types.common.mappers.BrowsePathsV2Mapper;
import com.linkedin.datahub.graphql.types.common.mappers.DataPlatformInstanceAspectMapper;
import com.linkedin.datahub.graphql.types.common.mappers.DeprecationMapper;
import com.linkedin.datahub.graphql.types.common.mappers.EmbedMapper;
Expand Down Expand Up @@ -93,6 +95,8 @@ public Chart apply(@Nonnull final EntityResponse entityResponse) {
chart.setInputFields(InputFieldsMapper.map(new InputFields(dataMap), entityUrn)));
mappingHelper.mapToResult(EMBED_ASPECT_NAME, (chart, dataMap) ->
chart.setEmbed(EmbedMapper.map(new Embed(dataMap))));
mappingHelper.mapToResult(BROWSE_PATHS_V2_ASPECT_NAME, (chart, dataMap) ->
chart.setBrowsePathV2(BrowsePathsV2Mapper.map(new BrowsePathsV2(dataMap))));
return mappingHelper.getResult();
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
package com.linkedin.datahub.graphql.types.common.mappers;

import com.linkedin.common.BrowsePathsV2;
import com.linkedin.datahub.graphql.generated.BrowsePathEntry;
import com.linkedin.datahub.graphql.generated.BrowsePathV2;
import com.linkedin.datahub.graphql.types.mappers.ModelMapper;

import javax.annotation.Nonnull;
import java.util.List;
import java.util.stream.Collectors;

public class BrowsePathsV2Mapper implements ModelMapper<BrowsePathsV2, BrowsePathV2> {

public static final BrowsePathsV2Mapper INSTANCE = new BrowsePathsV2Mapper();

public static BrowsePathV2 map(@Nonnull final BrowsePathsV2 metadata) {
return INSTANCE.apply(metadata);
}

@Override
public BrowsePathV2 apply(@Nonnull final BrowsePathsV2 input) {
final BrowsePathV2 result = new BrowsePathV2();
final List<BrowsePathEntry> path = input.getPath().stream().map(this::mapBrowsePathEntry).collect(Collectors.toList());
result.setPath(path);
return result;
}

private BrowsePathEntry mapBrowsePathEntry(com.linkedin.common.BrowsePathEntry pathEntry) {
final BrowsePathEntry entry = new BrowsePathEntry();
entry.setName(pathEntry.getId());
if (pathEntry.hasUrn() && pathEntry.getUrn() != null) {
entry.setEntity(UrnToEntityMapper.map(pathEntry.getUrn()));
}
return entry;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,8 @@ public class DashboardType implements SearchableEntityType<Dashboard, String>, B
INPUT_FIELDS_ASPECT_NAME,
SUB_TYPES_ASPECT_NAME,
EMBED_ASPECT_NAME,
DATA_PRODUCTS_ASPECT_NAME
DATA_PRODUCTS_ASPECT_NAME,
BROWSE_PATHS_V2_ASPECT_NAME
);
private static final Set<String> FACET_FIELDS = ImmutableSet.of("access", "tool");

Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.linkedin.datahub.graphql.types.dashboard.mappers;

import com.linkedin.common.BrowsePathsV2;
import com.linkedin.common.DataPlatformInstance;
import com.linkedin.common.Deprecation;
import com.linkedin.common.Embed;
Expand All @@ -24,6 +25,7 @@
import com.linkedin.datahub.graphql.generated.EntityType;
import com.linkedin.datahub.graphql.types.chart.mappers.InputFieldsMapper;
import com.linkedin.datahub.graphql.types.common.mappers.AuditStampMapper;
import com.linkedin.datahub.graphql.types.common.mappers.BrowsePathsV2Mapper;
import com.linkedin.datahub.graphql.types.common.mappers.DataPlatformInstanceAspectMapper;
import com.linkedin.datahub.graphql.types.common.mappers.DeprecationMapper;
import com.linkedin.datahub.graphql.types.common.mappers.EmbedMapper;
Expand Down Expand Up @@ -92,6 +94,8 @@ public Dashboard apply(@Nonnull final EntityResponse entityResponse) {
mappingHelper.mapToResult(SUB_TYPES_ASPECT_NAME, this::mapSubTypes);
mappingHelper.mapToResult(EMBED_ASPECT_NAME, (dashboard, dataMap) ->
dashboard.setEmbed(EmbedMapper.map(new Embed(dataMap))));
mappingHelper.mapToResult(BROWSE_PATHS_V2_ASPECT_NAME, (dashboard, dataMap) ->
dashboard.setBrowsePathV2(BrowsePathsV2Mapper.map(new BrowsePathsV2(dataMap))));
return mappingHelper.getResult();
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,8 @@ public class DataFlowType implements SearchableEntityType<DataFlow, String>, Bro
DOMAINS_ASPECT_NAME,
DEPRECATION_ASPECT_NAME,
DATA_PLATFORM_INSTANCE_ASPECT_NAME,
DATA_PRODUCTS_ASPECT_NAME
DATA_PRODUCTS_ASPECT_NAME,
BROWSE_PATHS_V2_ASPECT_NAME
);
private static final Set<String> FACET_FIELDS = ImmutableSet.of("orchestrator", "cluster");
private final EntityClient _entityClient;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.linkedin.datahub.graphql.types.dataflow.mappers;

import com.linkedin.common.BrowsePathsV2;
import com.linkedin.common.DataPlatformInstance;
import com.linkedin.common.Deprecation;
import com.linkedin.common.GlobalTags;
Expand All @@ -15,6 +16,7 @@
import com.linkedin.datahub.graphql.generated.DataFlowProperties;
import com.linkedin.datahub.graphql.generated.DataPlatform;
import com.linkedin.datahub.graphql.generated.EntityType;
import com.linkedin.datahub.graphql.types.common.mappers.BrowsePathsV2Mapper;
import com.linkedin.datahub.graphql.types.common.mappers.DataPlatformInstanceAspectMapper;
import com.linkedin.datahub.graphql.types.common.mappers.DeprecationMapper;
import com.linkedin.datahub.graphql.types.common.mappers.InstitutionalMemoryMapper;
Expand Down Expand Up @@ -76,6 +78,8 @@ public DataFlow apply(@Nonnull final EntityResponse entityResponse) {
dataFlow.setDeprecation(DeprecationMapper.map(new Deprecation(dataMap))));
mappingHelper.mapToResult(DATA_PLATFORM_INSTANCE_ASPECT_NAME, (dataset, dataMap) ->
dataset.setDataPlatformInstance(DataPlatformInstanceAspectMapper.map(new DataPlatformInstance(dataMap))));
mappingHelper.mapToResult(BROWSE_PATHS_V2_ASPECT_NAME, (dataFlow, dataMap) ->
dataFlow.setBrowsePathV2(BrowsePathsV2Mapper.map(new BrowsePathsV2(dataMap))));

return mappingHelper.getResult();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,8 @@ public class DataJobType implements SearchableEntityType<DataJob, String>, Brows
DOMAINS_ASPECT_NAME,
DEPRECATION_ASPECT_NAME,
DATA_PLATFORM_INSTANCE_ASPECT_NAME,
DATA_PRODUCTS_ASPECT_NAME
DATA_PRODUCTS_ASPECT_NAME,
BROWSE_PATHS_V2_ASPECT_NAME
);
private static final Set<String> FACET_FIELDS = ImmutableSet.of("flow");
private final EntityClient _entityClient;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package com.linkedin.datahub.graphql.types.datajob.mappers;

import com.google.common.collect.ImmutableList;
import com.linkedin.common.BrowsePathsV2;
import com.linkedin.common.DataPlatformInstance;
import com.linkedin.common.Deprecation;
import com.linkedin.common.GlobalTags;
Expand All @@ -18,6 +19,7 @@
import com.linkedin.datahub.graphql.generated.DataJobProperties;
import com.linkedin.datahub.graphql.generated.Dataset;
import com.linkedin.datahub.graphql.generated.EntityType;
import com.linkedin.datahub.graphql.types.common.mappers.BrowsePathsV2Mapper;
import com.linkedin.datahub.graphql.types.common.mappers.DataPlatformInstanceAspectMapper;
import com.linkedin.datahub.graphql.types.common.mappers.DeprecationMapper;
import com.linkedin.datahub.graphql.types.common.mappers.InstitutionalMemoryMapper;
Expand Down Expand Up @@ -98,6 +100,8 @@ public DataJob apply(@Nonnull final EntityResponse entityResponse) {
result.setDeprecation(DeprecationMapper.map(new Deprecation(data)));
} else if (DATA_PLATFORM_INSTANCE_ASPECT_NAME.equals(name)) {
result.setDataPlatformInstance(DataPlatformInstanceAspectMapper.map(new DataPlatformInstance(data)));
} else if (BROWSE_PATHS_V2_ASPECT_NAME.equals(name)) {
result.setBrowsePathV2(BrowsePathsV2Mapper.map(new BrowsePathsV2(data)));
}
});

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,8 @@ public class DatasetType implements SearchableEntityType<Dataset, String>, Brows
DATA_PLATFORM_INSTANCE_ASPECT_NAME,
SIBLINGS_ASPECT_NAME,
EMBED_ASPECT_NAME,
DATA_PRODUCTS_ASPECT_NAME
DATA_PRODUCTS_ASPECT_NAME,
BROWSE_PATHS_V2_ASPECT_NAME
);

private static final Set<String> FACET_FIELDS = ImmutableSet.of("origin", "platform");
Expand Down
Loading

0 comments on commit 03319c1

Please sign in to comment.