Merge branch 'master' into fix/metadata-io/use-proper-algorithm-to-ge…

…t-lineage

.github/workflows/docker-ingestion-smoke.yml

@@ -47,6 +47,7 @@ jobs:
     name: Build and Push Docker Image to Docker Hub
     runs-on: ubuntu-latest
     needs: setup
+    if: ${{ needs.setup.outputs.publish == 'true' }}
     steps:
       - name: Check out the repo
         uses: actions/checkout@v3

build.gradle

@@ -1,7 +1,7 @@
 buildscript {
   ext.junitJupiterVersion = '5.6.1'
   // Releases: https://github.com/linkedin/rest.li/blob/master/CHANGELOG.md
-  ext.pegasusVersion = '29.45.0'
+  ext.pegasusVersion = '29.46.8'
   ext.mavenVersion = '3.6.3'
   ext.springVersion = '5.3.29'
   ext.springBootVersion = '2.7.14'
@@ -216,7 +216,7 @@ project.ext.externalDependency = [
     'testContainersOpenSearch': 'org.opensearch:opensearch-testcontainers:2.0.0',
     'typesafeConfig':'com.typesafe:config:1.4.1',
     'wiremock':'com.github.tomakehurst:wiremock:2.10.0',
-    'zookeeper': 'org.apache.zookeeper:zookeeper:3.4.14',
+    'zookeeper': 'org.apache.zookeeper:zookeeper:3.7.2',
     'wire': 'com.squareup.wire:wire-compiler:3.7.1',
     'charle':  'com.charleskorn.kaml:kaml:0.53.0',
     'common': 'commons-io:commons-io:2.7',

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java

@@ -1433,6 +1433,10 @@ private void configureChartResolvers(final RuntimeWiring.Builder builder) {
             .dataFetcher("statsSummary", new ChartStatsSummaryResolver(this.timeseriesAspectService))
             .dataFetcher("privileges", new EntityPrivilegesResolver(entityClient))
             .dataFetcher("exists", new EntityExistsResolver(entityService))
+            .dataFetcher("subTypes", new SubTypesResolver(
+                this.entityClient,
+                "chart",
+                "subTypes"))
         );
         builder.type("ChartInfo", typeWiring -> typeWiring
             .dataFetcher("inputs", new LoadableTypeBatchResolver<>(datasetType,

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/ListPoliciesResolver.java

@@ -40,23 +40,15 @@ public CompletableFuture<ListPoliciesResult> get(final DataFetchingEnvironment e
       final Integer count = input.getCount() == null ? DEFAULT_COUNT : input.getCount();
       final String query = input.getQuery() == null ? DEFAULT_QUERY : input.getQuery();
 
-      return CompletableFuture.supplyAsync(() -> {
-        try {
-          // First, get all policy Urns.
-          final PolicyFetcher.PolicyFetchResult policyFetchResult =
-              _policyFetcher.fetchPolicies(start, count, query, context.getAuthentication());
-
-          // Now that we have entities we can bind this to a result.
-          final ListPoliciesResult result = new ListPoliciesResult();
-          result.setStart(start);
-          result.setCount(count);
-          result.setTotal(policyFetchResult.getTotal());
-          result.setPolicies(mapEntities(policyFetchResult.getPolicies()));
-          return result;
-        } catch (Exception e) {
-          throw new RuntimeException("Failed to list policies", e);
-        }
-      });
+      return _policyFetcher.fetchPolicies(start, query, count, context.getAuthentication())
+              .thenApply(policyFetchResult -> {
+                final ListPoliciesResult result = new ListPoliciesResult();
+                result.setStart(start);
+                result.setCount(count);
+                result.setTotal(policyFetchResult.getTotal());
+                result.setPolicies(mapEntities(policyFetchResult.getPolicies()));
+                return result;
+              });
     }
     throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator.");
   }

datahub-graphql-core/src/main/resources/entity.graphql

@@ -5249,6 +5249,11 @@ type Chart implements EntityWithRelationships & Entity & BrowsableEntity {
     Whether or not this entity exists on DataHub
     """
     exists: Boolean
+
+    """
+    Sub Types that this entity implements
+    """
+    subTypes: SubTypes
 }
 
 """

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/entity/steps/BackfillBrowsePathsV2Step.java

@@ -6,13 +6,15 @@
 import com.linkedin.common.BrowsePathsV2;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.common.urn.UrnUtils;
+import com.linkedin.data.template.StringArray;
 import com.linkedin.datahub.upgrade.UpgradeContext;
 import com.linkedin.datahub.upgrade.UpgradeStep;
 import com.linkedin.datahub.upgrade.UpgradeStepResult;
 import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult;
 import com.linkedin.events.metadata.ChangeType;
 import com.linkedin.metadata.Constants;
 import com.linkedin.metadata.entity.EntityService;
+import com.linkedin.metadata.query.SearchFlags;
 import com.linkedin.metadata.query.filter.Condition;
 import com.linkedin.metadata.query.filter.ConjunctiveCriterion;
 import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray;
@@ -37,6 +39,8 @@
 public class BackfillBrowsePathsV2Step implements UpgradeStep {
 
   public static final String BACKFILL_BROWSE_PATHS_V2 = "BACKFILL_BROWSE_PATHS_V2";
+  public static final String REPROCESS_DEFAULT_BROWSE_PATHS_V2 = "REPROCESS_DEFAULT_BROWSE_PATHS_V2";
+  public static final String DEFAULT_BROWSE_PATH_V2 = "␟Default";
 
   private static final Set<String> ENTITY_TYPES_TO_MIGRATE = ImmutableSet.of(
       Constants.DATASET_ENTITY_NAME,
@@ -81,37 +85,24 @@ public Function<UpgradeContext, UpgradeStepResult> executable() {
 
   private String backfillBrowsePathsV2(String entityType, AuditStamp auditStamp, String scrollId) {
 
-    // Condition: has `browsePaths` AND does NOT have `browsePathV2`
-    Criterion missingBrowsePathV2 = new Criterion();
-    missingBrowsePathV2.setCondition(Condition.IS_NULL);
-    missingBrowsePathV2.setField("browsePathV2");
-    // Excludes entities without browsePaths
-    Criterion hasBrowsePathV1 = new Criterion();
-    hasBrowsePathV1.setCondition(Condition.EXISTS);
-    hasBrowsePathV1.setField("browsePaths");
-
-    CriterionArray criterionArray = new CriterionArray();
-    criterionArray.add(missingBrowsePathV2);
-    criterionArray.add(hasBrowsePathV1);
-
-    ConjunctiveCriterion conjunctiveCriterion = new ConjunctiveCriterion();
-    conjunctiveCriterion.setAnd(criterionArray);
+    final Filter filter;
 
-    ConjunctiveCriterionArray conjunctiveCriterionArray = new ConjunctiveCriterionArray();
-    conjunctiveCriterionArray.add(conjunctiveCriterion);
-
-    Filter filter = new Filter();
-    filter.setOr(conjunctiveCriterionArray);
+    if (System.getenv().containsKey(REPROCESS_DEFAULT_BROWSE_PATHS_V2)
+            && Boolean.parseBoolean(System.getenv(REPROCESS_DEFAULT_BROWSE_PATHS_V2))) {
+      filter = backfillDefaultBrowsePathsV2Filter();
+    } else  {
+      filter = backfillBrowsePathsV2Filter();
+    }
 
     final ScrollResult scrollResult = _searchService.scrollAcrossEntities(
         ImmutableList.of(entityType),
         "*",
         filter,
         null,
         scrollId,
-        "5m",
+        null,
         BATCH_SIZE,
-        null
+        new SearchFlags().setFulltext(true).setSkipCache(true).setSkipHighlighting(true).setSkipAggregates(true)
     );
     if (scrollResult.getNumEntities() == 0 || scrollResult.getEntities().size() == 0) {
       return null;
@@ -129,6 +120,55 @@ private String backfillBrowsePathsV2(String entityType, AuditStamp auditStamp, S
     return scrollResult.getScrollId();
   }
 
+  private Filter backfillBrowsePathsV2Filter() {
+    // Condition: has `browsePaths` AND does NOT have `browsePathV2`
+    Criterion missingBrowsePathV2 = new Criterion();
+    missingBrowsePathV2.setCondition(Condition.IS_NULL);
+    missingBrowsePathV2.setField("browsePathV2");
+    // Excludes entities without browsePaths
+    Criterion hasBrowsePathV1 = new Criterion();
+    hasBrowsePathV1.setCondition(Condition.EXISTS);
+    hasBrowsePathV1.setField("browsePaths");
+
+    CriterionArray criterionArray = new CriterionArray();
+    criterionArray.add(missingBrowsePathV2);
+    criterionArray.add(hasBrowsePathV1);
+
+    ConjunctiveCriterion conjunctiveCriterion = new ConjunctiveCriterion();
+    conjunctiveCriterion.setAnd(criterionArray);
+
+    ConjunctiveCriterionArray conjunctiveCriterionArray = new ConjunctiveCriterionArray();
+    conjunctiveCriterionArray.add(conjunctiveCriterion);
+
+    Filter filter = new Filter();
+    filter.setOr(conjunctiveCriterionArray);
+    return filter;
+  }
+
+  private Filter backfillDefaultBrowsePathsV2Filter() {
+    // Condition: has default `browsePathV2`
+    Criterion hasDefaultBrowsePathV2 = new Criterion();
+    hasDefaultBrowsePathV2.setCondition(Condition.EQUAL);
+    hasDefaultBrowsePathV2.setField("browsePathV2");
+    StringArray values = new StringArray();
+    values.add(DEFAULT_BROWSE_PATH_V2);
+    hasDefaultBrowsePathV2.setValues(values);
+    hasDefaultBrowsePathV2.setValue(DEFAULT_BROWSE_PATH_V2); // not used, but required field?
+
+    CriterionArray criterionArray = new CriterionArray();
+    criterionArray.add(hasDefaultBrowsePathV2);
+
+    ConjunctiveCriterion conjunctiveCriterion = new ConjunctiveCriterion();
+    conjunctiveCriterion.setAnd(criterionArray);
+
+    ConjunctiveCriterionArray conjunctiveCriterionArray = new ConjunctiveCriterionArray();
+    conjunctiveCriterionArray.add(conjunctiveCriterion);
+
+    Filter filter = new Filter();
+    filter.setOr(conjunctiveCriterionArray);
+    return filter;
+  }
+
   private void ingestBrowsePathsV2(Urn urn, AuditStamp auditStamp) throws Exception {
     BrowsePathsV2 browsePathsV2 = _entityService.buildDefaultBrowsePathV2(urn, true);
     log.debug(String.format("Adding browse path v2 for urn %s with value %s", urn, browsePathsV2));
@@ -142,7 +182,7 @@ private void ingestBrowsePathsV2(Urn urn, AuditStamp auditStamp) throws Exceptio
     _entityService.ingestProposal(
         proposal,
         auditStamp,
-        false
+        true
     );
   }
 

datahub-web-react/codegen.yml

@@ -20,6 +20,9 @@ generates:
     src/types.generated.ts:
         plugins:
             - 'typescript'
+    src/possibleTypes.generated.ts:
+        plugins:
+            - 'fragment-matcher'
     src/:
         preset: near-operation-file
         presetConfig:

datahub-web-react/package.json

@@ -11,6 +11,7 @@
         "@apollo/client": "^3.3.19",
         "@craco/craco": "^6.1.1",
         "@data-ui/xy-chart": "^0.0.84",
+        "@graphql-codegen/fragment-matcher": "^5.0.0",
         "@miragejs/graphql": "^0.1.11",
         "@monaco-editor/react": "^4.3.1",
         "@react-hook/window-size": "^3.0.7",

datahub-web-react/src/App.tsx

@@ -36,6 +36,7 @@ import { DataPlatformEntity } from './app/entity/dataPlatform/DataPlatformEntity
 import { DataProductEntity } from './app/entity/dataProduct/DataProductEntity';
 import { DataPlatformInstanceEntity } from './app/entity/dataPlatformInstance/DataPlatformInstanceEntity';
 import { RoleEntity } from './app/entity/Access/RoleEntity';
+import possibleTypesResult from './possibleTypes.generated';
 
 /*
     Construct Apollo Client
@@ -77,6 +78,8 @@ const client = new ApolloClient({
                 },
             },
         },
+        // need to define possibleTypes to allow us to use Apollo cache with union types
+        possibleTypes: possibleTypesResult.possibleTypes,
     }),
     credentials: 'include',
     defaultOptions: {

datahub-web-react/src/app/entity/EntityPage.tsx

@@ -8,7 +8,6 @@ import { useEntityRegistry } from '../useEntityRegistry';
 import analytics, { EventType } from '../analytics';
 import { decodeUrn } from './shared/utils';
 import { useGetGrantedPrivilegesQuery } from '../../graphql/policy.generated';
-import { Message } from '../shared/Message';
 import { UnauthorizedPage } from '../authorization/UnauthorizedPage';
 import { ErrorSection } from '../shared/error/ErrorSection';
 import { VIEW_ENTITY_PAGE } from './shared/constants';
@@ -34,7 +33,7 @@ export const EntityPage = ({ entityType }: Props) => {
     const isLineageSupported = entity.isLineageEnabled();
     const isLineageMode = useIsLineageMode();
     const authenticatedUserUrn = useUserContext()?.user?.urn;
-    const { loading, error, data } = useGetGrantedPrivilegesQuery({
+    const { error, data } = useGetGrantedPrivilegesQuery({
         variables: {
             input: {
                 actorUrn: authenticatedUserUrn as string,
@@ -71,7 +70,6 @@ export const EntityPage = ({ entityType }: Props) => {
 
     return (
         <>
-            {loading && <Message type="loading" content="Loading..." style={{ marginTop: '10%' }} />}
             {error && <ErrorSection />}
             {data && !canViewEntityPage && <UnauthorizedPage />}
             {canViewEntityPage &&

datahub-web-react/src/app/entity/chart/ChartEntity.tsx

@@ -154,10 +154,12 @@ export class ChartEntity implements Entity<Chart> {
     getOverridePropertiesFromEntity = (chart?: Chart | null): GenericEntityProperties => {
         // TODO: Get rid of this once we have correctly formed platform coming back.
         const name = chart?.properties?.name;
+        const subTypes = chart?.subTypes;
         const externalUrl = chart?.properties?.externalUrl;
         return {
             name,
             externalUrl,
+            entityTypeOverride: subTypes ? capitalizeFirstLetterOnly(subTypes.typeNames?.[0]) : '',
         };
     };
 
@@ -187,6 +189,7 @@ export class ChartEntity implements Entity<Chart> {
         return (
             <ChartPreview
                 urn={data.urn}
+                subType={data.subTypes?.typeNames?.[0]}
                 platform={data?.platform?.properties?.displayName || capitalizeFirstLetterOnly(data?.platform?.name)}
                 platformInstanceId={data.dataPlatformInstance?.instanceId}
                 name={data.properties?.name}
@@ -222,6 +225,7 @@ export class ChartEntity implements Entity<Chart> {
             type: EntityType.Chart,
             icon: entity?.platform?.properties?.logoUrl || undefined,
             platform: entity?.platform,
+            subtype: entity?.subTypes?.typeNames?.[0] || undefined,
         };
     };
 

datahub-web-react/src/app/entity/chart/preview/ChartPreview.tsx

@@ -15,6 +15,7 @@ import {
     EntityPath,
 } from '../../../../types.generated';
 import DefaultPreviewCard from '../../../preview/DefaultPreviewCard';
+import { capitalizeFirstLetterOnly } from '../../../shared/textUtil';
 import { useEntityRegistry } from '../../../useEntityRegistry';
 import { IconStyleType } from '../../Entity';
 import { ChartStatsSummary as ChartStatsSummaryView } from '../shared/ChartStatsSummary';
@@ -43,6 +44,7 @@ export const ChartPreview = ({
     snippet,
     degree,
     paths,
+    subType,
 }: {
     urn: string;
     platform?: string;
@@ -67,6 +69,7 @@ export const ChartPreview = ({
     snippet?: React.ReactNode | null;
     degree?: number;
     paths?: EntityPath[];
+    subType?: string | null;
 }): JSX.Element => {
     const entityRegistry = useEntityRegistry();
 
@@ -76,7 +79,7 @@ export const ChartPreview = ({
             name={name || ''}
             urn={urn}
             description={description || ''}
-            type="Chart"
+            type={capitalizeFirstLetterOnly(subType) || 'Chart'}
             typeIcon={entityRegistry.getIcon(EntityType.Chart, 14, IconStyleType.ACCENT)}
             logoUrl={logoUrl || ''}
             platform={platform}

datahub-web-react/src/app/entity/dashboard/shared/DashboardStatsSummary.tsx

@@ -11,6 +11,9 @@ import ExpandingStat from '../../dataset/shared/ExpandingStat';
 
 const StatText = styled.span`
     color: ${ANTD_GRAY[8]};
+    @media (min-width: 1024px) {
+        width: 100%;
+        white-space: nowrap;
 `;
 
 const HelpIcon = styled(QuestionCircleOutlined)`

datahub-web-react/src/app/entity/dataset/shared/DatasetStatsSummary.tsx

@@ -12,6 +12,9 @@ import ExpandingStat from './ExpandingStat';
 
 const StatText = styled.span<{ color: string }>`
     color: ${(props) => props.color};
+    @media (min-width: 1160px) {
+        width: 100%;
+        white-space: nowrap;
 `;
 
 const PopoverContent = styled.div`

datahub-web-react/src/app/entity/dataset/shared/ExpandingStat.tsx

@@ -2,9 +2,7 @@ import React, { ReactNode, useEffect, useRef, useState } from 'react';
 import styled from 'styled-components';
 
 const ExpandingStatContainer = styled.span<{ disabled: boolean; expanded: boolean; width: string }>`
-    overflow: hidden;
-    white-space: nowrap;
-    width: ${(props) => props.width};
+    max-width: 100%;
     transition: width 250ms ease;
 `;
 
@@ -13,6 +11,7 @@ const ExpandingStat = ({
     render,
 }: {
     disabled?: boolean;
+
     render: (isExpanded: boolean) => ReactNode;
 }) => {
     const contentRef = useRef<HTMLSpanElement>(null);