diff --git a/.github/scripts/check_python_package.py b/.github/scripts/check_python_package.py index f1f30056917006..1b23d8e621ef06 100644 --- a/.github/scripts/check_python_package.py +++ b/.github/scripts/check_python_package.py @@ -1,18 +1,33 @@ import setuptools +import os folders = ["./smoke-test/tests"] for folder in folders: print(f"Checking folder {folder}") - a = [i for i in setuptools.find_packages(folder) if "cypress" not in i] - b = [i for i in setuptools.find_namespace_packages(folder) if "cypress" not in i] + packages = [i for i in setuptools.find_packages(folder) if "cypress" not in i] + namespace_packages = [ + i for i in setuptools.find_namespace_packages(folder) if "cypress" not in i + ] - in_a_not_b = set(a) - set(b) - in_b_not_a = set(b) - set(a) + print("Packages found:", packages) + print("Namespace packages found:", namespace_packages) + + in_packages_not_namespace = set(packages) - set(namespace_packages) + in_namespace_not_packages = set(namespace_packages) - set(packages) + + if in_packages_not_namespace: + print(f"Packages not in namespace packages: {in_packages_not_namespace}") + if in_namespace_not_packages: + print(f"Namespace packages not in packages: {in_namespace_not_packages}") + for pkg in in_namespace_not_packages: + pkg_path = os.path.join(folder, pkg.replace(".", os.path.sep)) + print(f"Contents of {pkg_path}:") + print(os.listdir(pkg_path)) assert ( - len(in_a_not_b) == 0 - ), f"Found packages in {folder} that are not in namespace packages: {in_a_not_b}" + len(in_packages_not_namespace) == 0 + ), f"Found packages in {folder} that are not in namespace packages: {in_packages_not_namespace}" assert ( - len(in_b_not_a) == 0 - ), f"Found namespace packages in {folder} that are not in packages: {in_b_not_a}" + len(in_namespace_not_packages) == 0 + ), f"Found namespace packages in {folder} that are not in packages: {in_namespace_not_packages}" diff --git a/.github/scripts/docker_helpers.sh b/.github/scripts/docker_helpers.sh index 0487c69eee0ef4..e031a6d2a4d843 100755 --- a/.github/scripts/docker_helpers.sh +++ b/.github/scripts/docker_helpers.sh @@ -16,11 +16,11 @@ function get_tag { } function get_tag_slim { - echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}-slim,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-slim,g'),${SHORT_SHA}-slim + echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}-slim,g" -e 's,refs/tags/\(.*\),\1-slim,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-slim,g'),${SHORT_SHA}-slim } function get_tag_full { - echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}-full,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-full,g'),${SHORT_SHA}-full + echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}-full,g" -e 's,refs/tags/\(.*\),\1-full,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-full,g'),${SHORT_SHA}-full } function get_python_docker_release_v { @@ -32,9 +32,9 @@ function get_unique_tag { } function get_unique_tag_slim { - echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${SHORT_SHA}-slim,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-slim,g') + echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${SHORT_SHA}-slim,g" -e 's,refs/tags/\(.*\),\1-slim,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-slim,g') } function get_unique_tag_full { - echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${SHORT_SHA}-full,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-full,g') + echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${SHORT_SHA}-full,g" -e 's,refs/tags/\(.*\),\1-full,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-full,g') } diff --git a/.github/workflows/airflow-plugin.yml b/.github/workflows/airflow-plugin.yml index ab5b3eb48da7f3..d4f0a1369da253 100644 --- a/.github/workflows/airflow-plugin.yml +++ b/.github/workflows/airflow-plugin.yml @@ -52,7 +52,7 @@ jobs: extra_pip_requirements: 'apache-airflow~=2.8.1 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.1/constraints-3.10.txt' extra_pip_extras: plugin-v2 - python-version: "3.11" - extra_pip_requirements: 'apache-airflow~=2.9.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.10.txt' + extra_pip_requirements: 'apache-airflow~=2.9.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.11.txt' extra_pip_extras: plugin-v2 fail-fast: false steps: diff --git a/.github/workflows/pr-labeler.yml b/.github/workflows/pr-labeler.yml index 77874abedaabd0..52f83b3be5283d 100644 --- a/.github/workflows/pr-labeler.yml +++ b/.github/workflows/pr-labeler.yml @@ -30,12 +30,10 @@ jobs: "treff7es", "yoonhyejin", "eboneil", - "ethan-cartwright", "gabe-lyons", "hsheth2", "jjoyce0510", "maggiehays", - "mrjefflewis", "pedro93", "RyanHolstien", "Kunal-kankriya", @@ -45,7 +43,8 @@ jobs: "kushagra-apptware", "Salman-Apptware", "mayurinehate", - "noggi" + "noggi", + "skrydal" ]'), github.actor ) @@ -60,7 +59,6 @@ jobs: ${{ contains( fromJson('[ - "skrydal", "siladitya2", "sgomezvillamor", "ngamanda", diff --git a/datahub-frontend/app/controllers/SsoCallbackController.java b/datahub-frontend/app/controllers/SsoCallbackController.java index 5e30bf976b8196..750886570bf406 100644 --- a/datahub-frontend/app/controllers/SsoCallbackController.java +++ b/datahub-frontend/app/controllers/SsoCallbackController.java @@ -66,7 +66,8 @@ public SsoCallbackController( public CompletionStage handleCallback(String protocol, Http.Request request) { if (shouldHandleCallback(protocol)) { - log.debug(String.format("Handling SSO callback. Protocol: %s", protocol)); + log.debug("Handling SSO callback. Protocol: {}", + _ssoManager.getSsoProvider().protocol().getCommonName()); return callback(request) .handle( (res, e) -> { diff --git a/datahub-graphql-core/build.gradle b/datahub-graphql-core/build.gradle index de264ce31b719b..49a7fa7fbfbc2f 100644 --- a/datahub-graphql-core/build.gradle +++ b/datahub-graphql-core/build.gradle @@ -22,6 +22,7 @@ dependencies { implementation externalDependency.opentelemetryAnnotations implementation externalDependency.slf4jApi + implementation externalDependency.springContext compileOnly externalDependency.lombok annotationProcessor externalDependency.lombok diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/Constants.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/Constants.java index f70c46ba943a5a..69306862a46ef7 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/Constants.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/Constants.java @@ -24,6 +24,7 @@ private Constants() {} public static final String PROPERTIES_SCHEMA_FILE = "properties.graphql"; public static final String FORMS_SCHEMA_FILE = "forms.graphql"; public static final String ASSERTIONS_SCHEMA_FILE = "assertions.graphql"; + public static final String COMMON_SCHEMA_FILE = "common.graphql"; public static final String INCIDENTS_SCHEMA_FILE = "incident.graphql"; public static final String CONTRACTS_SCHEMA_FILE = "contract.graphql"; public static final String CONNECTIONS_SCHEMA_FILE = "connection.graphql"; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java index b751849859be47..db9bf304a1085b 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java @@ -96,6 +96,7 @@ import com.linkedin.datahub.graphql.generated.MLPrimaryKey; import com.linkedin.datahub.graphql.generated.MLPrimaryKeyProperties; import com.linkedin.datahub.graphql.generated.MatchedField; +import com.linkedin.datahub.graphql.generated.MetadataAttribution; import com.linkedin.datahub.graphql.generated.Notebook; import com.linkedin.datahub.graphql.generated.Owner; import com.linkedin.datahub.graphql.generated.OwnershipTypeEntity; @@ -284,6 +285,8 @@ import com.linkedin.datahub.graphql.resolvers.search.SearchAcrossEntitiesResolver; import com.linkedin.datahub.graphql.resolvers.search.SearchAcrossLineageResolver; import com.linkedin.datahub.graphql.resolvers.search.SearchResolver; +import com.linkedin.datahub.graphql.resolvers.settings.docPropagation.DocPropagationSettingsResolver; +import com.linkedin.datahub.graphql.resolvers.settings.docPropagation.UpdateDocPropagationSettingsResolver; import com.linkedin.datahub.graphql.resolvers.settings.user.UpdateCorpUserViewsSettingsResolver; import com.linkedin.datahub.graphql.resolvers.settings.view.GlobalViewsSettingsResolver; import com.linkedin.datahub.graphql.resolvers.settings.view.UpdateGlobalViewsSettingsResolver; @@ -695,7 +698,8 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) { businessAttributeType)); this.loadableTypes = new ArrayList<>(entityTypes); // Extend loadable types with types from the plugins - // This allows us to offer search and browse capabilities out of the box for those types + // This allows us to offer search and browse capabilities out of the box for + // those types for (GmsGraphQLPlugin plugin : this.graphQLPlugins) { this.entityTypes.addAll(plugin.getEntityTypes()); Collection> pluginLoadableTypes = plugin.getLoadableTypes(); @@ -790,6 +794,7 @@ public void configureRuntimeWiring(final RuntimeWiring.Builder builder) { configureBusinessAttributeAssociationResolver(builder); configureConnectionResolvers(builder); configureDeprecationResolvers(builder); + configureMetadataAttributionResolver(builder); } private void configureOrganisationRoleResolvers(RuntimeWiring.Builder builder) { @@ -843,7 +848,8 @@ public GraphQLEngine.Builder builder() { .addSchema(fileBasedSchema(CONNECTIONS_SCHEMA_FILE)) .addSchema(fileBasedSchema(ASSERTIONS_SCHEMA_FILE)) .addSchema(fileBasedSchema(INCIDENTS_SCHEMA_FILE)) - .addSchema(fileBasedSchema(CONTRACTS_SCHEMA_FILE)); + .addSchema(fileBasedSchema(CONTRACTS_SCHEMA_FILE)) + .addSchema(fileBasedSchema(COMMON_SCHEMA_FILE)); for (GmsGraphQLPlugin plugin : this.graphQLPlugins) { List pluginSchemaFiles = plugin.getSchemaFiles(); @@ -1026,6 +1032,8 @@ private void configureQueryResolvers(final RuntimeWiring.Builder builder) { .dataFetcher("mlModel", getResolver(mlModelType)) .dataFetcher("mlModelGroup", getResolver(mlModelGroupType)) .dataFetcher("assertion", getResolver(assertionType)) + .dataFetcher("form", getResolver(formType)) + .dataFetcher("view", getResolver(dataHubViewType)) .dataFetcher("listPolicies", new ListPoliciesResolver(this.entityClient)) .dataFetcher("getGrantedPrivileges", new GetGrantedPrivilegesResolver()) .dataFetcher("listUsers", new ListUsersResolver(this.entityClient)) @@ -1087,8 +1095,10 @@ private void configureQueryResolvers(final RuntimeWiring.Builder builder) { new BrowseV2Resolver(this.entityClient, this.viewService, this.formService)) .dataFetcher("businessAttribute", getResolver(businessAttributeType)) .dataFetcher( - "listBusinessAttributes", - new ListBusinessAttributesResolver(this.entityClient))); + "listBusinessAttributes", new ListBusinessAttributesResolver(this.entityClient)) + .dataFetcher( + "docPropagationSettings", + new DocPropagationSettingsResolver(this.settingsService))); } private DataFetcher getEntitiesResolver() { @@ -1340,7 +1350,11 @@ private void configureMutationResolvers(final RuntimeWiring.Builder builder) { .dataFetcher( "createForm", new CreateFormResolver(this.entityClient, this.formService)) .dataFetcher("deleteForm", new DeleteFormResolver(this.entityClient)) - .dataFetcher("updateForm", new UpdateFormResolver(this.entityClient)); + .dataFetcher("updateForm", new UpdateFormResolver(this.entityClient)) + .dataFetcher( + "updateDocPropagationSettings", + new UpdateDocPropagationSettingsResolver(this.settingsService)); + if (featureFlags.isBusinessAttributeEntityEnabled()) { typeWiring .dataFetcher( @@ -1869,7 +1883,9 @@ private void configureCorpGroupResolvers(final RuntimeWiring.Builder builder) { "CorpGroup", typeWiring -> typeWiring - .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)) + .dataFetcher( + "relationships", + new EntityRelationshipsResultResolver(graphClient, entityService)) .dataFetcher("privileges", new EntityPrivilegesResolver(entityClient)) .dataFetcher( "aspects", new WeaklyTypedAspectsResolver(entityClient, entityRegistry)) @@ -2821,7 +2837,8 @@ private void configureContractResolvers(final RuntimeWiring.Builder builder) { } private void configurePolicyResolvers(final RuntimeWiring.Builder builder) { - // Register resolvers for "resolvedUsers" and "resolvedGroups" field of the Policy type. + // Register resolvers for "resolvedUsers" and "resolvedGroups" field of the + // Policy type. builder.type( "ActorFilter", typeWiring -> @@ -3174,4 +3191,20 @@ private void configureDeprecationResolvers(final RuntimeWiring.Builder builder) new EntityTypeResolver( entityTypes, (env) -> ((Deprecation) env.getSource()).getActorEntity()))); } + + private void configureMetadataAttributionResolver(final RuntimeWiring.Builder builder) { + builder.type( + "MetadataAttribution", + typeWiring -> + typeWiring + .dataFetcher( + "actor", + new EntityTypeResolver( + entityTypes, (env) -> ((MetadataAttribution) env.getSource()).getActor())) + .dataFetcher( + "source", + new EntityTypeResolver( + entityTypes, + (env) -> ((MetadataAttribution) env.getSource()).getSource()))); + } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java index b9b4e4f4ef292b..4fb49d79a0aa70 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java @@ -398,6 +398,11 @@ public static boolean canManageForms(@Nonnull QueryContext context) { PoliciesConfig.MANAGE_DOCUMENTATION_FORMS_PRIVILEGE); } + public static boolean canManageFeatures(@Nonnull QueryContext context) { + return AuthUtil.isAuthorized( + context.getAuthorizer(), context.getActorUrn(), PoliciesConfig.MANAGE_FEATURES_PRIVILEGE); + } + public static boolean isAuthorized( @Nonnull Authorizer authorizer, @Nonnull String actor, diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/auth/RevokeAccessTokenResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/auth/RevokeAccessTokenResolver.java index 53ae6d4509e7df..eb152087699024 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/auth/RevokeAccessTokenResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/auth/RevokeAccessTokenResolver.java @@ -41,7 +41,7 @@ public CompletableFuture get(DataFetchingEnvironment environment) throw final QueryContext context = environment.getContext(); final String tokenId = bindArgument(environment.getArgument("tokenId"), String.class); - log.info("User {} revoking access token {}", context.getActorUrn(), tokenId); + log.info("User {} revoking access token", context.getActorUrn()); if (isAuthorizedToRevokeToken(context, tokenId)) { try { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/DeleteSecretResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/DeleteSecretResolver.java index f557b9889f604d..da81d560c6dbd6 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/DeleteSecretResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/DeleteSecretResolver.java @@ -23,16 +23,16 @@ public DeleteSecretResolver(final EntityClient entityClient) { public CompletableFuture get(final DataFetchingEnvironment environment) throws Exception { final QueryContext context = environment.getContext(); if (IngestionAuthUtils.canManageSecrets(context)) { - final String secretUrn = environment.getArgument("urn"); - final Urn urn = Urn.createFromString(secretUrn); + final String inputUrn = environment.getArgument("urn"); + final Urn urn = Urn.createFromString(inputUrn); return GraphQLConcurrencyUtils.supplyAsync( () -> { try { _entityClient.deleteEntity(context.getOperationContext(), urn); - return secretUrn; + return inputUrn; } catch (Exception e) { throw new RuntimeException( - String.format("Failed to perform delete against secret with urn %s", secretUrn), + String.format("Failed to perform delete against secret with urn %s", inputUrn), e); } }, diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/SecretUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/SecretUtils.java index 225a5801adec94..87a3e5cb79ebfc 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/SecretUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/SecretUtils.java @@ -25,7 +25,7 @@ static String encrypt(String value, String secret) { } catch (NoSuchAlgorithmException e) { e.printStackTrace(); } - Cipher cipher = Cipher.getInstance("AES/ECB/PKCS5Padding"); + Cipher cipher = Cipher.getInstance("AES"); cipher.init(Cipher.ENCRYPT_MODE, secretKey); return Base64.getEncoder() .encodeToString(cipher.doFinal(value.getBytes(StandardCharsets.UTF_8))); @@ -48,7 +48,7 @@ static String decrypt(String encryptedValue, String secret) { } catch (NoSuchAlgorithmException e) { e.printStackTrace(); } - Cipher cipher = Cipher.getInstance("AES/ECB/PKCS5PADDING"); + Cipher cipher = Cipher.getInstance("AES"); cipher.init(Cipher.DECRYPT_MODE, secretKey); return new String(cipher.doFinal(Base64.getDecoder().decode(encryptedValue))); } catch (Exception e) { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/source/UpsertIngestionSourceResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/source/UpsertIngestionSourceResolver.java index 77fabd7167300e..12266db05b6d10 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/source/UpsertIngestionSourceResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/source/UpsertIngestionSourceResolver.java @@ -25,12 +25,16 @@ import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; import java.net.URISyntaxException; +import java.time.DateTimeException; +import java.time.ZoneId; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.support.CronExpression; /** Creates or updates an ingestion source. Requires the MANAGE_INGESTION privilege. */ @Slf4j @@ -46,55 +50,51 @@ public UpsertIngestionSourceResolver(final EntityClient entityClient) { public CompletableFuture get(final DataFetchingEnvironment environment) throws Exception { final QueryContext context = environment.getContext(); - return GraphQLConcurrencyUtils.supplyAsync( - () -> { - if (IngestionAuthUtils.canManageIngestion(context)) { - - final Optional ingestionSourceUrn = - Optional.ofNullable(environment.getArgument("urn")); - final UpdateIngestionSourceInput input = - bindArgument(environment.getArgument("input"), UpdateIngestionSourceInput.class); + if (!IngestionAuthUtils.canManageIngestion(context)) { + throw new AuthorizationException( + "Unauthorized to perform this action. Please contact your DataHub administrator."); + } + final Optional ingestionSourceUrn = Optional.ofNullable(environment.getArgument("urn")); + final UpdateIngestionSourceInput input = + bindArgument(environment.getArgument("input"), UpdateIngestionSourceInput.class); - // Create the policy info. - final DataHubIngestionSourceInfo info = mapIngestionSourceInfo(input); - final MetadataChangeProposal proposal; - if (ingestionSourceUrn.isPresent()) { - // Update existing ingestion source - try { - proposal = - buildMetadataChangeProposalWithUrn( - Urn.createFromString(ingestionSourceUrn.get()), - INGESTION_INFO_ASPECT_NAME, - info); - } catch (URISyntaxException e) { - throw new DataHubGraphQLException( - String.format("Malformed urn %s provided.", ingestionSourceUrn.get()), - DataHubGraphQLErrorCode.BAD_REQUEST); - } - } else { - // Create new ingestion source - // Since we are creating a new Ingestion Source, we need to generate a unique UUID. - final UUID uuid = UUID.randomUUID(); - final String uuidStr = uuid.toString(); - final DataHubIngestionSourceKey key = new DataHubIngestionSourceKey(); - key.setId(uuidStr); - proposal = - buildMetadataChangeProposalWithKey( - key, INGESTION_SOURCE_ENTITY_NAME, INGESTION_INFO_ASPECT_NAME, info); - } + // Create the policy info. + final DataHubIngestionSourceInfo info = mapIngestionSourceInfo(input); + final MetadataChangeProposal proposal; + if (ingestionSourceUrn.isPresent()) { + // Update existing ingestion source + try { + proposal = + buildMetadataChangeProposalWithUrn( + Urn.createFromString(ingestionSourceUrn.get()), INGESTION_INFO_ASPECT_NAME, info); + } catch (URISyntaxException e) { + throw new DataHubGraphQLException( + String.format("Malformed urn %s provided.", ingestionSourceUrn.get()), + DataHubGraphQLErrorCode.BAD_REQUEST); + } + } else { + // Create new ingestion source + // Since we are creating a new Ingestion Source, we need to generate a unique UUID. + final UUID uuid = UUID.randomUUID(); + final String uuidStr = uuid.toString(); + final DataHubIngestionSourceKey key = new DataHubIngestionSourceKey(); + key.setId(uuidStr); + proposal = + buildMetadataChangeProposalWithKey( + key, INGESTION_SOURCE_ENTITY_NAME, INGESTION_INFO_ASPECT_NAME, info); + } - try { - return _entityClient.ingestProposal(context.getOperationContext(), proposal, false); - } catch (Exception e) { - throw new RuntimeException( - String.format( - "Failed to perform update against ingestion source with urn %s", - input.toString()), - e); - } + return GraphQLConcurrencyUtils.supplyAsync( + () -> { + try { + return _entityClient.ingestProposal(context.getOperationContext(), proposal, false); + } catch (Exception e) { + throw new RuntimeException( + String.format( + "Failed to perform update against ingestion source with urn %s", + input.toString()), + e); } - throw new AuthorizationException( - "Unauthorized to perform this action. Please contact your DataHub administrator."); }, this.getClass().getSimpleName(), "get"); @@ -137,9 +137,38 @@ private DataHubIngestionSourceConfig mapConfig(final UpdateIngestionSourceConfig private DataHubIngestionSourceSchedule mapSchedule( final UpdateIngestionSourceScheduleInput input) { + + final String modifiedCronInterval = adjustCronInterval(input.getInterval()); + try { + CronExpression.parse(modifiedCronInterval); + } catch (IllegalArgumentException e) { + throw new DataHubGraphQLException( + String.format("Invalid cron schedule `%s`: %s", input.getInterval(), e.getMessage()), + DataHubGraphQLErrorCode.BAD_REQUEST); + } + try { + ZoneId.of(input.getTimezone()); + } catch (DateTimeException e) { + throw new DataHubGraphQLException( + String.format("Invalid timezone `%s`: %s", input.getTimezone(), e.getMessage()), + DataHubGraphQLErrorCode.BAD_REQUEST); + } + final DataHubIngestionSourceSchedule result = new DataHubIngestionSourceSchedule(); result.setInterval(input.getInterval()); result.setTimezone(input.getTimezone()); return result; } + + // Copied from IngestionScheduler.java + private String adjustCronInterval(final String origCronInterval) { + Objects.requireNonNull(origCronInterval, "origCronInterval must not be null"); + // Typically we support 5-character cron. Spring's lib only supports 6 character cron so we make + // an adjustment here. + final String[] originalCronParts = origCronInterval.split(" "); + if (originalCronParts.length == 5) { + return String.format("0 %s", origCronInterval); + } + return origCronInterval; + } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityRelationshipsResultResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityRelationshipsResultResolver.java index f775853dd59567..fd72edb2972e36 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityRelationshipsResultResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityRelationshipsResultResolver.java @@ -5,6 +5,7 @@ import com.linkedin.common.EntityRelationship; import com.linkedin.common.EntityRelationships; +import com.linkedin.common.urn.Urn; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.concurrency.GraphQLConcurrencyUtils; import com.linkedin.datahub.graphql.generated.Entity; @@ -12,11 +13,13 @@ import com.linkedin.datahub.graphql.generated.RelationshipsInput; import com.linkedin.datahub.graphql.types.common.mappers.AuditStampMapper; import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper; +import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.graph.GraphClient; import com.linkedin.metadata.query.filter.RelationshipDirection; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; import java.util.List; +import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; import javax.annotation.Nullable; @@ -29,8 +32,16 @@ public class EntityRelationshipsResultResolver private final GraphClient _graphClient; + private final EntityService _entityService; + public EntityRelationshipsResultResolver(final GraphClient graphClient) { + this(graphClient, null); + } + + public EntityRelationshipsResultResolver( + final GraphClient graphClient, final EntityService entityService) { _graphClient = graphClient; + _entityService = entityService; } @Override @@ -47,13 +58,16 @@ public CompletableFuture get(DataFetchingEnvironment final Integer count = input.getCount(); // Optional! final RelationshipDirection resolvedDirection = RelationshipDirection.valueOf(relationshipDirection.toString()); + final boolean includeSoftDelete = input.getIncludeSoftDelete(); + return GraphQLConcurrencyUtils.supplyAsync( () -> mapEntityRelationships( context, fetchEntityRelationships( urn, relationshipTypes, resolvedDirection, start, count, context.getActorUrn()), - resolvedDirection), + resolvedDirection, + includeSoftDelete), this.getClass().getSimpleName(), "get"); } @@ -72,13 +86,28 @@ private EntityRelationships fetchEntityRelationships( private EntityRelationshipsResult mapEntityRelationships( @Nullable final QueryContext context, final EntityRelationships entityRelationships, - final RelationshipDirection relationshipDirection) { + final RelationshipDirection relationshipDirection, + final boolean includeSoftDelete) { final EntityRelationshipsResult result = new EntityRelationshipsResult(); + final Set existentUrns; + if (context != null && _entityService != null && !includeSoftDelete) { + Set allRelatedUrns = + entityRelationships.getRelationships().stream() + .map(EntityRelationship::getEntity) + .collect(Collectors.toSet()); + existentUrns = _entityService.exists(context.getOperationContext(), allRelatedUrns, false); + } else { + existentUrns = null; + } + List viewable = entityRelationships.getRelationships().stream() .filter( - rel -> context == null || canView(context.getOperationContext(), rel.getEntity())) + rel -> + (existentUrns == null || existentUrns.contains(rel.getEntity())) + && (context == null + || canView(context.getOperationContext(), rel.getEntity()))) .collect(Collectors.toList()); result.setStart(entityRelationships.getStart()); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/settings/docPropagation/DocPropagationSettingsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/settings/docPropagation/DocPropagationSettingsResolver.java new file mode 100644 index 00000000000000..84d3bcd7b376c0 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/settings/docPropagation/DocPropagationSettingsResolver.java @@ -0,0 +1,57 @@ +package com.linkedin.datahub.graphql.resolvers.settings.docPropagation; + +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.concurrency.GraphQLConcurrencyUtils; +import com.linkedin.datahub.graphql.generated.DocPropagationSettings; +import com.linkedin.metadata.service.SettingsService; +import com.linkedin.settings.global.GlobalSettingsInfo; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import javax.annotation.Nonnull; +import lombok.extern.slf4j.Slf4j; + +/** Retrieves the Global Settings related to the Actions feature. */ +@Slf4j +public class DocPropagationSettingsResolver + implements DataFetcher> { + + private final SettingsService _settingsService; + + public DocPropagationSettingsResolver(final SettingsService settingsService) { + _settingsService = Objects.requireNonNull(settingsService, "settingsService must not be null"); + } + + @Override + public CompletableFuture get(final DataFetchingEnvironment environment) + throws Exception { + final QueryContext context = environment.getContext(); + return GraphQLConcurrencyUtils.supplyAsync( + () -> { + try { + final GlobalSettingsInfo globalSettings = + _settingsService.getGlobalSettings(context.getOperationContext()); + final DocPropagationSettings defaultSettings = new DocPropagationSettings(); + defaultSettings.setDocColumnPropagation(true); + return globalSettings != null && globalSettings.hasDocPropagation() + ? mapDocPropagationSettings(globalSettings.getDocPropagation()) + : defaultSettings; + } catch (Exception e) { + throw new RuntimeException("Failed to retrieve Action Settings", e); + } + }, + this.getClass().getSimpleName(), + "get"); + } + + private static DocPropagationSettings mapDocPropagationSettings( + @Nonnull final com.linkedin.settings.global.DocPropagationFeatureSettings settings) { + final DocPropagationSettings result = new DocPropagationSettings(); + + // Map docColumnPropagation settings field + result.setDocColumnPropagation(settings.isColumnPropagationEnabled()); + + return result; + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/settings/docPropagation/UpdateDocPropagationSettingsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/settings/docPropagation/UpdateDocPropagationSettingsResolver.java new file mode 100644 index 00000000000000..198c36faad0bd8 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/settings/docPropagation/UpdateDocPropagationSettingsResolver.java @@ -0,0 +1,77 @@ +package com.linkedin.datahub.graphql.resolvers.settings.docPropagation; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; + +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; +import com.linkedin.datahub.graphql.concurrency.GraphQLConcurrencyUtils; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.UpdateDocPropagationSettingsInput; +import com.linkedin.metadata.service.SettingsService; +import com.linkedin.settings.global.DocPropagationFeatureSettings; +import com.linkedin.settings.global.GlobalSettingsInfo; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import javax.annotation.Nonnull; + +/** Resolver responsible for updating the actions settings. */ +public class UpdateDocPropagationSettingsResolver + implements DataFetcher> { + + private final SettingsService _settingsService; + + public UpdateDocPropagationSettingsResolver(@Nonnull final SettingsService settingsService) { + _settingsService = Objects.requireNonNull(settingsService, "settingsService must not be null"); + } + + @Override + public CompletableFuture get(final DataFetchingEnvironment environment) + throws Exception { + final QueryContext context = environment.getContext(); + final UpdateDocPropagationSettingsInput input = + bindArgument(environment.getArgument("input"), UpdateDocPropagationSettingsInput.class); + + return GraphQLConcurrencyUtils.supplyAsync( + () -> { + if (AuthorizationUtils.canManageFeatures(context)) { + try { + // First, fetch the existing global settings. This does a R-M-F. + final GlobalSettingsInfo maybeGlobalSettings = + _settingsService.getGlobalSettings(context.getOperationContext()); + + final GlobalSettingsInfo newGlobalSettings = + maybeGlobalSettings != null ? maybeGlobalSettings : new GlobalSettingsInfo(); + + final DocPropagationFeatureSettings newDocPropagationSettings = + newGlobalSettings.hasDocPropagation() + ? newGlobalSettings.getDocPropagation() + : new DocPropagationFeatureSettings().setEnabled(true); + + // Next, patch the actions settings. + updateDocPropagationSettings(newDocPropagationSettings, input); + newGlobalSettings.setDocPropagation(newDocPropagationSettings); + + // Finally, write back to GMS. + _settingsService.updateGlobalSettings( + context.getOperationContext(), newGlobalSettings); + return true; + } catch (Exception e) { + throw new RuntimeException( + String.format("Failed to update action settings! %s", input), e); + } + } + throw new AuthorizationException( + "Unauthorized to perform this action. Please contact your DataHub administrator."); + }, + this.getClass().getSimpleName(), + "get"); + } + + private static void updateDocPropagationSettings( + @Nonnull final com.linkedin.settings.global.DocPropagationFeatureSettings settings, + @Nonnull final UpdateDocPropagationSettingsInput input) { + settings.setColumnPropagationEnabled(input.getDocColumnPropagation()); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/mappers/InputFieldsMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/mappers/InputFieldsMapper.java index 49c2d17ce09585..269fb7d4ddf793 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/mappers/InputFieldsMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/mappers/InputFieldsMapper.java @@ -5,10 +5,13 @@ import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.InputField; import com.linkedin.datahub.graphql.types.dataset.mappers.SchemaFieldMapper; +import java.net.URISyntaxException; import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import lombok.extern.slf4j.Slf4j; +@Slf4j public class InputFieldsMapper { public static final InputFieldsMapper INSTANCE = new InputFieldsMapper(); @@ -31,13 +34,24 @@ public com.linkedin.datahub.graphql.generated.InputFields apply( .map( field -> { InputField fieldResult = new InputField(); + Urn parentUrn = entityUrn; - if (field.hasSchemaField()) { - fieldResult.setSchemaField( - SchemaFieldMapper.map(context, field.getSchemaField(), entityUrn)); - } if (field.hasSchemaFieldUrn()) { fieldResult.setSchemaFieldUrn(field.getSchemaFieldUrn().toString()); + try { + parentUrn = + Urn.createFromString(field.getSchemaFieldUrn().getEntityKey().get(0)); + } catch (URISyntaxException e) { + log.error( + "Field urn resolution: failed to extract parentUrn successfully from {}. Falling back to {}", + field.getSchemaFieldUrn(), + entityUrn, + e); + } + } + if (field.hasSchemaField()) { + fieldResult.setSchemaField( + SchemaFieldMapper.map(context, field.getSchemaField(), parentUrn)); } return fieldResult; }) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/DocumentationMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/DocumentationMapper.java new file mode 100644 index 00000000000000..dcb4921d353981 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/DocumentationMapper.java @@ -0,0 +1,54 @@ +package com.linkedin.datahub.graphql.types.common.mappers; + +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.DataHubConnection; +import com.linkedin.datahub.graphql.generated.Documentation; +import com.linkedin.datahub.graphql.generated.DocumentationAssociation; +import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.datahub.graphql.types.mappers.ModelMapper; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +public class DocumentationMapper + implements ModelMapper { + + public static final DocumentationMapper INSTANCE = new DocumentationMapper(); + + public static Documentation map( + @Nullable final QueryContext context, + @Nonnull final com.linkedin.common.Documentation metadata) { + return INSTANCE.apply(context, metadata); + } + + @Override + public Documentation apply( + @Nullable final QueryContext context, + @Nonnull final com.linkedin.common.Documentation input) { + final Documentation result = new Documentation(); + result.setDocumentations( + input.getDocumentations().stream() + .map(docAssociation -> mapDocAssociation(context, docAssociation)) + .collect(Collectors.toList())); + return result; + } + + private DocumentationAssociation mapDocAssociation( + @Nullable final QueryContext context, + @Nonnull final com.linkedin.common.DocumentationAssociation association) { + final DocumentationAssociation result = new DocumentationAssociation(); + result.setDocumentation(association.getDocumentation()); + if (association.getAttribution() != null) { + result.setAttribution(MetadataAttributionMapper.map(context, association.getAttribution())); + } + return result; + } + + private DataHubConnection mapConnectionEntity(@Nonnull final Urn urn) { + DataHubConnection connection = new DataHubConnection(); + connection.setUrn(urn.toString()); + connection.setType(EntityType.DATAHUB_CONNECTION); + return connection; + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/MetadataAttributionMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/MetadataAttributionMapper.java new file mode 100644 index 00000000000000..55fb7ad6f3a2b8 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/MetadataAttributionMapper.java @@ -0,0 +1,35 @@ +package com.linkedin.datahub.graphql.types.common.mappers; + +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.MetadataAttribution; +import com.linkedin.datahub.graphql.types.mappers.ModelMapper; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +public class MetadataAttributionMapper + implements ModelMapper { + + public static final MetadataAttributionMapper INSTANCE = new MetadataAttributionMapper(); + + public static MetadataAttribution map( + @Nullable final QueryContext context, + @Nonnull final com.linkedin.common.MetadataAttribution metadata) { + return INSTANCE.apply(context, metadata); + } + + @Override + public MetadataAttribution apply( + @Nullable final QueryContext context, + @Nonnull final com.linkedin.common.MetadataAttribution input) { + final MetadataAttribution result = new MetadataAttribution(); + result.setTime(input.getTime()); + result.setActor(UrnToEntityMapper.map(context, input.getActor())); + if (input.getSource() != null) { + result.setSource(UrnToEntityMapper.map(context, input.getSource())); + } + if (input.getSourceDetail() != null) { + result.setSourceDetail(StringMapMapper.map(context, input.getSourceDetail())); + } + return result; + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/schemafield/SchemaFieldMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/schemafield/SchemaFieldMapper.java index 85a6b9108cb54e..b1f27357d45504 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/schemafield/SchemaFieldMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/schemafield/SchemaFieldMapper.java @@ -1,14 +1,15 @@ package com.linkedin.datahub.graphql.types.schemafield; -import static com.linkedin.metadata.Constants.BUSINESS_ATTRIBUTE_ASPECT; -import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTIES_ASPECT_NAME; +import static com.linkedin.metadata.Constants.*; import com.linkedin.businessattribute.BusinessAttributes; +import com.linkedin.common.Documentation; import com.linkedin.common.urn.Urn; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.SchemaFieldEntity; import com.linkedin.datahub.graphql.types.businessattribute.mappers.BusinessAttributesMapper; +import com.linkedin.datahub.graphql.types.common.mappers.DocumentationMapper; import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper; import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper; import com.linkedin.datahub.graphql.types.mappers.ModelMapper; @@ -46,6 +47,10 @@ public SchemaFieldEntity apply( (((schemaField, dataMap) -> schemaField.setBusinessAttributes( BusinessAttributesMapper.map(new BusinessAttributes(dataMap), entityUrn))))); + mappingHelper.mapToResult( + DOCUMENTATION_ASPECT_NAME, + (entity, dataMap) -> + entity.setDocumentation(DocumentationMapper.map(context, new Documentation(dataMap)))); return result; } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/schemafield/SchemaFieldType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/schemafield/SchemaFieldType.java index f9fd7c7b819c56..2fa26d8cf2cdd7 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/schemafield/SchemaFieldType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/schemafield/SchemaFieldType.java @@ -1,8 +1,6 @@ package com.linkedin.datahub.graphql.types.schemafield; -import static com.linkedin.metadata.Constants.BUSINESS_ATTRIBUTE_ASPECT; -import static com.linkedin.metadata.Constants.SCHEMA_FIELD_ENTITY_NAME; -import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTIES_ASPECT_NAME; +import static com.linkedin.metadata.Constants.*; import com.google.common.collect.ImmutableSet; import com.linkedin.common.urn.Urn; @@ -32,7 +30,8 @@ public class SchemaFieldType implements com.linkedin.datahub.graphql.types.EntityType { public static final Set ASPECTS_TO_FETCH = - ImmutableSet.of(STRUCTURED_PROPERTIES_ASPECT_NAME, BUSINESS_ATTRIBUTE_ASPECT); + ImmutableSet.of( + STRUCTURED_PROPERTIES_ASPECT_NAME, BUSINESS_ATTRIBUTE_ASPECT, DOCUMENTATION_ASPECT_NAME); private final EntityClient _entityClient; private final FeatureFlags _featureFlags; diff --git a/datahub-graphql-core/src/main/resources/app.graphql b/datahub-graphql-core/src/main/resources/app.graphql index 23c2468bae19a1..024a7a989f9db5 100644 --- a/datahub-graphql-core/src/main/resources/app.graphql +++ b/datahub-graphql-core/src/main/resources/app.graphql @@ -17,6 +17,11 @@ extend type Query { Requires the 'Manage Global Views' Platform Privilege. """ globalViewsSettings: GlobalViewsSettings + + """ + Fetch the global settings related to the docs propagation feature. + """ + docPropagationSettings: DocPropagationSettings } extend type Mutation { @@ -25,6 +30,11 @@ extend type Mutation { Requires the 'Manage Global Views' Platform Privilege. """ updateGlobalViewsSettings(input: UpdateGlobalViewsSettingsInput!): Boolean! + + """ + Update the doc propagation settings. + """ + updateDocPropagationSettings(input: UpdateDocPropagationSettingsInput!): Boolean! } """ @@ -469,7 +479,7 @@ type FeatureFlagsConfig { platformBrowseV2: Boolean! """ - Whether we should show CTAs in the UI related to moving to Managed DataHub by Acryl. + Whether we should show CTAs in the UI related to moving to DataHub Cloud by Acryl. """ showAcrylInfo: Boolean! """ @@ -525,3 +535,23 @@ type GlobalViewsSettings { """ defaultView: String } + +""" +Input required to update doc propagation settings. +""" +input UpdateDocPropagationSettingsInput { + """ + The default doc propagation setting for the platform. + """ + docColumnPropagation: Boolean +} + +""" +Global (platform-level) settings related to the doc propagation feature +""" +type DocPropagationSettings { + """ + The default doc propagation setting for the platform. + """ + docColumnPropagation: Boolean +} \ No newline at end of file diff --git a/datahub-graphql-core/src/main/resources/common.graphql b/datahub-graphql-core/src/main/resources/common.graphql new file mode 100644 index 00000000000000..bac56c97f61cf7 --- /dev/null +++ b/datahub-graphql-core/src/main/resources/common.graphql @@ -0,0 +1,49 @@ +""" +Object containing the documentation aspect for an entity +""" +type Documentation { + """ + Structured properties on this entity + """ + documentations: [DocumentationAssociation!]! +} + +""" +Object containing the documentation aspect for an entity +""" +type DocumentationAssociation { + """ + Structured properties on this entity + """ + documentation: String! + + """ + Information about who, why, and how this metadata was applied + """ + attribution: MetadataAttribution +} + +""" +Information about who, why, and how this metadata was applied +""" +type MetadataAttribution { + """ + The time this metadata was applied + """ + time: Long! + + """ + The actor responsible for this metadata application + """ + actor: Entity! + + """ + The source of this metadata application. If propagated, this will be an action. + """ + source: Entity + + """ + Extra details about how this metadata was applied + """ + sourceDetail: [StringMapEntry!] +} diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql index 246ace2fc0f5f1..941a6a28ceb2c7 100644 --- a/datahub-graphql-core/src/main/resources/entity.graphql +++ b/datahub-graphql-core/src/main/resources/entity.graphql @@ -68,6 +68,17 @@ type Query { Fetch a Tag by primary key (urn) """ tag(urn: String!): Tag + + """ + Fetch a View by primary key (urn) + """ + view(urn: String!): DataHubView + + """ + Fetch a Form by primary key (urn) + """ + form(urn: String!): Form + """ Fetch a Role by primary key (urn) """ @@ -890,7 +901,7 @@ type Mutation { """ Creates a filter for a form to apply it to certain entities. Entities that match this filter will have a given form applied to them. - This feature is ONLY supported in Acryl DataHub. + This feature is ONLY supported in DataHub Cloud. """ createDynamicFormAssignment(input: CreateDynamicFormAssignmentInput!): Boolean @@ -1267,6 +1278,11 @@ input RelationshipsInput { The number of results to be returned """ count: Int + + """ + Whether to include soft-deleted, related, entities + """ + includeSoftDelete: Boolean = true } """ @@ -3220,6 +3236,11 @@ type SchemaFieldEntity implements Entity { Business Attribute associated with the field """ businessAttributes: BusinessAttributes + + """ + Documentation aspect for this schema field + """ + documentation: Documentation } """ diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/source/UpsertIngestionSourceResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/source/UpsertIngestionSourceResolverTest.java index b453958e7af474..57d96030a32aaa 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/source/UpsertIngestionSourceResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/source/UpsertIngestionSourceResolverTest.java @@ -7,6 +7,7 @@ import static org.testng.Assert.*; import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.exception.DataHubGraphQLException; import com.linkedin.datahub.graphql.generated.UpdateIngestionSourceConfigInput; import com.linkedin.datahub.graphql.generated.UpdateIngestionSourceInput; import com.linkedin.datahub.graphql.generated.UpdateIngestionSourceScheduleInput; @@ -22,14 +23,17 @@ public class UpsertIngestionSourceResolverTest { - private static final UpdateIngestionSourceInput TEST_INPUT = - new UpdateIngestionSourceInput( - "Test source", - "mysql", - "Test source description", - new UpdateIngestionSourceScheduleInput("* * * * *", "UTC"), - new UpdateIngestionSourceConfigInput( - "my test recipe", "0.8.18", "executor id", false, null)); + private static final UpdateIngestionSourceInput TEST_INPUT = makeInput(); + + private static UpdateIngestionSourceInput makeInput() { + return new UpdateIngestionSourceInput( + "Test source", + "mysql", + "Test source description", + new UpdateIngestionSourceScheduleInput("* * * * *", "UTC"), + new UpdateIngestionSourceConfigInput( + "my test recipe", "0.8.18", "executor id", false, null)); + } @Test public void testGetSuccess() throws Exception { @@ -104,4 +108,54 @@ public void testGetEntityClientException() throws Exception { assertThrows(RuntimeException.class, () -> resolver.get(mockEnv).join()); } + + @Test + public void testUpsertWithInvalidCron() throws Exception { + final UpdateIngestionSourceInput input = makeInput(); + input.setSchedule(new UpdateIngestionSourceScheduleInput("* * * * 123", "UTC")); + + // Create resolver + EntityClient mockClient = Mockito.mock(EntityClient.class); + UpsertIngestionSourceResolver resolver = new UpsertIngestionSourceResolver(mockClient); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("urn"))) + .thenReturn(TEST_INGESTION_SOURCE_URN.toString()); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(DataHubGraphQLException.class, () -> resolver.get(mockEnv).join()); + Mockito.verify(mockClient, Mockito.times(0)).ingestProposal(any(), any(), anyBoolean()); + + input.setSchedule(new UpdateIngestionSourceScheduleInput("null", "UTC")); + assertThrows(DataHubGraphQLException.class, () -> resolver.get(mockEnv).join()); + Mockito.verify(mockClient, Mockito.times(0)).ingestProposal(any(), any(), anyBoolean()); + } + + @Test + public void testUpsertWithInvalidTimezone() throws Exception { + final UpdateIngestionSourceInput input = makeInput(); + input.setSchedule(new UpdateIngestionSourceScheduleInput("* * * * *", "Invalid")); + + // Create resolver + EntityClient mockClient = Mockito.mock(EntityClient.class); + UpsertIngestionSourceResolver resolver = new UpsertIngestionSourceResolver(mockClient); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("urn"))) + .thenReturn(TEST_INGESTION_SOURCE_URN.toString()); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(DataHubGraphQLException.class, () -> resolver.get(mockEnv).join()); + Mockito.verify(mockClient, Mockito.times(0)).ingestProposal(any(), any(), anyBoolean()); + + input.setSchedule(new UpdateIngestionSourceScheduleInput("* * * * *", "America/Los_Angel")); + assertThrows(DataHubGraphQLException.class, () -> resolver.get(mockEnv).join()); + Mockito.verify(mockClient, Mockito.times(0)).ingestProposal(any(), any(), anyBoolean()); + } } diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/load/EntityRelationshipsResultResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/load/EntityRelationshipsResultResolverTest.java new file mode 100644 index 00000000000000..d2799278c1238d --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/load/EntityRelationshipsResultResolverTest.java @@ -0,0 +1,124 @@ +package com.linkedin.datahub.graphql.resolvers.load; + +import static com.linkedin.datahub.graphql.TestUtils.getMockAllowContext; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; + +import com.linkedin.common.EntityRelationship; +import com.linkedin.common.EntityRelationshipArray; +import com.linkedin.common.EntityRelationships; +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.*; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.graph.GraphClient; +import graphql.schema.DataFetchingEnvironment; +import java.net.URISyntaxException; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +public class EntityRelationshipsResultResolverTest { + private final Urn existentUser = Urn.createFromString("urn:li:corpuser:johndoe"); + private final Urn softDeletedUser = Urn.createFromString("urn:li:corpuser:deletedUser"); + + private CorpUser existentEntity; + private CorpUser softDeletedEntity; + + private EntityService _entityService; + private GraphClient _graphClient; + + private EntityRelationshipsResultResolver resolver; + private RelationshipsInput input; + private DataFetchingEnvironment mockEnv; + + public EntityRelationshipsResultResolverTest() throws URISyntaxException {} + + @BeforeMethod + public void setupTest() { + _entityService = mock(EntityService.class); + _graphClient = mock(GraphClient.class); + resolver = new EntityRelationshipsResultResolver(_graphClient, _entityService); + + mockEnv = mock(DataFetchingEnvironment.class); + QueryContext context = getMockAllowContext(); + when(mockEnv.getContext()).thenReturn(context); + + CorpGroup source = new CorpGroup(); + source.setUrn("urn:li:corpGroup:group1"); + when(mockEnv.getSource()).thenReturn(source); + + when(_entityService.exists(any(), eq(Set.of(existentUser, softDeletedUser)), eq(true))) + .thenReturn(Set.of(existentUser, softDeletedUser)); + when(_entityService.exists(any(), eq(Set.of(existentUser, softDeletedUser)), eq(false))) + .thenReturn(Set.of(existentUser)); + + input = new RelationshipsInput(); + input.setStart(0); + input.setCount(10); + input.setDirection(RelationshipDirection.INCOMING); + input.setTypes(List.of("SomeType")); + + EntityRelationships entityRelationships = + new EntityRelationships() + .setStart(0) + .setCount(2) + .setTotal(2) + .setRelationships( + new EntityRelationshipArray( + new EntityRelationship().setEntity(existentUser).setType("SomeType"), + new EntityRelationship().setEntity(softDeletedUser).setType("SomeType"))); + + // always expected INCOMING, and "SomeType" in all tests + when(_graphClient.getRelatedEntities( + eq(source.getUrn()), + eq(input.getTypes()), + same(com.linkedin.metadata.query.filter.RelationshipDirection.INCOMING), + eq(input.getStart()), + eq(input.getCount()), + any())) + .thenReturn(entityRelationships); + + when(mockEnv.getArgument(eq("input"))).thenReturn(input); + + existentEntity = new CorpUser(); + existentEntity.setUrn(existentUser.toString()); + existentEntity.setType(EntityType.CORP_USER); + + softDeletedEntity = new CorpUser(); + softDeletedEntity.setUrn(softDeletedUser.toString()); + softDeletedEntity.setType(EntityType.CORP_USER); + } + + @Test + public void testIncludeSoftDeleted() throws ExecutionException, InterruptedException { + EntityRelationshipsResult expected = new EntityRelationshipsResult(); + expected.setRelationships( + List.of(resultRelationship(existentEntity), resultRelationship(softDeletedEntity))); + expected.setStart(0); + expected.setCount(2); + expected.setTotal(2); + assertEquals(resolver.get(mockEnv).get().toString(), expected.toString()); + } + + @Test + public void testExcludeSoftDeleted() throws ExecutionException, InterruptedException { + input.setIncludeSoftDelete(false); + EntityRelationshipsResult expected = new EntityRelationshipsResult(); + expected.setRelationships(List.of(resultRelationship(existentEntity))); + expected.setStart(0); + expected.setCount(1); + expected.setTotal(1); + assertEquals(resolver.get(mockEnv).get().toString(), expected.toString()); + } + + private com.linkedin.datahub.graphql.generated.EntityRelationship resultRelationship( + Entity entity) { + return new com.linkedin.datahub.graphql.generated.EntityRelationship( + "SomeType", RelationshipDirection.INCOMING, entity, null); + } +} diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/ClearSystemMetadataServiceStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/ClearSystemMetadataServiceStep.java new file mode 100644 index 00000000000000..889d2f0a79edf6 --- /dev/null +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/ClearSystemMetadataServiceStep.java @@ -0,0 +1,56 @@ +package com.linkedin.datahub.upgrade.common.steps; + +import com.linkedin.datahub.upgrade.UpgradeContext; +import com.linkedin.datahub.upgrade.UpgradeStep; +import com.linkedin.datahub.upgrade.UpgradeStepResult; +import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult; +import com.linkedin.datahub.upgrade.nocode.NoCodeUpgrade; +import com.linkedin.metadata.systemmetadata.SystemMetadataService; +import java.util.function.Function; + +public class ClearSystemMetadataServiceStep implements UpgradeStep { + + private final SystemMetadataService _systemMetadataService; + private final boolean _alwaysRun; + + public ClearSystemMetadataServiceStep( + final SystemMetadataService systemMetadataService, final boolean alwaysRun) { + _systemMetadataService = systemMetadataService; + _alwaysRun = alwaysRun; + } + + @Override + public String id() { + return "ClearSystemMetadataServiceStep"; + } + + @Override + public boolean skip(UpgradeContext context) { + if (_alwaysRun) { + return false; + } + if (context.parsedArgs().containsKey(NoCodeUpgrade.CLEAN_ARG_NAME)) { + return false; + } + context.report().addLine("Cleanup has not been requested."); + return true; + } + + @Override + public int retryCount() { + return 1; + } + + @Override + public Function executable() { + return (context) -> { + try { + _systemMetadataService.clear(); + } catch (Exception e) { + context.report().addLine("Failed to clear system metadata service", e); + return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.FAILED); + } + return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.SUCCEEDED); + }; + } +} diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/GMSDisableWriteModeStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/GMSDisableWriteModeStep.java index 42b3c6b3ccc490..a80adabc60e743 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/GMSDisableWriteModeStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/GMSDisableWriteModeStep.java @@ -13,7 +13,7 @@ @RequiredArgsConstructor public class GMSDisableWriteModeStep implements UpgradeStep { - private final SystemEntityClient entityClient; + private final SystemEntityClient systemEntityClient; @Override public String id() { @@ -29,7 +29,7 @@ public int retryCount() { public Function executable() { return (context) -> { try { - entityClient.setWritable(context.opContext(), false); + systemEntityClient.setWritable(context.opContext(), false); } catch (Exception e) { log.error("Failed to turn write mode off in GMS", e); context.report().addLine("Failed to turn write mode off in GMS"); diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/GMSEnableWriteModeStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/GMSEnableWriteModeStep.java index c43644fda7ddf5..a6b2942fabffd7 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/GMSEnableWriteModeStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/GMSEnableWriteModeStep.java @@ -12,7 +12,7 @@ @Slf4j @RequiredArgsConstructor public class GMSEnableWriteModeStep implements UpgradeStep { - private final SystemEntityClient entityClient; + private final SystemEntityClient systemEntityClient; @Override public String id() { @@ -28,7 +28,7 @@ public int retryCount() { public Function executable() { return (context) -> { try { - entityClient.setWritable(context.opContext(), true); + systemEntityClient.setWritable(context.opContext(), true); } catch (Exception e) { log.error("Failed to turn write mode back on in GMS", e); context.report().addLine("Failed to turn write mode back on in GMS"); diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/RestoreBackupConfig.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/RestoreBackupConfig.java index ec6e5a4a8f04d1..b4eafb4ad3d241 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/RestoreBackupConfig.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/RestoreBackupConfig.java @@ -6,6 +6,7 @@ import com.linkedin.metadata.graph.GraphService; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.search.EntitySearchService; +import com.linkedin.metadata.systemmetadata.SystemMetadataService; import io.ebean.Database; import javax.annotation.Nonnull; import lombok.extern.slf4j.Slf4j; @@ -26,8 +27,9 @@ public class RestoreBackupConfig { "ebeanServer", "entityService", "systemEntityClient", - "graphService", + "systemMetadataService", "searchService", + "graphService", "entityRegistry" }) @ConditionalOnProperty(name = "entityService.impl", havingValue = "ebean", matchIfMissing = true) @@ -35,13 +37,23 @@ public class RestoreBackupConfig { public RestoreBackup createInstance() { final Database ebeanServer = applicationContext.getBean(Database.class); final EntityService entityService = applicationContext.getBean(EntityService.class); - final SystemEntityClient entityClient = applicationContext.getBean(SystemEntityClient.class); - final GraphService graphClient = applicationContext.getBean(GraphService.class); - final EntitySearchService searchClient = applicationContext.getBean(EntitySearchService.class); + final SystemEntityClient systemEntityClient = + applicationContext.getBean(SystemEntityClient.class); + final SystemMetadataService systemMetadataService = + applicationContext.getBean(SystemMetadataService.class); + final EntitySearchService entitySearchService = + applicationContext.getBean(EntitySearchService.class); + final GraphService graphService = applicationContext.getBean(GraphService.class); final EntityRegistry entityRegistry = applicationContext.getBean(EntityRegistry.class); return new RestoreBackup( - ebeanServer, entityService, entityRegistry, entityClient, graphClient, searchClient); + ebeanServer, + entityService, + entityRegistry, + systemEntityClient, + systemMetadataService, + entitySearchService, + graphService); } @Bean(name = "restoreBackup") @@ -49,6 +61,6 @@ public RestoreBackup createInstance() { @Nonnull public RestoreBackup createNotImplInstance() { log.warn("restoreIndices is not supported for cassandra!"); - return new RestoreBackup(null, null, null, null, null, null); + return new RestoreBackup(null, null, null, null, null, null, null); } } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/RestoreIndicesConfig.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/RestoreIndicesConfig.java index 008bdf5cfac388..949b75edaa6ba0 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/RestoreIndicesConfig.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/RestoreIndicesConfig.java @@ -4,6 +4,7 @@ import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.graph.GraphService; import com.linkedin.metadata.search.EntitySearchService; +import com.linkedin.metadata.systemmetadata.SystemMetadataService; import io.ebean.Database; import javax.annotation.Nonnull; import lombok.extern.slf4j.Slf4j; @@ -20,17 +21,26 @@ public class RestoreIndicesConfig { @Autowired ApplicationContext applicationContext; @Bean(name = "restoreIndices") - @DependsOn({"ebeanServer", "entityService", "searchService", "graphService"}) + @DependsOn({ + "ebeanServer", + "entityService", + "systemMetadataService", + "searchService", + "graphService" + }) @ConditionalOnProperty(name = "entityService.impl", havingValue = "ebean", matchIfMissing = true) @Nonnull public RestoreIndices createInstance() { final Database ebeanServer = applicationContext.getBean(Database.class); final EntityService entityService = applicationContext.getBean(EntityService.class); + final SystemMetadataService systemMetadataService = + applicationContext.getBean(SystemMetadataService.class); final EntitySearchService entitySearchService = applicationContext.getBean(EntitySearchService.class); final GraphService graphService = applicationContext.getBean(GraphService.class); - return new RestoreIndices(ebeanServer, entityService, entitySearchService, graphService); + return new RestoreIndices( + ebeanServer, entityService, systemMetadataService, entitySearchService, graphService); } @Bean(name = "restoreIndices") @@ -38,6 +48,6 @@ public RestoreIndices createInstance() { @Nonnull public RestoreIndices createNotImplInstance() { log.warn("restoreIndices is not supported for cassandra!"); - return new RestoreIndices(null, null, null, null); + return new RestoreIndices(null, null, null, null, null); } } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restorebackup/RestoreBackup.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restorebackup/RestoreBackup.java index bcaeaa34e8936d..7496655e581b09 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restorebackup/RestoreBackup.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restorebackup/RestoreBackup.java @@ -6,6 +6,7 @@ import com.linkedin.datahub.upgrade.UpgradeStep; import com.linkedin.datahub.upgrade.common.steps.ClearGraphServiceStep; import com.linkedin.datahub.upgrade.common.steps.ClearSearchServiceStep; +import com.linkedin.datahub.upgrade.common.steps.ClearSystemMetadataServiceStep; import com.linkedin.datahub.upgrade.common.steps.GMSDisableWriteModeStep; import com.linkedin.datahub.upgrade.common.steps.GMSEnableWriteModeStep; import com.linkedin.entity.client.SystemEntityClient; @@ -13,6 +14,7 @@ import com.linkedin.metadata.graph.GraphService; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.search.EntitySearchService; +import com.linkedin.metadata.systemmetadata.SystemMetadataService; import io.ebean.Database; import java.util.ArrayList; import java.util.List; @@ -26,13 +28,20 @@ public RestoreBackup( @Nullable final Database server, final EntityService entityService, final EntityRegistry entityRegistry, - final SystemEntityClient entityClient, - final GraphService graphClient, - final EntitySearchService searchClient) { + final SystemEntityClient systemEntityClient, + final SystemMetadataService systemMetadataService, + final EntitySearchService entitySearchService, + final GraphService graphClient) { if (server != null) { _steps = buildSteps( - server, entityService, entityRegistry, entityClient, graphClient, searchClient); + server, + entityService, + entityRegistry, + systemEntityClient, + systemMetadataService, + entitySearchService, + graphClient); } else { _steps = List.of(); } @@ -52,16 +61,18 @@ private List buildSteps( final Database server, final EntityService entityService, final EntityRegistry entityRegistry, - final SystemEntityClient entityClient, - final GraphService graphClient, - final EntitySearchService searchClient) { + final SystemEntityClient systemEntityClient, + final SystemMetadataService systemMetadataService, + final EntitySearchService entitySearchService, + final GraphService graphClient) { final List steps = new ArrayList<>(); - steps.add(new GMSDisableWriteModeStep(entityClient)); - steps.add(new ClearSearchServiceStep(searchClient, true)); + steps.add(new GMSDisableWriteModeStep(systemEntityClient)); + steps.add(new ClearSystemMetadataServiceStep(systemMetadataService, true)); + steps.add(new ClearSearchServiceStep(entitySearchService, true)); steps.add(new ClearGraphServiceStep(graphClient, true)); steps.add(new ClearAspectV2TableStep(server)); steps.add(new RestoreStorageStep(entityService, entityRegistry)); - steps.add(new GMSEnableWriteModeStep(entityClient)); + steps.add(new GMSEnableWriteModeStep(systemEntityClient)); return steps; } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restoreindices/RestoreIndices.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restoreindices/RestoreIndices.java index 9bc42e23a99746..9d239a56224862 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restoreindices/RestoreIndices.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restoreindices/RestoreIndices.java @@ -6,9 +6,11 @@ import com.linkedin.datahub.upgrade.UpgradeStep; import com.linkedin.datahub.upgrade.common.steps.ClearGraphServiceStep; import com.linkedin.datahub.upgrade.common.steps.ClearSearchServiceStep; +import com.linkedin.datahub.upgrade.common.steps.ClearSystemMetadataServiceStep; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.graph.GraphService; import com.linkedin.metadata.search.EntitySearchService; +import com.linkedin.metadata.systemmetadata.SystemMetadataService; import io.ebean.Database; import java.util.ArrayList; import java.util.List; @@ -32,10 +34,13 @@ public class RestoreIndices implements Upgrade { public RestoreIndices( @Nullable final Database server, final EntityService entityService, + final SystemMetadataService systemMetadataService, final EntitySearchService entitySearchService, final GraphService graphService) { if (server != null) { - _steps = buildSteps(server, entityService, entitySearchService, graphService); + _steps = + buildSteps( + server, entityService, systemMetadataService, entitySearchService, graphService); } else { _steps = List.of(); } @@ -54,9 +59,11 @@ public List steps() { private List buildSteps( final Database server, final EntityService entityService, + final SystemMetadataService systemMetadataService, final EntitySearchService entitySearchService, final GraphService graphService) { final List steps = new ArrayList<>(); + steps.add(new ClearSystemMetadataServiceStep(systemMetadataService, false)); steps.add(new ClearSearchServiceStep(entitySearchService, false)); steps.add(new ClearGraphServiceStep(graphService, false)); steps.add(new SendMAEStep(server, entityService)); diff --git a/datahub-web-react/src/App.tsx b/datahub-web-react/src/App.tsx index 5be31528fe780f..2fdd7c8ed68004 100644 --- a/datahub-web-react/src/App.tsx +++ b/datahub-web-react/src/App.tsx @@ -1,6 +1,5 @@ import React from 'react'; import Cookies from 'js-cookie'; -import { message } from 'antd'; import { BrowserRouter as Router } from 'react-router-dom'; import { ApolloClient, ApolloProvider, createHttpLink, InMemoryCache, ServerError } from '@apollo/client'; import { onError } from '@apollo/client/link/error'; @@ -21,7 +20,7 @@ import { useCustomTheme } from './customThemeContext'; const httpLink = createHttpLink({ uri: '/api/v2/graphql' }); const errorLink = onError((error) => { - const { networkError, graphQLErrors } = error; + const { networkError } = error; if (networkError) { const serverError = networkError as ServerError; if (serverError.statusCode === ErrorCodes.Unauthorized) { @@ -31,13 +30,14 @@ const errorLink = onError((error) => { window.location.replace(`${PageRoutes.AUTHENTICATE}?redirect_uri=${encodeURIComponent(currentPath)}`); } } - if (graphQLErrors && graphQLErrors.length) { - const firstError = graphQLErrors[0]; - const { extensions } = firstError; - const errorCode = extensions && (extensions.code as number); - // Fallback in case the calling component does not handle. - message.error(`${firstError.message} (code ${errorCode})`, 3); - } + // Disabled behavior for now -> Components are expected to handle their errors. + // if (graphQLErrors && graphQLErrors.length) { + // const firstError = graphQLErrors[0]; + // const { extensions } = firstError; + // const errorCode = extensions && (extensions.code as number); + // // Fallback in case the calling component does not handle. + // message.error(`${firstError.message} (code ${errorCode})`, 3); // TODO: Decide if we want this back. + // } }); const client = new ApolloClient({ diff --git a/datahub-web-react/src/app/ProtectedRoutes.tsx b/datahub-web-react/src/app/ProtectedRoutes.tsx index 0e4a1a260f5532..d975e6d4d99c2d 100644 --- a/datahub-web-react/src/app/ProtectedRoutes.tsx +++ b/datahub-web-react/src/app/ProtectedRoutes.tsx @@ -1,15 +1,26 @@ -import React from 'react'; -import { Switch, Route } from 'react-router-dom'; +import React, { useEffect } from 'react'; +import { Switch, Route, useLocation, useHistory } from 'react-router-dom'; import { Layout } from 'antd'; import { HomePage } from './home/HomePage'; import { SearchRoutes } from './SearchRoutes'; import EmbedRoutes from './EmbedRoutes'; -import { PageRoutes } from '../conf/Global'; +import { NEW_ROUTE_MAP, PageRoutes } from '../conf/Global'; +import { getRedirectUrl } from '../conf/utils'; /** * Container for all views behind an authentication wall. */ export const ProtectedRoutes = (): JSX.Element => { + const location = useLocation(); + const history = useHistory(); + + useEffect(() => { + if (location.pathname.indexOf('/Validation') !== -1) { + history.replace(getRedirectUrl(NEW_ROUTE_MAP)); + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [location]); + return ( diff --git a/datahub-web-react/src/app/context/UserContextProvider.tsx b/datahub-web-react/src/app/context/UserContextProvider.tsx index 3bcff15cc27485..66593d346f3df4 100644 --- a/datahub-web-react/src/app/context/UserContextProvider.tsx +++ b/datahub-web-react/src/app/context/UserContextProvider.tsx @@ -127,6 +127,7 @@ const UserContextProvider = ({ children }: { children: React.ReactNode }) => { return ( { }, }, { - name: 'Validation', + name: 'Quality', component: ValidationsTab, display: { visible: (_, _1) => true, enabled: (_, dataset: GetDatasetQuery) => { - return ( - (dataset?.dataset?.assertions?.total || 0) > 0 || dataset?.dataset?.testResults !== null - ); + return (dataset?.dataset?.assertions?.total || 0) > 0; + }, + }, + }, + { + name: 'Governance', + component: GovernanceTab, + display: { + visible: (_, _1) => true, + enabled: (_, dataset: GetDatasetQuery) => { + return dataset?.dataset?.testResults !== null; }, }, }, diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityHealth.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityHealth.tsx index 30713afa888b84..ad14d92a3915ae 100644 --- a/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityHealth.tsx +++ b/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityHealth.tsx @@ -24,7 +24,7 @@ export const EntityHealth = ({ health, baseUrl, fontSize, tooltipPlacement }: Pr return ( <> {(unhealthy && ( - + {icon} diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Governance/GovernanceTab.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Governance/GovernanceTab.tsx new file mode 100644 index 00000000000000..213716ed501c5b --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Governance/GovernanceTab.tsx @@ -0,0 +1,85 @@ +import React, { useEffect } from 'react'; +import { Button } from 'antd'; +import { useHistory, useLocation } from 'react-router'; +import styled from 'styled-components'; +import { FileDoneOutlined } from '@ant-design/icons'; +import { useEntityData } from '../../../EntityContext'; +import { TestResults } from './TestResults'; +import TabToolbar from '../../../components/styled/TabToolbar'; +import { ANTD_GRAY } from '../../../constants'; +import { useGetValidationsTab } from '../Validations/useGetValidationsTab'; + +const TabTitle = styled.span` + margin-left: 4px; +`; + +const TabButton = styled(Button)<{ selected: boolean }>` + background-color: ${(props) => (props.selected && ANTD_GRAY[3]) || 'none'}; + margin-left: 4px; +`; + +enum TabPaths { + TESTS = 'Tests', +} + +const DEFAULT_TAB = TabPaths.TESTS; + +/** + * Component used for rendering the Entity Governance Tab. + */ +export const GovernanceTab = () => { + const { entityData } = useEntityData(); + const history = useHistory(); + const { pathname } = useLocation(); + + const passingTests = (entityData as any)?.testResults?.passing || []; + const maybeFailingTests = (entityData as any)?.testResults?.failing || []; + const totalTests = maybeFailingTests.length + passingTests.length; + + const { selectedTab, basePath } = useGetValidationsTab(pathname, Object.values(TabPaths)); + + // If no tab was selected, select a default tab. + useEffect(() => { + if (!selectedTab) { + // Route to the default tab. + history.replace(`${basePath}/${DEFAULT_TAB}`); + } + }, [selectedTab, basePath, history]); + + /** + * The top-level Toolbar tabs to display. + */ + const tabs = [ + { + title: ( + <> + + Tests ({totalTests}) + + ), + path: TabPaths.TESTS, + disabled: totalTests === 0, + content: , + }, + ]; + + return ( + <> + +
+ {tabs.map((tab) => ( + history.replace(`${basePath}/${tab.path}`)} + > + {tab.title} + + ))} +
+ + {tabs.filter((tab) => tab.path === selectedTab).map((tab) => tab.content)} + + ); +}; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/TestResults.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Governance/TestResults.tsx similarity index 100% rename from datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/TestResults.tsx rename to datahub-web-react/src/app/entity/shared/tabs/Dataset/Governance/TestResults.tsx diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/TestResultsList.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Governance/TestResultsList.tsx similarity index 100% rename from datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/TestResultsList.tsx rename to datahub-web-react/src/app/entity/shared/tabs/Dataset/Governance/TestResultsList.tsx diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/TestResultsSummary.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Governance/TestResultsSummary.tsx similarity index 100% rename from datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/TestResultsSummary.tsx rename to datahub-web-react/src/app/entity/shared/tabs/Dataset/Governance/TestResultsSummary.tsx diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/testUtils.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Governance/testUtils.tsx similarity index 100% rename from datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/testUtils.tsx rename to datahub-web-react/src/app/entity/shared/tabs/Dataset/Governance/testUtils.tsx diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/DatasetAssertionsList.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/DatasetAssertionsList.tsx index 4d0e475d5dad14..bfcb30b6c5e7ac 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/DatasetAssertionsList.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/DatasetAssertionsList.tsx @@ -188,7 +188,7 @@ export const DatasetAssertionsList = ({ to={`${entityRegistry.getEntityUrl( EntityType.Dataset, entityData.urn, - )}/Validation/Data Contract`} + )}/Quality/Data Contract`} style={{ color: REDESIGN_COLORS.BLUE }} > view @@ -200,7 +200,7 @@ export const DatasetAssertionsList = ({ to={`${entityRegistry.getEntityUrl( EntityType.Dataset, entityData.urn, - )}/Validation/Data Contract`} + )}/Quality/Data Contract`} > diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/ValidationsTab.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/ValidationsTab.tsx index 92af9bfc2b567b..006823db53fd44 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/ValidationsTab.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/ValidationsTab.tsx @@ -2,9 +2,8 @@ import React, { useEffect } from 'react'; import { Button } from 'antd'; import { useHistory, useLocation } from 'react-router'; import styled from 'styled-components'; -import { AuditOutlined, FileDoneOutlined, FileProtectOutlined } from '@ant-design/icons'; +import { AuditOutlined, FileProtectOutlined } from '@ant-design/icons'; import { useEntityData } from '../../../EntityContext'; -import { TestResults } from './TestResults'; import { Assertions } from './Assertions'; import TabToolbar from '../../../components/styled/TabToolbar'; import { useGetValidationsTab } from './useGetValidationsTab'; @@ -22,8 +21,7 @@ const TabButton = styled(Button)<{ selected: boolean }>` `; enum TabPaths { - ASSERTIONS = 'Assertions', - TESTS = 'Tests', + ASSERTIONS = 'List', DATA_CONTRACT = 'Data Contract', } @@ -39,9 +37,6 @@ export const ValidationsTab = () => { const appConfig = useAppConfig(); const totalAssertions = (entityData as any)?.assertions?.total; - const passingTests = (entityData as any)?.testResults?.passing || []; - const maybeFailingTests = (entityData as any)?.testResults?.failing || []; - const totalTests = maybeFailingTests.length + passingTests.length; const { selectedTab, basePath } = useGetValidationsTab(pathname, Object.values(TabPaths)); @@ -68,17 +63,6 @@ export const ValidationsTab = () => { disabled: totalAssertions === 0, content: , }, - { - title: ( - <> - - Tests ({totalTests}) - - ), - path: TabPaths.TESTS, - disabled: totalTests === 0, - content: , - }, ]; if (appConfig.config.featureFlags?.dataContractsEnabled) { diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/__tests__/useGetValidationsTab.test.ts b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/__tests__/useGetValidationsTab.test.ts index 52689a225eae15..f65c337215ed90 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/__tests__/useGetValidationsTab.test.ts +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/__tests__/useGetValidationsTab.test.ts @@ -2,37 +2,37 @@ import { useGetValidationsTab } from '../useGetValidationsTab'; describe('useGetValidationsTab', () => { it('should correctly extract valid tab', () => { - const pathname = '/dataset/urn:li:abc/Validation/Assertions'; - const tabNames = ['Assertions']; + const pathname = '/dataset/urn:li:abc/Quality/List'; + const tabNames = ['List']; const res = useGetValidationsTab(pathname, tabNames); - expect(res.selectedTab).toEqual('Assertions'); - expect(res.basePath).toEqual('/dataset/urn:li:abc/Validation'); + expect(res.selectedTab).toEqual('List'); + expect(res.basePath).toEqual('/dataset/urn:li:abc/Quality'); }); it('should extract undefined for invalid tab', () => { - const pathname = '/dataset/urn:li:abc/Validation/Assertions'; + const pathname = '/dataset/urn:li:abc/Quality/Assertions'; const tabNames = ['Tests']; const res = useGetValidationsTab(pathname, tabNames); expect(res.selectedTab).toBeUndefined(); - expect(res.basePath).toEqual('/dataset/urn:li:abc/Validation'); + expect(res.basePath).toEqual('/dataset/urn:li:abc/Quality'); }); it('should extract undefined for missing tab', () => { - const pathname = '/dataset/urn:li:abc/Validation'; + const pathname = '/dataset/urn:li:abc/Quality'; const tabNames = ['Tests']; const res = useGetValidationsTab(pathname, tabNames); expect(res.selectedTab).toBeUndefined(); - expect(res.basePath).toEqual('/dataset/urn:li:abc/Validation'); + expect(res.basePath).toEqual('/dataset/urn:li:abc/Quality'); }); it('should handle trailing slashes', () => { - let pathname = '/dataset/urn:li:abc/Validation/Assertions/'; + let pathname = '/dataset/urn:li:abc/Quality/Assertions/'; let tabNames = ['Assertions']; let res = useGetValidationsTab(pathname, tabNames); expect(res.selectedTab).toEqual('Assertions'); - expect(res.basePath).toEqual('/dataset/urn:li:abc/Validation'); + expect(res.basePath).toEqual('/dataset/urn:li:abc/Quality'); - pathname = '/dataset/urn:li:abc/Validation/'; + pathname = '/dataset/urn:li:abc/Quality/'; tabNames = ['Assertions']; res = useGetValidationsTab(pathname, tabNames); expect(res.selectedTab).toBeUndefined(); - expect(res.basePath).toEqual('/dataset/urn:li:abc/Validation'); + expect(res.basePath).toEqual('/dataset/urn:li:abc/Quality'); }); }); diff --git a/datahub-web-react/src/app/home/AcrylDemoBanner.tsx b/datahub-web-react/src/app/home/AcrylDemoBanner.tsx index 0a85c0c3d7f6c2..debcaab879ba94 100644 --- a/datahub-web-react/src/app/home/AcrylDemoBanner.tsx +++ b/datahub-web-react/src/app/home/AcrylDemoBanner.tsx @@ -42,7 +42,7 @@ export default function AcrylDemoBanner() { - Schedule a Demo of Managed DataHub + Schedule a Demo of DataHub Cloud DataHub is already the industry's #1 Open Source Data Catalog.{' '} Schedule a demo {' '} - of Acryl DataHub to see the advanced features that take it to the next level or purchase Acryl Cloud - on{' '} + of DataHub Cloud to see the advanced features that take it to the next level or purchase DataHub + Cloud on{' '} { * Determines which view should be visible: ingestion sources or secrets. */ const me = useUserContext(); - const { config } = useAppConfig(); + const { config, loaded } = useAppConfig(); const isIngestionEnabled = config?.managedIngestionConfig.enabled; const showIngestionTab = isIngestionEnabled && me && me.platformPrivileges?.manageIngestion; const showSecretsTab = isIngestionEnabled && me && me.platformPrivileges?.manageSecrets; - const defaultTab = showIngestionTab ? TabType.Sources : TabType.Secrets; - const [selectedTab, setSelectedTab] = useState(defaultTab); + const [selectedTab, setSelectedTab] = useState(TabType.Sources); + + // defaultTab might not be calculated correctly on mount, if `config` or `me` haven't been loaded yet + useEffect(() => { + if (loaded && me.loaded && !showIngestionTab && selectedTab === TabType.Sources) { + setSelectedTab(TabType.Secrets); + } + }, [loaded, me.loaded, showIngestionTab, selectedTab]); const onClickTab = (newTab: string) => { setSelectedTab(TabType[newTab]); diff --git a/datahub-web-react/src/app/ingest/source/IngestionSourceTableColumns.tsx b/datahub-web-react/src/app/ingest/source/IngestionSourceTableColumns.tsx index 155e75f1895f53..4b7fb472226172 100644 --- a/datahub-web-react/src/app/ingest/source/IngestionSourceTableColumns.tsx +++ b/datahub-web-react/src/app/ingest/source/IngestionSourceTableColumns.tsx @@ -106,7 +106,13 @@ export function LastExecutionColumn(time: any) { } export function ScheduleColumn(schedule: any, record: any) { - const tooltip = schedule && `Runs ${cronstrue.toString(schedule).toLowerCase()} (${record.timezone})`; + let tooltip: string; + try { + tooltip = schedule && `Runs ${cronstrue.toString(schedule).toLowerCase()} (${record.timezone})`; + } catch (e) { + tooltip = 'Invalid cron schedule'; + console.debug('Error parsing cron schedule', e); + } return ( {schedule || 'None'} diff --git a/datahub-web-react/src/app/settings/SettingsPage.tsx b/datahub-web-react/src/app/settings/SettingsPage.tsx index e0a15c73a626d0..24bcd17ca7f9c0 100644 --- a/datahub-web-react/src/app/settings/SettingsPage.tsx +++ b/datahub-web-react/src/app/settings/SettingsPage.tsx @@ -8,6 +8,7 @@ import { FilterOutlined, TeamOutlined, PushpinOutlined, + ControlOutlined, } from '@ant-design/icons'; import { Redirect, Route, useHistory, useLocation, useRouteMatch, Switch } from 'react-router'; import styled from 'styled-components'; @@ -17,11 +18,17 @@ import { ManagePermissions } from '../permissions/ManagePermissions'; import { useAppConfig } from '../useAppConfig'; import { AccessTokens } from './AccessTokens'; import { Preferences } from './Preferences'; +import { Features } from './features/Features'; import { ManageViews } from '../entity/view/ManageViews'; import { useUserContext } from '../context/useUserContext'; import { ManageOwnership } from '../entity/ownership/ManageOwnership'; import ManagePosts from './posts/ManagePosts'; +const MenuItem = styled(Menu.Item)` + display: flex; + align-items: center; +`; + const PageContainer = styled.div` display: flex; overflow: auto; @@ -59,6 +66,17 @@ const ItemTitle = styled.span` const menuStyle = { width: 256, 'margin-top': 8, overflow: 'hidden auto' }; +const NewTag = styled.span` + padding: 4px 8px; + margin-left: 8px; + + border-radius: 24px; + background: #f1fbfe; + + color: #09739a; + font-size: 12px; +`; + /** * URL Paths for each settings page. */ @@ -70,6 +88,7 @@ const PATHS = [ { path: 'views', content: }, { path: 'ownership', content: }, { path: 'posts', content: }, + { path: 'features', content: }, ]; /** @@ -80,6 +99,7 @@ const DEFAULT_PATH = PATHS[0]; export const SettingsPage = () => { const { path, url } = useRouteMatch(); const { pathname } = useLocation(); + const history = useHistory(); const subRoutes = PATHS.map((p) => p.path.replace('/', '')); const currPathName = pathname.replace(path, ''); @@ -101,6 +121,7 @@ export const SettingsPage = () => { const showViews = isViewsEnabled || false; const showOwnershipTypes = me && me?.platformPrivileges?.manageOwnershipTypes; const showHomePagePosts = me && me?.platformPrivileges?.manageGlobalAnnouncements && !readOnlyModeEnabled; + const showFeatures = true; // TODO: Add feature flag for this return ( @@ -143,6 +164,13 @@ export const SettingsPage = () => { )} {(showViews || showOwnershipTypes || showHomePagePosts) && ( + {showFeatures && ( + + + Features + New! + + )} {showViews && ( My Views diff --git a/datahub-web-react/src/app/settings/features/Feature.tsx b/datahub-web-react/src/app/settings/features/Feature.tsx new file mode 100644 index 00000000000000..2c090aae696f88 --- /dev/null +++ b/datahub-web-react/src/app/settings/features/Feature.tsx @@ -0,0 +1,179 @@ +import React from 'react'; + +import styled from 'styled-components'; + +import { Divider, Typography, Switch, Card, Button, Tooltip } from 'antd'; +import { ArrowRightOutlined } from '@ant-design/icons'; +import { ANTD_GRAY } from '../../entity/shared/constants'; + +const Title = styled(Typography.Title)` + && { + margin-bottom: 8px; + } +`; + +const FeatureRow = styled.div` + display: flex; + align-items: flex-start; + justify-content: space-between; +`; + +const FeatureOptionRow = styled.div` + display: flex; + justify-content: space-between; + + &:not(:last-child) { + margin-bottom: 8px; + } +`; + +const SettingsOptionRow = styled.div` + display: flex; + justify-content: space-between; + align-items: center; + padding: 0 16px; + + &:not(:last-child) { + margin-bottom: 8px; + } +`; + +const DescriptionText = styled(Typography.Text)` + color: ${ANTD_GRAY[7]}; + font-size: 11px; +`; + +const SettingTitle = styled.div` + display: flex; + align-items: center; + gap: 8px; + font-size: 14px; + margin-bottom: 4px; +`; + +const OptionTitle = styled(Typography.Text)` + display: flex; + align-items: center; + gap: 8px; + font-size: 12px; +`; + +const learnMoreLinkStyle = { + flex: 1, + display: 'flex', + alignItems: 'center', + gap: '8px', + color: '#1890FF', + fontSize: '12px', + cursor: 'pointer', +}; + +const NewTag = styled.div` + padding: 4px 8px; + + border-radius: 24px; + background: #f1fbfe; + + color: #09739a; + font-size: 12px; +`; + +const DataHubOnlyTag = styled.div` + padding: 2px 8px; + + border-radius: 24px; + background: #c9fff2; + + color: #50a494; + font-size: 12px; +`; + +export interface FeatureType { + key: string; + title: string; + description: string; + settings: Array<{ + key: string; + title: string; + isAvailable: boolean; + buttonText: string; + onClick?: () => void; + }>; + options: Array<{ + key: string; + title: string; + description: string; + isAvailable: boolean; + checked: boolean; + onChange?: (checked: boolean) => void; + }>; + isNew: boolean; + learnMoreLink?: string; +} + +export const Feature = ({ key, title, description, settings, options, isNew, learnMoreLink }: FeatureType) => ( + + +
+ + + {title} + + {isNew && New!} + +
+ {description} +
+
+
+ {learnMoreLink && ( + + Learn more + + )} +
+ + + {settings.map((option) => ( + <> + + + + {option.title} + + + + + + + + ))} + + {options.map((option, index) => ( + <> + + + + {option.title} + {!option.isAvailable && ( + Only available on DataHub Cloud + )} + +
+ {option.description} +
+ + (option.onChange ? option.onChange(checked) : null)} + disabled={!option.isAvailable} + /> + + {index !== options.length - 1 && } + + ))} + + +); diff --git a/datahub-web-react/src/app/settings/features/Features.tsx b/datahub-web-react/src/app/settings/features/Features.tsx new file mode 100644 index 00000000000000..ee8d7c628c1eff --- /dev/null +++ b/datahub-web-react/src/app/settings/features/Features.tsx @@ -0,0 +1,110 @@ +import React from 'react'; + +import styled from 'styled-components'; + +import { Divider, Typography } from 'antd'; +import { v4 as uuidv4 } from 'uuid'; + +import { Feature, FeatureType } from './Feature'; + +import { useGetDocPropagationSettings, useUpdateDocPropagationSettings } from './useDocPropagationSettings'; + +const Page = styled.div` + width: 100%; + display: flex; + justify-content: center; +`; + +const SourceContainer = styled.div` + width: 80%; + padding-top: 20px; + padding-right: 40px; + padding-left: 40px; +`; +const Container = styled.div` + padding-top: 0px; +`; + +const Title = styled(Typography.Title)` + && { + margin-bottom: 8px; + } +`; + +export const Features = () => { + /* + * Note: When adding new features, make sure to update the features array below + * and create a hook file for the new feature in the same directory + */ + + // Hooks to get and update the document propagation settings + const { isColPropagateChecked, setIsColPropagateChecked } = useGetDocPropagationSettings(); + const { updateDocPropagation } = useUpdateDocPropagationSettings(); + + // Features to display + const features: FeatureType[] = [ + { + key: uuidv4(), + title: 'Documentation Propagation', + description: 'Automatically propagate documentation from upstream to downstream columns and assets.', + settings: [ + { + key: uuidv4(), + title: 'Rollback Propagation Changes', + isAvailable: false, + buttonText: 'Rollback', + }, + { + key: uuidv4(), + title: 'Backfill existing documentation from upstream to downstream columns/assets', + isAvailable: false, + buttonText: 'Initialize', + }, + ], + options: [ + { + key: uuidv4(), + title: 'Column Level Propagation', + description: + 'Propagate new documentation from upstream to downstream columns based on column-level lineage relationships.', + isAvailable: true, + checked: isColPropagateChecked, + onChange: (checked: boolean) => { + setIsColPropagateChecked(checked); + updateDocPropagation(checked); + }, + }, + { + key: uuidv4(), + title: 'Asset Level Propagation', + description: + 'Propagate new documentation from upstream to downstream assets based on data lineage relationships.', + isAvailable: false, + checked: false, + }, + ], + isNew: true, + learnMoreLink: 'https://datahubproject.io/docs/automations/doc-propagation', + }, + ]; + + // Render + return ( + + + +
+ Features + + Explore and configure specific features + +
+ + + {features.map((feature) => ( + + ))} + + + ); +}; diff --git a/datahub-web-react/src/app/settings/features/useDocPropagationSettings.ts b/datahub-web-react/src/app/settings/features/useDocPropagationSettings.ts new file mode 100644 index 00000000000000..c93b610cff9d1b --- /dev/null +++ b/datahub-web-react/src/app/settings/features/useDocPropagationSettings.ts @@ -0,0 +1,50 @@ +import { useEffect, useState } from 'react'; + +import { message } from 'antd'; + +import { + useGetDocPropagationSettingsQuery, + useUpdateDocPropagationSettingsMutation, +} from '../../../graphql/app.generated'; + +// Hook to get the document propagation settings & manage state +export const useGetDocPropagationSettings = () => { + const { data, refetch } = useGetDocPropagationSettingsQuery(); + const [isColPropagateChecked, setIsColPropagateChecked] = useState(false); + + useEffect(() => { + const docPropSetting = data?.docPropagationSettings?.docColumnPropagation; + if (docPropSetting !== undefined) setIsColPropagateChecked(!!docPropSetting); + }, [data]); + + return { + isColPropagateChecked, + setIsColPropagateChecked, + refetch, + }; +}; + +// Hook to update the document propagation settings +export const useUpdateDocPropagationSettings = () => { + const [updateDocPropagationSettings] = useUpdateDocPropagationSettingsMutation(); + const { refetch } = useGetDocPropagationSettingsQuery(); + + const updateDocPropagation = async (checked: boolean) => { + try { + await updateDocPropagationSettings({ + variables: { + input: { + docColumnPropagation: checked, + }, + }, + }); + refetch(); + message.success('Successfully updated documentation propagation settings'); + } catch (e) { + message.error('Failed to update documentation propagation settings'); + refetch(); + } + }; + + return { updateDocPropagation }; +}; diff --git a/datahub-web-react/src/app/shared/health/healthUtils.tsx b/datahub-web-react/src/app/shared/health/healthUtils.tsx index a3745da26ceea6..426e5d986ca55c 100644 --- a/datahub-web-react/src/app/shared/health/healthUtils.tsx +++ b/datahub-web-react/src/app/shared/health/healthUtils.tsx @@ -145,7 +145,7 @@ export const getHealthIcon = (type: HealthStatusType, status: HealthStatus, font export const getHealthRedirectPath = (type: HealthStatusType) => { switch (type) { case HealthStatusType.Assertions: { - return 'Validation/Assertions'; + return 'Quality/List'; } case HealthStatusType.Incidents: { return 'Incidents'; diff --git a/datahub-web-react/src/conf/Global.ts b/datahub-web-react/src/conf/Global.ts index 433b0b6416e780..e0172d3f07e41e 100644 --- a/datahub-web-react/src/conf/Global.ts +++ b/datahub-web-react/src/conf/Global.ts @@ -41,3 +41,11 @@ export const CLIENT_AUTH_COOKIE = 'actor'; * Name of the unique browser id cookie generated on client side */ export const BROWSER_ID_COOKIE = 'bid'; + +/** New Routes Map for redirection */ +export const NEW_ROUTE_MAP = { + '/Validation/Assertions': '/Quality/List', + '/Validation/Tests': '/Governance/Tests', + '/Validation/Data%20Contract': '/Quality/Data%20Contract', + '/Validation': '/Quality', +}; diff --git a/datahub-web-react/src/conf/utils.ts b/datahub-web-react/src/conf/utils.ts new file mode 100644 index 00000000000000..7adb82cf71be52 --- /dev/null +++ b/datahub-web-react/src/conf/utils.ts @@ -0,0 +1,26 @@ +/** + * + * as per the new route object + * We are redirecting older routes to new one + * e.g. + * { + '/Validation/Assertions': '/Quality/List', + } + * */ + +export const getRedirectUrl = (newRoutes: { [key: string]: string }) => { + let newPathname = `${window.location.pathname}${window.location.search}`; + if (!newRoutes) { + return newPathname; + } + + // eslint-disable-next-line no-restricted-syntax + for (const path of Object.keys(newRoutes)) { + if (newPathname.indexOf(path) !== -1) { + newPathname = newPathname.replace(path, newRoutes[path]); + break; + } + } + + return `${newPathname}${window.location.search}`; +}; diff --git a/datahub-web-react/src/graphql/app.graphql b/datahub-web-react/src/graphql/app.graphql index bfca27a4ad106a..e058a6fbb58e00 100644 --- a/datahub-web-react/src/graphql/app.graphql +++ b/datahub-web-react/src/graphql/app.graphql @@ -89,6 +89,16 @@ query getGlobalViewsSettings { } } +query getDocPropagationSettings { + docPropagationSettings { + docColumnPropagation + } +} + mutation updateGlobalViewsSettings($input: UpdateGlobalViewsSettingsInput!) { updateGlobalViewsSettings(input: $input) } + +mutation updateDocPropagationSettings($input: UpdateDocPropagationSettingsInput!) { + updateDocPropagationSettings(input: $input) +} diff --git a/datahub-web-react/src/graphql/fragments.graphql b/datahub-web-react/src/graphql/fragments.graphql index e8b621df649a95..adaa8aa2535810 100644 --- a/datahub-web-react/src/graphql/fragments.graphql +++ b/datahub-web-react/src/graphql/fragments.graphql @@ -736,6 +736,37 @@ fragment schemaFieldFields on SchemaField { ...businessAttribute } } + documentation { + documentations { + documentation + attribution { + time + actor { + urn + type + ...entityDisplayNameFields + } + source { + urn + type + } + sourceDetail { + key + value + } + } + } + } + parent { + urn + type + ...entityDisplayNameFields + ... on Dataset { + platform { + ...platformFields + } + } + } } } diff --git a/datahub-web-react/src/graphql/group.graphql b/datahub-web-react/src/graphql/group.graphql index ee04489540f9c7..60da627fd254df 100644 --- a/datahub-web-react/src/graphql/group.graphql +++ b/datahub-web-react/src/graphql/group.graphql @@ -48,6 +48,7 @@ query getGroup($urn: String!, $membersCount: Int!) { direction: INCOMING start: 0 count: $membersCount + includeSoftDelete: false } ) { start @@ -86,6 +87,7 @@ query getAllGroupMembers($urn: String!, $start: Int!, $count: Int!) { direction: INCOMING start: $start count: $count + includeSoftDelete: false } ) { start @@ -121,7 +123,15 @@ query getAllGroupMembers($urn: String!, $start: Int!, $count: Int!) { query getGroupMembers($urn: String!, $start: Int!, $count: Int!) { corpGroup(urn: $urn) { - relationships(input: { types: ["IsMemberOfGroup"], direction: INCOMING, start: $start, count: $count }) { + relationships( + input: { + types: ["IsMemberOfGroup"] + direction: INCOMING + start: $start + count: $count + includeSoftDelete: false + } + ) { start count total @@ -155,7 +165,15 @@ query getGroupMembers($urn: String!, $start: Int!, $count: Int!) { query getNativeGroupMembers($urn: String!, $start: Int!, $count: Int!) { corpGroup(urn: $urn) { - relationships(input: { types: ["IsMemberOfNativeGroup"], direction: INCOMING, start: $start, count: $count }) { + relationships( + input: { + types: ["IsMemberOfNativeGroup"] + direction: INCOMING + start: $start + count: $count + includeSoftDelete: false + } + ) { start count total @@ -209,7 +227,13 @@ query listGroups($input: ListGroupsInput!) { pictureLink } memberCount: relationships( - input: { types: ["IsMemberOfGroup", "IsMemberOfNativeGroup"], direction: INCOMING, start: 0, count: 1 } + input: { + types: ["IsMemberOfGroup", "IsMemberOfNativeGroup"] + direction: INCOMING + start: 0 + count: 1 + includeSoftDelete: false + } ) { total } diff --git a/datahub-web-react/src/graphql/search.graphql b/datahub-web-react/src/graphql/search.graphql index b0acc454fa55b2..547f8351f6a2a6 100644 --- a/datahub-web-react/src/graphql/search.graphql +++ b/datahub-web-react/src/graphql/search.graphql @@ -974,6 +974,12 @@ fragment entityField on SchemaFieldEntity { parent { urn type + ...entityDisplayNameFields + ... on Dataset { + platform { + ...platformFields + } + } } fieldPath structuredProperties { diff --git a/datahub-web-react/src/graphql/view.graphql b/datahub-web-react/src/graphql/view.graphql index 9d38a09b8280bc..f7b82a1e46f046 100644 --- a/datahub-web-react/src/graphql/view.graphql +++ b/datahub-web-react/src/graphql/view.graphql @@ -39,6 +39,12 @@ query listGlobalViews($start: Int!, $count: Int!, $query: String) { } } +query getView($urn: String!) { + view(urn: $urn) { + ...view + } +} + mutation createView($input: CreateViewInput!) { createView(input: $input) { ...view diff --git a/docker/datahub-frontend/Dockerfile b/docker/datahub-frontend/Dockerfile index a828f1d8c27ad5..2a9354cbf6a04f 100644 --- a/docker/datahub-frontend/Dockerfile +++ b/docker/datahub-frontend/Dockerfile @@ -25,24 +25,25 @@ RUN apk --no-cache --update-cache --available upgrade \ ENV LD_LIBRARY_PATH="/lib:/lib64" -FROM base as prod-install +FROM base as unpack COPY ./datahub-frontend.zip / -RUN unzip datahub-frontend.zip -d /datahub-frontend \ - && mv /datahub-frontend/main/* /datahub-frontend \ - && rmdir /datahub-frontend/main \ - && rm datahub-frontend.zip +RUN unzip datahub-frontend.zip -d /tmp/out \ + && mv /tmp/out/main /datahub-frontend COPY ./docker/monitoring/client-prometheus-config.yaml /datahub-frontend/ RUN chown -R datahub:datahub /datahub-frontend && chmod 755 /datahub-frontend +FROM base as prod-install + +COPY --from=unpack /datahub-frontend/ /datahub-frontend/ + FROM base as dev-install # Dummy stage for development. Assumes code is built on your machine and mounted to this image. # See this excellent thread https://github.com/docker/cli/issues/1134 VOLUME [ "/datahub-frontend" ] FROM ${APP_ENV}-install as final -COPY ./docker/datahub-frontend/start.sh / -RUN chown datahub:datahub /start.sh && chmod 755 /start.sh +COPY --chown=datahub:datahub --chmod=755 ./docker/datahub-frontend/start.sh / USER datahub ARG SERVER_PORT=9002 diff --git a/docker/datahub-gms/env/docker-without-neo4j.env b/docker/datahub-gms/env/docker-without-neo4j.env index cc0dd6b4278b56..37b7ba1797af5b 100644 --- a/docker/datahub-gms/env/docker-without-neo4j.env +++ b/docker/datahub-gms/env/docker-without-neo4j.env @@ -23,6 +23,8 @@ PE_CONSUMER_ENABLED=true UI_INGESTION_ENABLED=true ENTITY_SERVICE_ENABLE_RETENTION=true +ELASTIC_ID_HASH_ALGO=MD5 + # Uncomment to disable persistence of client-side analytics events # DATAHUB_ANALYTICS_ENABLED=false diff --git a/docker/datahub-gms/env/docker.env b/docker/datahub-gms/env/docker.env index 59fc4bdde02ff4..0ecaa32c4cb123 100644 --- a/docker/datahub-gms/env/docker.env +++ b/docker/datahub-gms/env/docker.env @@ -27,6 +27,8 @@ MCE_CONSUMER_ENABLED=true PE_CONSUMER_ENABLED=true UI_INGESTION_ENABLED=true +ELASTIC_ID_HASH_ALGO=MD5 + # Uncomment to enable Metadata Service Authentication METADATA_SERVICE_AUTH_ENABLED=false diff --git a/docker/datahub-ingestion-base/Dockerfile b/docker/datahub-ingestion-base/Dockerfile index 383478b675640f..8a238c32704bb6 100644 --- a/docker/datahub-ingestion-base/Dockerfile +++ b/docker/datahub-ingestion-base/Dockerfile @@ -85,18 +85,18 @@ RUN apt-get update && apt-get install -y -qq \ RUN if [ $(arch) = "x86_64" ]; then \ mkdir /opt/oracle && \ cd /opt/oracle && \ - wget --no-verbose -c https://download.oracle.com/otn_software/linux/instantclient/216000/instantclient-basic-linux.x64-21.6.0.0.0dbru.zip && \ - unzip instantclient-basic-linux.x64-21.6.0.0.0dbru.zip && \ - rm instantclient-basic-linux.x64-21.6.0.0.0dbru.zip && \ - sh -c "echo /opt/oracle/instantclient_21_6 > /etc/ld.so.conf.d/oracle-instantclient.conf" && \ + wget --no-verbose -c https://download.oracle.com/otn_software/linux/instantclient/2115000/instantclient-basic-linux.x64-21.15.0.0.0dbru.zip && \ + unzip instantclient-basic-linux.x64-21.15.0.0.0dbru.zip && \ + rm instantclient-basic-linux.x64-21.15.0.0.0dbru.zip && \ + sh -c "echo /opt/oracle/instantclient_21_15 > /etc/ld.so.conf.d/oracle-instantclient.conf" && \ ldconfig; \ else \ mkdir /opt/oracle && \ cd /opt/oracle && \ - wget --no-verbose -c https://download.oracle.com/otn_software/linux/instantclient/191000/instantclient-basic-linux.arm64-19.10.0.0.0dbru.zip && \ - unzip instantclient-basic-linux.arm64-19.10.0.0.0dbru.zip && \ - rm instantclient-basic-linux.arm64-19.10.0.0.0dbru.zip && \ - sh -c "echo /opt/oracle/instantclient_19_10 > /etc/ld.so.conf.d/oracle-instantclient.conf" && \ + wget --no-verbose -c https://download.oracle.com/otn_software/linux/instantclient/1923000/instantclient-basic-linux.arm64-19.23.0.0.0dbru.zip && \ + unzip instantclient-basic-linux.arm64-19.23.0.0.0dbru.zip && \ + rm instantclient-basic-linux.arm64-19.23.0.0.0dbru.zip && \ + sh -c "echo /opt/oracle/instantclient_19_23 > /etc/ld.so.conf.d/oracle-instantclient.conf" && \ ldconfig; \ fi; diff --git a/docker/datahub-ingestion-base/base-requirements.txt b/docker/datahub-ingestion-base/base-requirements.txt index 2f2b64723f0edd..fa07b4184a6bc0 100644 --- a/docker/datahub-ingestion-base/base-requirements.txt +++ b/docker/datahub-ingestion-base/base-requirements.txt @@ -1,19 +1,20 @@ # Generated requirements file. Run ./regenerate-base-requirements.sh to regenerate. -acryl-datahub-classify==0.0.10 +acryl-datahub-classify==0.0.11 acryl-PyHive==0.6.16 -acryl-sqlglot==24.0.1.dev7 +acryl-sqlglot==25.3.1.dev3 aenum==3.1.15 -aiohttp==3.9.5 +aiohappyeyeballs==2.3.2 +aiohttp==3.10.0 aiosignal==1.3.1 -alembic==1.13.1 +alembic==1.13.2 altair==4.2.0 anyio==4.4.0 -apache-airflow==2.9.2 +apache-airflow==2.9.3 apache-airflow-providers-common-io==1.3.2 -apache-airflow-providers-common-sql==1.14.0 -apache-airflow-providers-fab==1.1.1 -apache-airflow-providers-ftp==3.9.1 -apache-airflow-providers-http==4.11.1 +apache-airflow-providers-common-sql==1.14.2 +apache-airflow-providers-fab==1.2.2 +apache-airflow-providers-ftp==3.10.0 +apache-airflow-providers-http==4.12.0 apache-airflow-providers-imap==3.6.1 apache-airflow-providers-smtp==1.7.1 apache-airflow-providers-sqlite==3.8.1 @@ -26,25 +27,30 @@ asgiref==3.8.1 asn1crypto==1.5.1 asttokens==2.4.1 async-timeout==4.0.3 -asynch==0.2.3 +asynch==0.2.4 attrs==23.2.0 avro==1.11.3 avro-gen3==0.7.13 +azure-common==1.1.28 +azure-core==1.29.4 +azure-identity==1.14.1 +azure-storage-blob==12.21.0 +azure-storage-file-datalake==12.16.0 Babel==2.15.0 backoff==2.2.1 beautifulsoup4==4.12.3 bleach==6.1.0 blinker==1.8.2 blis==0.7.11 -boto3==1.34.129 -botocore==1.34.129 +boto3==1.34.151 +botocore==1.34.151 bracex==2.4 cached-property==1.5.2 cachelib==0.9.0 -cachetools==5.3.3 +cachetools==5.4.0 catalogue==2.0.10 cattrs==23.2.3 -certifi==2024.6.2 +certifi==2024.7.4 cffi==1.16.0 chardet==5.2.0 charset-normalizer==3.3.2 @@ -55,25 +61,26 @@ click-spinner==0.1.10 clickclick==20.10.2 clickhouse-driver==0.2.8 clickhouse-sqlalchemy==0.2.4 +cloudpathlib==0.18.1 cloudpickle==3.0.0 colorama==0.4.6 colorlog==4.8.0 comm==0.2.2 confection==0.1.5 ConfigUpdater==3.2 -confluent-kafka==2.4.0 -connexion==2.14.1 +confluent-kafka==2.5.0 +connexion==2.14.2 cron-descriptor==1.4.3 -croniter==2.0.5 +croniter==3.0.3 cryptography==42.0.8 cx_Oracle==8.3.0 cymem==2.0.8 databricks-dbapi==0.6.0 -databricks-sdk==0.28.0 +databricks-sdk==0.29.0 databricks-sql-connector==2.9.6 dataflows-tabulator==1.54.3 db-dtypes==1.2.0 -debugpy==1.8.1 +debugpy==1.8.2 decorator==5.1.1 defusedxml==0.7.1 deltalake==0.17.4 @@ -84,42 +91,42 @@ docker==7.1.0 docutils==0.21.2 ecdsa==0.19.0 elasticsearch==7.13.4 -email_validator==2.1.2 +email_validator==2.2.0 entrypoints==0.4 et-xmlfile==1.1.0 -exceptiongroup==1.2.1 +exceptiongroup==1.2.2 executing==2.0.1 expandvars==0.12.0 -fastavro==1.9.4 +fastavro==1.9.5 fastjsonschema==2.20.0 -filelock==3.15.1 +filelock==3.15.4 Flask==2.2.5 flatdict==4.0.1 frozenlist==1.4.1 fsspec==2023.12.2 future==1.0.0 -GeoAlchemy2==0.15.1 +GeoAlchemy2==0.15.2 gitdb==4.0.11 GitPython==3.1.43 -google-api-core==2.19.0 -google-auth==2.30.0 -google-cloud-appengine-logging==1.4.3 +google-api-core==2.19.1 +google-auth==2.32.0 +google-cloud-appengine-logging==1.4.5 google-cloud-audit-log==0.2.5 -google-cloud-bigquery==3.24.0 +google-cloud-bigquery==3.25.0 google-cloud-core==2.4.1 -google-cloud-datacatalog==3.19.0 +google-cloud-datacatalog==3.20.0 google-cloud-datacatalog-lineage==0.2.2 google-cloud-logging==3.5.0 google-crc32c==1.5.0 -google-re2==1.1.20240601 +google-re2==1.1.20240702 google-resumable-media==2.7.1 -googleapis-common-protos==1.63.1 +googleapis-common-protos==1.63.2 gql==3.5.0 graphql-core==3.2.3 great-expectations==0.15.50 greenlet==3.0.3 -grpc-google-iam-v1==0.13.0 -grpcio==1.64.1 +grpc-google-iam-v1==0.13.1 +grpcio==1.65.2 grpcio-status==1.62.2 grpcio-tools==1.62.2 gssapi==1.8.3 @@ -130,7 +137,7 @@ httpx==0.27.0 humanfriendly==10.0 idna==3.7 ijson==3.3.0 -importlib_metadata==7.1.0 +importlib_metadata==7.2.1 importlib_resources==6.4.0 inflection==0.5.1 ipaddress==1.0.23 @@ -148,7 +155,7 @@ jsonlines==4.0.0 jsonpatch==1.33 jsonpointer==3.0.0 jsonref==1.1.0 -jsonschema==4.22.0 +jsonschema==4.23.0 jsonschema-specifications==2023.12.1 jupyter-server==1.16.0 jupyter_client==7.4.9 @@ -159,16 +166,16 @@ langcodes==3.4.0 language_data==1.2.0 lark==1.1.4 lazy-object-proxy==1.10.0 -leb128==1.0.7 -limits==3.12.0 +leb128==1.0.8 +limits==3.13.0 linear-tsv==1.1.0 linkify-it-py==2.0.3 -lkml==1.3.4 +lkml==1.3.5 lockfile==0.12.2 looker-sdk==23.0.0 lxml==5.2.2 lz4==4.3.3 -makefun==1.15.2 +makefun==1.15.4 Mako==1.3.5 marisa-trie==1.2.0 markdown-it-py==3.0.0 @@ -182,11 +189,12 @@ mdurl==0.1.2 methodtools==0.4.7 mistune==3.0.2 mixpanel==4.10.1 -mlflow-skinny==2.14.0 +mlflow-skinny==2.15.0 mmhash3==3.0.1 more-itertools==10.3.0 moto==4.2.14 msal==1.22.0 +msal-extensions==1.1.0 multidict==6.0.5 murmurhash==1.0.10 mypy-extensions==1.0.0 @@ -201,47 +209,46 @@ notebook_shim==0.2.4 numpy==1.26.4 oauthlib==3.2.2 okta==1.7.0 -openlineage-airflow==1.12.0 -openlineage-integration-common==1.12.0 -openlineage-python==1.12.0 -openlineage_sql==1.12.0 -openpyxl==3.1.4 -opentelemetry-api==1.25.0 -opentelemetry-exporter-otlp==1.25.0 -opentelemetry-exporter-otlp-proto-common==1.25.0 -opentelemetry-exporter-otlp-proto-grpc==1.25.0 -opentelemetry-exporter-otlp-proto-http==1.25.0 -opentelemetry-proto==1.25.0 -opentelemetry-sdk==1.25.0 -opentelemetry-semantic-conventions==0.46b0 +openlineage-airflow==1.18.0 +openlineage-integration-common==1.18.0 +openlineage-python==1.18.0 +openlineage_sql==1.18.0 +openpyxl==3.1.5 +opentelemetry-api==1.26.0 +opentelemetry-exporter-otlp==1.26.0 +opentelemetry-exporter-otlp-proto-common==1.26.0 +opentelemetry-exporter-otlp-proto-grpc==1.26.0 +opentelemetry-exporter-otlp-proto-http==1.26.0 +opentelemetry-proto==1.26.0 +opentelemetry-sdk==1.26.0 +opentelemetry-semantic-conventions==0.47b0 ordered-set==4.1.0 packaging==24.1 pandas==2.1.4 pandocfilters==1.5.1 parse==1.20.2 parso==0.8.4 -pathlib_abc==0.1.1 pathspec==0.12.1 -pathy==0.11.0 pendulum==3.0.0 pexpect==4.9.0 phonenumbers==8.13.0 platformdirs==4.2.2 pluggy==1.5.0 +portalocker==2.10.1 preshed==3.0.9 prison==0.2.1 progressbar2==4.4.2 prometheus_client==0.20.0 prompt_toolkit==3.0.47 -proto-plus==1.23.0 -protobuf==4.25.3 +proto-plus==1.24.0 +protobuf==4.25.4 psutil==6.0.0 psycopg2-binary==2.9.9 ptyprocess==0.7.0 -pure-eval==0.2.2 pure-sasl==0.6.2 +pure_eval==0.2.3 py-partiql-parser==0.5.0 -pyarrow==16.1.0 +pyarrow==17.0.0 pyarrow-hotfix==0.6 pyasn1==0.6.0 pyasn1_modules==0.4.0 @@ -249,16 +256,16 @@ pyathena==2.25.2 pycountry==24.6.1 pycparser==2.22 pycryptodome==3.20.0 -pydantic==1.10.16 -pydash==8.0.1 +pydantic==1.10.17 +pydash==8.0.3 pydruid==0.6.9 Pygments==2.18.0 pyiceberg==0.4.0 -pymongo==4.7.3 +pymongo==4.8.0 PyMySQL==1.1.1 -pyOpenSSL==24.1.0 +pyOpenSSL==24.2.1 pyparsing==3.0.9 -pyspnego==0.11.0 +pyspnego==0.11.1 python-daemon==3.0.1 python-dateutil==2.9.0.post0 python-jose==3.3.0 @@ -273,9 +280,9 @@ pytz==2024.1 PyYAML==6.0.1 pyzmq==26.0.3 redash-toolbelt==0.1.9 -redshift-connector==2.1.1 +redshift-connector==2.1.2 referencing==0.35.1 -regex==2024.5.15 +regex==2024.7.24 requests==2.32.3 requests-file==2.1.0 requests-gssapi==1.3.0 @@ -286,31 +293,32 @@ rfc3339-validator==0.1.4 rfc3986==2.0.0 rich==13.7.1 rich-argparse==1.5.2 -rpds-py==0.18.1 +rpds-py==0.19.1 rsa==4.9 rstr==3.2.2 ruamel.yaml==0.17.17 -s3transfer==0.10.1 +s3transfer==0.10.2 schwifty==2024.6.1 -scipy==1.13.1 +scipy==1.14.0 scramp==1.4.5 Send2Trash==1.8.3 -sentry-sdk==2.5.1 +sentry-sdk==2.12.0 setproctitle==1.3.3 +shellingham==1.5.4 simple-salesforce==1.12.6 six==1.16.0 slack-sdk==3.18.1 -smart-open==6.4.0 +smart-open==7.0.4 smmap==5.0.1 sniffio==1.3.1 -snowflake-connector-python==3.11.0 -snowflake-sqlalchemy==1.5.3 +snowflake-connector-python==3.12.0 +snowflake-sqlalchemy==1.6.1 sortedcontainers==2.4.0 soupsieve==2.5 -spacy==3.5.0 +spacy==3.7.5 spacy-legacy==3.0.12 spacy-loggers==1.0.5 -sql-metadata==2.2.2 +sql_metadata==2.12.0 SQLAlchemy==1.4.44 sqlalchemy-bigquery==1.11.0 sqlalchemy-cockroachdb==1.4.4 @@ -318,7 +326,7 @@ SQLAlchemy-JSONField==1.0.2 sqlalchemy-pytds==0.3.5 sqlalchemy-redshift==0.8.14 SQLAlchemy-Utils==0.41.2 -sqlglotrs==0.2.5 +sqlglotrs==0.2.7 sqllineage==1.3.8 sqlparse==0.4.4 srsly==2.4.8 @@ -327,25 +335,25 @@ strictyaml==1.7.3 tableauserverclient==0.25 tableschema==1.20.11 tabulate==0.9.0 -tenacity==8.4.1 -teradatasql==20.0.0.12 +tenacity==9.0.0 +teradatasql==20.0.0.14 teradatasqlalchemy==20.0.0.1 termcolor==2.4.0 terminado==0.18.1 text-unidecode==1.3 -thinc==8.1.12 +thinc==8.2.5 thrift==0.16.0 thrift-sasl==0.4.3 -time-machine==2.14.1 +time-machine==2.14.2 tinycss2==1.3.0 toml==0.10.2 -tomlkit==0.12.5 +tomlkit==0.13.0 toolz==0.12.1 tornado==6.4.1 tqdm==4.66.4 traitlets==5.2.1.post0 -trino==0.328.0 -typer==0.7.0 +trino==0.329.0 +typer==0.12.3 typing-inspect==0.9.0 typing_extensions==4.12.2 tzdata==2024.1 @@ -355,15 +363,16 @@ ujson==5.10.0 unicodecsv==0.14.1 universal_pathlib==0.2.2 urllib3==1.26.19 -vertica-python==1.3.8 +vertica-python==1.4.0 vertica-sqlalchemy-dialect==0.0.8.2 vininfo==1.8.0 wasabi==1.1.3 wcmatch==8.5.2 wcwidth==0.2.13 +weasel==0.4.1 webencodings==0.5.1 websocket-client==1.8.0 -Werkzeug==2.3.8 +Werkzeug==2.2.3 widgetsnbextension==4.0.11 wirerope==0.4.7 wrapt==1.16.0 diff --git a/docker/datahub-ingestion-base/build.gradle b/docker/datahub-ingestion-base/build.gradle index faa0589cfbfbbf..5652fedcd93b3b 100644 --- a/docker/datahub-ingestion-base/build.gradle +++ b/docker/datahub-ingestion-base/build.gradle @@ -12,7 +12,7 @@ ext { docker_target = project.getProperties().getOrDefault("dockerTarget", "slim") docker_version = "${version}${docker_target == 'slim' ? '-slim' : ''}" - revision = 3 // increment to trigger rebuild + revision = 4 // increment to trigger rebuild } docker { diff --git a/docker/datahub-ingestion/Dockerfile b/docker/datahub-ingestion/Dockerfile index 068911695811f5..b8eda548491224 100644 --- a/docker/datahub-ingestion/Dockerfile +++ b/docker/datahub-ingestion/Dockerfile @@ -1,7 +1,7 @@ # Defining environment ARG APP_ENV=full ARG BASE_IMAGE=acryldata/datahub-ingestion-base -ARG DOCKER_VERSION=head +ARG DOCKER_VERSION=head-full ARG DEBIAN_REPO_URL=https://deb.debian.org/debian ARG PIP_MIRROR_URL=https://pypi.python.org/simple diff --git a/docker/datahub-ingestion/build.gradle b/docker/datahub-ingestion/build.gradle index b9ab546674a031..6757be7cd6f221 100644 --- a/docker/datahub-ingestion/build.gradle +++ b/docker/datahub-ingestion/build.gradle @@ -12,7 +12,7 @@ ext { docker_target = project.getProperties().getOrDefault("dockerTarget", "slim") docker_version = "${version}${docker_target == 'slim' ? '-slim' : ''}" - revision = 3 // increment to trigger rebuild + revision = 4 // increment to trigger rebuild } dependencies { diff --git a/docker/datahub-mae-consumer/env/docker-without-neo4j.env b/docker/datahub-mae-consumer/env/docker-without-neo4j.env index b6899f7e6d63b2..6a82f235b29711 100644 --- a/docker/datahub-mae-consumer/env/docker-without-neo4j.env +++ b/docker/datahub-mae-consumer/env/docker-without-neo4j.env @@ -13,6 +13,8 @@ ES_BULK_REFRESH_POLICY=WAIT_UNTIL GRAPH_SERVICE_IMPL=elasticsearch ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mae-consumer/resources/entity-registry.yml +ELASTIC_ID_HASH_ALGO=MD5 + # Uncomment to disable persistence of client-side analytics events # DATAHUB_ANALYTICS_ENABLED=false diff --git a/docker/datahub-mae-consumer/env/docker.env b/docker/datahub-mae-consumer/env/docker.env index 5a6daa6eaeaed7..1f0ee4b05b3820 100644 --- a/docker/datahub-mae-consumer/env/docker.env +++ b/docker/datahub-mae-consumer/env/docker.env @@ -17,6 +17,8 @@ NEO4J_PASSWORD=datahub GRAPH_SERVICE_IMPL=neo4j ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mae-consumer/resources/entity-registry.yml +ELASTIC_ID_HASH_ALGO=MD5 + # Uncomment to disable persistence of client-side analytics events # DATAHUB_ANALYTICS_ENABLED=false diff --git a/docker/datahub-mce-consumer/env/docker-without-neo4j.env b/docker/datahub-mce-consumer/env/docker-without-neo4j.env index e7be7d8ed4ddc5..b0edfc0a75b669 100644 --- a/docker/datahub-mce-consumer/env/docker-without-neo4j.env +++ b/docker/datahub-mce-consumer/env/docker-without-neo4j.env @@ -24,6 +24,8 @@ MAE_CONSUMER_ENABLED=false PE_CONSUMER_ENABLED=false UI_INGESTION_ENABLED=false +ELASTIC_ID_HASH_ALGO=MD5 + # Uncomment to configure kafka topic names # Make sure these names are consistent across the whole deployment # METADATA_CHANGE_PROPOSAL_TOPIC_NAME=MetadataChangeProposal_v1 diff --git a/docker/datahub-mce-consumer/env/docker.env b/docker/datahub-mce-consumer/env/docker.env index 8618f3f5f7af7a..c0f85ef667546e 100644 --- a/docker/datahub-mce-consumer/env/docker.env +++ b/docker/datahub-mce-consumer/env/docker.env @@ -24,6 +24,8 @@ MAE_CONSUMER_ENABLED=false PE_CONSUMER_ENABLED=false UI_INGESTION_ENABLED=false +ELASTIC_ID_HASH_ALGO=MD5 + # Uncomment to configure kafka topic names # Make sure these names are consistent across the whole deployment # METADATA_CHANGE_PROPOSAL_TOPIC_NAME=MetadataChangeProposal_v1 diff --git a/docker/docker-compose-with-cassandra.yml b/docker/docker-compose-with-cassandra.yml index d722b07b9a7af4..de766f76cb626e 100644 --- a/docker/docker-compose-with-cassandra.yml +++ b/docker/docker-compose-with-cassandra.yml @@ -144,7 +144,7 @@ services: - neo4jdata:/data schema-registry: hostname: schema-registry - image: confluentinc/cp-schema-registry:7.4.0 + image: ${DATAHUB_CONFLUENT_SCHEMA_REGISTRY_IMAGE:-confluentinc/cp-schema-registry}:${DATAHUB_CONFLUENT_VERSION:-7.4.0} ports: - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 env_file: schema-registry/env/docker.env @@ -159,7 +159,7 @@ services: condition: service_healthy broker: hostname: broker - image: confluentinc/cp-kafka:7.4.0 + image: ${DATAHUB_CONFLUENT_KAFKA_IMAGE:-confluentinc/cp-kafka}:${DATAHUB_CONFLUENT_VERSION:-7.4.0} ports: - 29092:29092 - 9092:9092 @@ -177,7 +177,7 @@ services: - broker:/var/lib/kafka/data/ zookeeper: hostname: zookeeper - image: confluentinc/cp-zookeeper:7.4.0 + image: ${DATAHUB_CONFLUENT_ZOOKEEPER_IMAGE:-confluentinc/cp-zookeeper}:${DATAHUB_CONFLUENT_VERSION:-7.4.0} ports: - 2181:2181 env_file: zookeeper/env/docker.env diff --git a/docker/docker-compose-without-neo4j.yml b/docker/docker-compose-without-neo4j.yml index eae36fb849fd5c..748a2cc9e04167 100644 --- a/docker/docker-compose-without-neo4j.yml +++ b/docker/docker-compose-without-neo4j.yml @@ -123,7 +123,7 @@ services: - esdata:/usr/share/elasticsearch/data schema-registry: hostname: schema-registry - image: confluentinc/cp-schema-registry:7.4.0 + image: ${DATAHUB_CONFLUENT_SCHEMA_REGISTRY_IMAGE:-confluentinc/cp-schema-registry}:${DATAHUB_CONFLUENT_VERSION:-7.4.0} ports: - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 env_file: schema-registry/env/docker.env @@ -138,7 +138,7 @@ services: condition: service_healthy broker: hostname: broker - image: confluentinc/cp-kafka:7.4.0 + image: ${DATAHUB_CONFLUENT_KAFKA_IMAGE:-confluentinc/cp-kafka}:${DATAHUB_CONFLUENT_VERSION:-7.4.0} ports: - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 env_file: broker/env/docker.env @@ -155,7 +155,7 @@ services: - broker:/var/lib/kafka/data/ zookeeper: hostname: zookeeper - image: confluentinc/cp-zookeeper:7.4.0 + image: ${DATAHUB_CONFLUENT_ZOOKEEPER_IMAGE:-confluentinc/cp-zookeeper}:${DATAHUB_CONFLUENT_VERSION:-7.4.0} ports: - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181 env_file: zookeeper/env/docker.env diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 96f37496859a46..ae55861580becd 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -143,7 +143,7 @@ services: - neo4jdata:/data schema-registry: hostname: schema-registry - image: confluentinc/cp-schema-registry:7.4.0 + image: ${DATAHUB_CONFLUENT_SCHEMA_REGISTRY_IMAGE:-confluentinc/cp-schema-registry}:${DATAHUB_CONFLUENT_VERSION:-7.4.0} ports: - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 env_file: schema-registry/env/docker.env @@ -158,7 +158,7 @@ services: condition: service_healthy broker: hostname: broker - image: confluentinc/cp-kafka:7.4.0 + image: ${DATAHUB_CONFLUENT_KAFKA_IMAGE:-confluentinc/cp-kafka}:${DATAHUB_CONFLUENT_VERSION:-7.4.0} ports: - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 env_file: broker/env/docker.env @@ -175,7 +175,7 @@ services: - broker:/var/lib/kafka/data/ zookeeper: hostname: zookeeper - image: confluentinc/cp-zookeeper:7.4.0 + image: ${DATAHUB_CONFLUENT_ZOOKEEPER_IMAGE:-confluentinc/cp-zookeeper}:${DATAHUB_CONFLUENT_VERSION:-7.4.0} ports: - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181 env_file: zookeeper/env/docker.env diff --git a/docker/kafka-setup/env_to_properties.py b/docker/kafka-setup/env_to_properties.py new file mode 100644 index 00000000000000..8d8b8c3cc7b59f --- /dev/null +++ b/docker/kafka-setup/env_to_properties.py @@ -0,0 +1,24 @@ +import os +import re +import sys + + +def env_to_properties(env_prefix: str, properties_file: str): + pattern = re.compile('(?<=[^_])_(?=[^_])') + props = {} + + for (env_name, val) in os.environ.items(): + if env_name.startswith(env_prefix): + raw_name = env_name[len(env_prefix):].lower() + prop_dot = '.'.join(pattern.split(raw_name)) + props[prop_dot] = val + + with open(properties_file, 'a') as f: + for k, v in props.items(): + f.writelines(f'{k}={v}\n') + + +if __name__ == '__main__': + env_prefix = sys.argv[1] + properties_file = sys.argv[2] + env_to_properties(env_prefix, properties_file) diff --git a/docker/kafka-setup/kafka-setup.sh b/docker/kafka-setup/kafka-setup.sh index 439ffb4d4d8295..392cca94666419 100755 --- a/docker/kafka-setup/kafka-setup.sh +++ b/docker/kafka-setup/kafka-setup.sh @@ -10,46 +10,8 @@ fi . kafka-config.sh echo "bootstrap.servers=$KAFKA_BOOTSTRAP_SERVER" > $CONNECTION_PROPERTIES_PATH -echo "security.protocol=$KAFKA_PROPERTIES_SECURITY_PROTOCOL" >> $CONNECTION_PROPERTIES_PATH -## Add support for SASL_PLAINTEXT -if [[ $KAFKA_PROPERTIES_SECURITY_PROTOCOL == "SASL_PLAINTEXT" ]]; then - echo "sasl.mechanism=$KAFKA_PROPERTIES_SASL_MECHANISM" >> $CONNECTION_PROPERTIES_PATH - echo "sasl.jaas.config=$KAFKA_PROPERTIES_SASL_JAAS_CONFIG" >> $CONNECTION_PROPERTIES_PATH - echo "sasl.kerberos.service.name=$KAFKA_PROPERTIES_SASL_KERBEROS_SERVICE_NAME" >> $CONNECTION_PROPERTIES_PATH -fi - -## Add support for SASL_SSL -if [[ $KAFKA_PROPERTIES_SECURITY_PROTOCOL == "SASL_SSL" ]]; then - echo "sasl.jaas.config=$KAFKA_PROPERTIES_SASL_JAAS_CONFIG" >> $CONNECTION_PROPERTIES_PATH - echo "sasl.mechanism=$KAFKA_PROPERTIES_SASL_MECHANISM" >> $CONNECTION_PROPERTIES_PATH -fi - -if [[ $KAFKA_PROPERTIES_SECURITY_PROTOCOL == "SSL" ]]; then - if [[ -n $KAFKA_PROPERTIES_SSL_KEYSTORE_LOCATION ]]; then - echo "ssl.keystore.location=$KAFKA_PROPERTIES_SSL_KEYSTORE_LOCATION" >> $CONNECTION_PROPERTIES_PATH - echo "ssl.keystore.password=$KAFKA_PROPERTIES_SSL_KEYSTORE_PASSWORD" >> $CONNECTION_PROPERTIES_PATH - echo "ssl.key.password=$KAFKA_PROPERTIES_SSL_KEY_PASSWORD" >> $CONNECTION_PROPERTIES_PATH - if [[ -n $KAFKA_PROPERTIES_SSL_KEYSTORE_TYPE ]]; then - echo "ssl.keystore.type=$KAFKA_PROPERTIES_SSL_KEYSTORE_TYPE" >> $CONNECTION_PROPERTIES_PATH - fi - fi - if [[ -n $KAFKA_PROPERTIES_SSL_TRUSTSTORE_LOCATION ]]; then - echo "ssl.truststore.location=$KAFKA_PROPERTIES_SSL_TRUSTSTORE_LOCATION" >> $CONNECTION_PROPERTIES_PATH - if [[ $KAFKA_PROPERTIES_SSL_TRUSTSTORE_TYPE != "PEM" ]]; then - echo "ssl.truststore.password=$KAFKA_PROPERTIES_SSL_TRUSTSTORE_PASSWORD" >> $CONNECTION_PROPERTIES_PATH - fi - if [[ -n $KAFKA_PROPERTIES_SSL_TRUSTSTORE_TYPE ]]; then - echo "ssl.truststore.type=$KAFKA_PROPERTIES_SSL_TRUSTSTORE_TYPE" >> $CONNECTION_PROPERTIES_PATH - fi - fi - echo "ssl.endpoint.identification.algorithm=$KAFKA_PROPERTIES_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM" >> $CONNECTION_PROPERTIES_PATH -fi - -# Add support for SASL_CLIENT_CALLBACK_HANDLER_CLASS -if [[ -n "$KAFKA_PROPERTIES_SASL_CLIENT_CALLBACK_HANDLER_CLASS" ]]; then - echo "sasl.client.callback.handler.class=$KAFKA_PROPERTIES_SASL_CLIENT_CALLBACK_HANDLER_CLASS" >> $CONNECTION_PROPERTIES_PATH -fi +python env_to_properties.py KAFKA_PROPERTIES_ $CONNECTION_PROPERTIES_PATH # cub kafka-ready -c $CONNECTION_PROPERTIES_PATH -b $KAFKA_BOOTSTRAP_SERVER 1 180 . kafka-ready.sh diff --git a/docker/profiles/docker-compose.prerequisites.yml b/docker/profiles/docker-compose.prerequisites.yml index 08ebc8b65d8c9c..7cd9c9039539cc 100644 --- a/docker/profiles/docker-compose.prerequisites.yml +++ b/docker/profiles/docker-compose.prerequisites.yml @@ -210,7 +210,7 @@ services: - neo4jdata:/data kafka-broker: hostname: broker - image: confluentinc/cp-kafka:7.4.0 + image: ${DATAHUB_CONFLUENT_KAFKA_IMAGE:-confluentinc/cp-kafka}:${DATAHUB_CONFLUENT_VERSION:-7.4.0} command: - /bin/bash - -c diff --git a/docker/quickstart/docker-compose-m1.quickstart.yml b/docker/quickstart/docker-compose-m1.quickstart.yml index 834d55096468f6..a0f60d23710a07 100644 --- a/docker/quickstart/docker-compose-m1.quickstart.yml +++ b/docker/quickstart/docker-compose-m1.quickstart.yml @@ -86,6 +86,7 @@ services: - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true - ELASTICSEARCH_PORT=9200 + - ELASTIC_ID_HASH_ALGO=MD5 - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml - ENTITY_SERVICE_ENABLE_RETENTION=true - ES_BULK_REFRESH_POLICY=WAIT_UNTIL diff --git a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml index 47fb50f78e4f0c..11e33a9950ba9b 100644 --- a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml @@ -86,6 +86,7 @@ services: - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true - ELASTICSEARCH_PORT=9200 + - ELASTIC_ID_HASH_ALGO=MD5 - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml - ENTITY_SERVICE_ENABLE_RETENTION=true - ES_BULK_REFRESH_POLICY=WAIT_UNTIL diff --git a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml index 3fa13a9e56b421..2efa8959834183 100644 --- a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml @@ -86,6 +86,7 @@ services: - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true - ELASTICSEARCH_PORT=9200 + - ELASTIC_ID_HASH_ALGO=MD5 - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml - ENTITY_SERVICE_ENABLE_RETENTION=true - ES_BULK_REFRESH_POLICY=WAIT_UNTIL diff --git a/docker/quickstart/docker-compose.consumers-without-neo4j.quickstart.yml b/docker/quickstart/docker-compose.consumers-without-neo4j.quickstart.yml index a4211acedcf102..4f47a3da24eb1b 100644 --- a/docker/quickstart/docker-compose.consumers-without-neo4j.quickstart.yml +++ b/docker/quickstart/docker-compose.consumers-without-neo4j.quickstart.yml @@ -19,6 +19,7 @@ services: - ES_BULK_REFRESH_POLICY=WAIT_UNTIL - GRAPH_SERVICE_IMPL=elasticsearch - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mae-consumer/resources/entity-registry.yml + - ELASTIC_ID_HASH_ALGO=MD5 hostname: datahub-mae-consumer image: ${DATAHUB_MAE_CONSUMER_IMAGE:-acryldata/datahub-mae-consumer}:${DATAHUB_VERSION:-head} ports: @@ -37,6 +38,7 @@ services: - EBEAN_DATASOURCE_USERNAME=datahub - ELASTICSEARCH_HOST=elasticsearch - ELASTICSEARCH_PORT=9200 + - ELASTIC_ID_HASH_ALGO=MD5 - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mce-consumer/resources/entity-registry.yml - ENTITY_SERVICE_ENABLE_RETENTION=true - ES_BULK_REFRESH_POLICY=WAIT_UNTIL diff --git a/docker/quickstart/docker-compose.consumers.quickstart.yml b/docker/quickstart/docker-compose.consumers.quickstart.yml index e7571e4baf8b4e..7dd7388b939884 100644 --- a/docker/quickstart/docker-compose.consumers.quickstart.yml +++ b/docker/quickstart/docker-compose.consumers.quickstart.yml @@ -26,6 +26,7 @@ services: - NEO4J_PASSWORD=datahub - GRAPH_SERVICE_IMPL=neo4j - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mae-consumer/resources/entity-registry.yml + - ELASTIC_ID_HASH_ALGO=MD5 hostname: datahub-mae-consumer image: ${DATAHUB_MAE_CONSUMER_IMAGE:-acryldata/datahub-mae-consumer}:${DATAHUB_VERSION:-head} ports: @@ -47,6 +48,7 @@ services: - EBEAN_DATASOURCE_USERNAME=datahub - ELASTICSEARCH_HOST=elasticsearch - ELASTICSEARCH_PORT=9200 + - ELASTIC_ID_HASH_ALGO=MD5 - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mce-consumer/resources/entity-registry.yml - ENTITY_SERVICE_ENABLE_RETENTION=true - ES_BULK_REFRESH_POLICY=WAIT_UNTIL diff --git a/docker/quickstart/docker-compose.quickstart.yml b/docker/quickstart/docker-compose.quickstart.yml index c63b6d1d61b030..f42ed1f40c2467 100644 --- a/docker/quickstart/docker-compose.quickstart.yml +++ b/docker/quickstart/docker-compose.quickstart.yml @@ -86,6 +86,7 @@ services: - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true - ELASTICSEARCH_PORT=9200 + - ELASTIC_ID_HASH_ALGO=MD5 - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml - ENTITY_SERVICE_ENABLE_RETENTION=true - ES_BULK_REFRESH_POLICY=WAIT_UNTIL diff --git a/docs-website/README.md b/docs-website/README.md index c70ae0ea730d93..3b24cb869a444d 100644 --- a/docs-website/README.md +++ b/docs-website/README.md @@ -36,12 +36,12 @@ Please use the following steps when adding/managing content for the docs site. - [Feature Guide Template](../docs/_feature-guide-template.md) - [Metadata Ingestion Source Template](../metadata-ingestion/source-docs-template.md) -### Self-Hosted vs. Managed DataHub +### Self-Hosted vs. DataHub Cloud -The docs site includes resources for both self-hosted (aka open-source) DataHub and Managed DataHub alike. +The docs site includes resources for both self-hosted (aka open-source) DataHub and DataHub Cloud alike. - All Feature Guides should include the `FeatureAvailability` component within the markdown file itself -- Features only available via Managed DataHub should have the `saasOnly` class if they are included in `sidebar.js` to display the small "cloud" icon: +- Features only available via DataHub Cloud should have the `saasOnly` class if they are included in `sidebar.js` to display the small "cloud" icon: ``` { diff --git a/docs-website/adoptionStoriesIndexes.json b/docs-website/adoptionStoriesIndexes.json new file mode 100644 index 00000000000000..3fe666ccf1c134 --- /dev/null +++ b/docs-website/adoptionStoriesIndexes.json @@ -0,0 +1,330 @@ +{ + "companies": [ + { + "name": "Netflix", + "slug": "netflix", + "imageUrl": "/img/logos/companies/netflix.png", + "imageSize": "large", + "link": "https://blog.datahubproject.io/how-netflix-is-collaborating-with-datahub-to-enhance-its-extensibility-a34d33f45947", + "linkType": "blog", + "tagline": "How Netflix is collaborating with DataHub to enhance its extensibility", + "category": "B2B & B2C", + "description": "\"DataHub gave us the extensibility features we needed to define new entity types easily and augment existing ones. DataHub performed exceptionally well in managing our traffic load and data volume. It offers a great developer experience, a well-documented taxonomy, and — very importantly — solid community support.\"

— Ajoy Majumdar, Software Architect at Netflix

" + }, + { + "name": "Visa", + "slug": "visa", + "imageUrl": "/img/logos/companies/visa.png", + "imageSize": "large", + "link": "https://blog.datahubproject.io/how-visa-uses-datahub-to-scale-data-governance-cace052d61c5", + "linkType": "blog", + "tagline": "How Visa uses DataHub to scale data governance", + "category": "Financial & Fintech", + "description": "\"We found DataHub to provide excellent coverage for our needs. What we appreciate most about DataHub is its powerful API platform.\"

— Jean-Pierre Dijcks, Sr. Dir. Product Management at VISA

" + }, + { + "name": "Optum", + "slug": "optum", + "imageUrl": "/img/logos/companies/optum.jpg", + "imageSize": "medium", + "link": "https://opensource.optum.com/blog/2022/03/23/data-mesh-via-datahub", + "linkType": "blog", + "tagline": "Data Mesh via DataHub", + "category": "And More", + "description": "“DataHub’s event driven architecture provides us a mechanism to act on any metadata changes in real time. This allows us to perform various actions like provisioning access to a data product, notifying consumers on any schema changes that may affect their application or triggering data movement jobs to move data from source to sink platforms.”" + }, + { + "name": "Pinterest", + "slug": "pinterest", + "imageUrl": "/img/logos/companies/pinterest.png", + "imageSize": "small", + "link": "https://www.youtube.com/watch?v=YoxTg8tQSwg&feature=youtu.be", + "linkType": "blog", + "tagline": "DataHub Project at Pinterest", + "category": "B2B & B2C", + "description": "Pinterest adopted a DataHub project to enhance metadata management for its big data query platform, facilitating better data navigation and understanding." + }, + { + "name": "Airtel", + "slug": "airtel", + "imageUrl": "/img/logos/companies/airtel.png", + "imageSize": "large", + "link": "https://www.youtube.com/watch?v=yr24mM91BN4", + "linkType": "video", + "tagline": "A transformative journey to Airtel's data mesh architecture with DataHub", + "category": "B2B & B2C", + "description": "Airtel is a leading global telecommunication provider. DataHub is the bedrock of Data Mesh at Airtel by providing the requisite governance and metadata management functionality to ensure their Data Products are discoverable, addressable, trustworthy, self-describing, and secure.

Get a closer look at how the Airtel team has successfully integrated DataHub to take their data mesh implementation to the next level." + }, + { + "name": "Coursera", + "slug": "coursera", + "imageUrl": "/img/logos/companies/coursera.svg", + "imageSize": "small", + "link": "https://www.youtube.com/watch?v=bd5v4fn4d4s", + "linkType": "video", + "tagline": "Coursera's DataHub Journey", + "category": "B2B & B2C", + "description": "“DataHub aligns with our needs [for] data documentation, a unified search experience, lineage information, and additional metadata. We are also very impressed with the vibrant and supportive community.”" + }, + { + "name": "Zynga", + "slug": "zynga", + "imageUrl": "/img/logos/companies/zynga.png", + "imageSize": "default", + "link": "https://www.youtube.com/watch?v=VCU3-Hd_glI", + "linkType": "video", + "tagline": "Zynga's DataHub Implementation", + "category": "B2B & B2C", + "description": "“We looked around for data catalog tool, and DataHub was a clear winner.”

Zynga levels up data management using DataHub, highlighting its role in enhancing data management, tracing data lineage, and ensuring data quality." + }, + { + "name": "Saxo Bank", + "slug": "saxo-bank", + "imageUrl": "/img/logos/companies/saxobank.svg", + "imageSize": "default", + "link": "https://blog.datahubproject.io/enabling-data-discovery-in-a-data-mesh-the-saxo-journey-451b06969c8f", + "linkType": "blog", + "tagline": "Enabling Data Discovery in a Data Mesh", + "category": "Financial & Fintech", + "description": "Saxo Bank adopted DataHub to enhance data quality and streamline governance, facilitating efficient data management through self-service capabilities.

By integrating Apache Kafka and Snowflake with DataHub, the bank embraced Data Mesh principles to democratize data, support rapid growth, and improve business processes." + }, + { + "name": "MediaMarkt Saturn", + "slug": "mediamarkt-saturn", + "imageUrl": "/img/logos/companies/mediamarkt-saturn.png", + "imageSize": "large", + "link": "https://www.youtube.com/watch?v=wsCFnElN_Wo", + "linkType": "video", + "tagline": "DataHub + MediaMarktSaturn Access Management Journey", + "category": "B2B & B2C", + "description": "Europe’s #1 consumer electronics retailer implemented DataHub for three reasons:

1. DataHub provides an extremely flexible and customizable metadata platform at scale.
2. Open-source means lower cost to implement and removes the headache of license management.
3. Community-driven project which continually evolves with industry trends and best practices." + }, + { + "name": "Adevinta", + "slug": "adevinta", + "imageUrl": "/img/logos/companies/adevinta.png", + "imageSize": "medium", + "link": "https://medium.com/@adevinta/building-the-data-catalogue-the-beginning-of-a-journey-d64e828f955c", + "linkType": "blog", + "tagline": "Building the data catalogue", + "category": "E-Commerce", + "description": "“DataHub allows us to solve the data discovery problem, which was a big challenge in our organization, and now we are solving it.”" + }, + { + "name": "Wolt", + "slug": "wolt", + "imageUrl": "/img/logos/companies/wolt.png", + "imageSize": "default", + "link": "https://blog.datahubproject.io/humans-of-datahub-fredrik-sannholm-d673b1877f2b", + "linkType": "blog", + "tagline": "Wolt's DataHub Integration", + "category": "E-Commerce", + "description": "“[DataHub] has made our legal team very happy with being able to keep track of our sensitive data [to answer questions like] Where’s it going? How’s it being processed? Where’s it ending up? Which third party tool or API’s are we sending it to and why? Who is responsible for this integration?”" + }, + { + "name": "Geotab", + "slug": "geotab", + "imageUrl": "/img/logos/companies/geotab.jpg", + "imageSize": "small", + "link": "https://www.youtube.com/watch?v=boyjT2OrlU4", + "linkType": "video", + "tagline": "Geotab's Experience with DataHub", + "category": "B2B & B2C", + "description": "“The key evaluation metric for selecting DataHub was the approachability and technical capabilities of its leading development team.”

Geotab’s data adoption journey explores challenges in data management, governance, and the decision to utilize DataHub for improved productivity and collaboration." + }, + { + "name": "Hurb", + "slug": "hurb", + "imageUrl": "/img/logos/companies/hurb.png", + "imageSize": "medium", + "link": "https://blog.datahubproject.io/humans-of-datahub-patrick-franco-braz-b02b55a4c5384", + "linkType": "blog", + "tagline": "Hurb's DataHub Journey", + "category": "B2B & B2C", + "description": "“The main points that drove our decision to implement DataHub were its user-friendly interface, active and receptive community, contribution opportunities, and built-in ingestion sources for our primary services.”

Hurb implemented DataHub to enhance data governance, streamline data access, and improve decision-making through a structured integration process." + }, + { + "name": "Grofers", + "slug": "grofers", + "imageUrl": "/img/logos/companies/grofers.png", + "imageSize": "medium", + "link": "https://www.youtube.com/watch?v=m9kUYAuezFI", + "linkType": "video", + "tagline": "Grofers' Success with DataHub", + "category": "E-Commerce", + "description": "Grofers provides a closer look into how their team has leveraged DataHub as the source of truth for data governance." + }, + { + "name": "Viasat", + "slug": "viasat", + "imageUrl": "/img/logos/companies/viasat.png", + "imageSize": "medium", + "link": "https://www.youtube.com/watch?v=2SrDAJnzkjE", + "linkType": "video", + "tagline": "Viasat's DataHub Implementation", + "category": "And More", + "description": "Viasat highlights why they chose DataHub over other open source and commercial technologies and their plans with it." + }, + { + "name": "LinkedIn", + "slug": "linkedin", + "imageUrl": "/img/logos/companies/linkedin.svg", + "imageSize": "medium", + "category": "B2B & B2C" + }, + { + "name": "Udemy", + "slug": "udemy", + "imageUrl": "/img/logos/companies/udemy.png", + "imageSize": "medium", + "category": "B2B & B2C" + }, + { + "name": "ThoughtWorks", + "slug": "thoughtworks", + "imageUrl": "/img/logos/companies/thoughtworks.png", + "imageSize": "medium", + "category": "B2B & B2C" + }, + { + "name": "Expedia Group", + "slug": "expedia-group", + "imageUrl": "/img/logos/companies/expedia.svg", + "imageSize": "medium", + "category": "B2B & B2C" + }, + { + "name": "Typeform", + "slug": "typeform", + "imageUrl": "/img/logos/companies/typeform.svg", + "imageSize": "medium", + "category": "B2B & B2C" + }, + { + "name": "Peloton", + "slug": "peloton", + "imageUrl": "/img/logos/companies/peloton.png", + "imageSize": "default", + "category": "B2B & B2C" + }, + { + "name": "Razer", + "slug": "razer", + "imageUrl": "/img/logos/companies/razer.jpeg", + "imageSize": "large", + "category": "B2B & B2C" + }, + { + "name": "ClassDojo", + "slug": "classdojo", + "imageUrl": "/img/logos/companies/classdojo.png", + "imageSize": "medium", + "category": "B2B & B2C" + }, + { + "name": "Klarna", + "slug": "klarna", + "imageUrl": "/img/logos/companies/klarna.svg", + "imageSize": "medium", + "category": "Financial & Fintech" + }, + { + "name": "N26", + "slug": "n26", + "imageUrl": "/img/logos/companies/n26.svg", + "imageSize": "medium", + "category": "Financial & Fintech" + }, + { + "name": "BankSalad", + "slug": "banksalad", + "imageUrl": "/img/logos/companies/banksalad.png", + "imageSize": "default", + "category": "Financial & Fintech" + }, + { + "name": "Uphold", + "slug": "uphold", + "imageUrl": "/img/logos/companies/uphold.png", + "imageSize": "default", + "category": "Financial & Fintech" + }, + { + "name": "Stash", + "slug": "stash", + "imageUrl": "/img/logos/companies/stash.svg", + "imageSize": "medium", + "category": "Financial & Fintech" + }, + { + "name": "SumUp", + "slug": "sumup", + "imageUrl": "/img/logos/companies/sumup.png", + "imageSize": "medium", + "category": "Financial & Fintech" + }, + { + "name": "VanMoof", + "slug": "vanmoof", + "imageUrl": "/img/logos/companies/vanmoof.png", + "imageSize": "small", + "category": "E-Commerce" + }, + { + "name": "SpotHero", + "slug": "spothero", + "imageUrl": "/img/logos/companies/spothero.png", + "imageSize": "default", + "category": "E-Commerce" + }, + { + "name": "hipages", + "slug": "hipages", + "imageUrl": "/img/logos/companies/hipages.png", + "imageSize": "medium", + "category": "E-Commerce" + }, + { + "name": "Showroomprive.com", + "slug": "showroomprive-com", + "imageUrl": "/img/logos/companies/showroomprive.png", + "imageSize": "small", + "category": "E-Commerce" + }, + { + "name": "Wikimedia Foundation", + "slug": "wikimedia-foundation", + "imageUrl": "/img/logos/companies/wikimedia-foundation.png", + "imageSize": "medium", + "category": "And More" + }, + { + "name": "Cabify", + "slug": "cabify", + "imageUrl": "/img/logos/companies/cabify.png", + "imageSize": "medium", + "category": "And More" + }, + { + "name": "Digital Turbine", + "slug": "digital-turbine", + "imageUrl": "/img/logos/companies/digitalturbine.svg", + "imageSize": "medium", + "category": "And More" + }, + { + "name": "DFDS", + "slug": "dfds", + "imageUrl": "/img/logos/companies/dfds.png", + "imageSize": "medium", + "category": "And More" + }, + { + "name": "Moloco", + "slug": "moloco", + "imageUrl": "/img/logos/companies/moloco.png", + "imageSize": "medium", + "category": "And More" + } + ] +} \ No newline at end of file diff --git a/docs-website/docusaurus.config.js b/docs-website/docusaurus.config.js index 5f823a93b4b439..1a40c986b31671 100644 --- a/docs-website/docusaurus.config.js +++ b/docs-website/docusaurus.config.js @@ -56,7 +56,7 @@ module.exports = { announcementBar: { id: "announcement", content: - '

Acryl DataHub  Acryl Data delivers an easy to consume DataHub platform for the enterprise

Sign Up for Acryl DataHub →', + '

DataHub Cloud  Acryl Data delivers an easy to consume DataHub platform for the enterprise

Sign Up for DataHub Cloud →', backgroundColor: "#070707", textColor: "#ffffff", isCloseable: false, @@ -76,6 +76,12 @@ module.exports = { label: "Docs", position: "right", }, + { + to: "/cloud", + activeBasePath: "cloud", + label: "Cloud", + position: "right", + }, { to: "/learn", activeBasePath: "learn", @@ -141,8 +147,8 @@ module.exports = { label: "YouTube", }, { - href: "https://www.youtube.com/playlist?list=PLdCtLs64vZvGCKMQC2dJEZ6cUqWsREbFi", - label: "Case Studies", + href: "/adoption-stories", + label: "Adoption Stories", }, { href: "https://www.youtube.com/playlist?list=PLdCtLs64vZvErAXMiqUYH9e63wyDaMBgg", diff --git a/docs-website/filterTagIndexes.json b/docs-website/filterTagIndexes.json index 2fa00120fd2cc3..e1e63ab5a9dbd4 100644 --- a/docs-website/filterTagIndexes.json +++ b/docs-website/filterTagIndexes.json @@ -103,7 +103,7 @@ "Path": "docs/generated/ingestion/sources/datahub", "imgPath": "img/acryl-logo-light-mark.png", "Title": "DataHub", - "Description": "Integrate your open source DataHub instance with Acryl Cloud or other on-prem DataHub instances", + "Description": "Integrate your open source DataHub instance with DataHub Cloud or other on-prem DataHub instances", "tags": { "Platform Type": "Metadata", "Connection Type": "Pull", diff --git a/docs-website/generateDocsDir.ts b/docs-website/generateDocsDir.ts index 9116218290d32d..23888d9000161d 100644 --- a/docs-website/generateDocsDir.ts +++ b/docs-website/generateDocsDir.ts @@ -176,7 +176,7 @@ const hardcoded_titles = { "docs/actions/README.md": "Introduction", "docs/actions/concepts.md": "Concepts", "docs/actions/quickstart.md": "Quickstart", - "docs/saas.md": "Managed DataHub", + "docs/saas.md": "DataHub Cloud", }; // titles that have been hardcoded in sidebars.js // (for cases where doc is reference multiple times with different titles) diff --git a/docs-website/package.json b/docs-website/package.json index 62d12888323036..58820fbf42b21b 100644 --- a/docs-website/package.json +++ b/docs-website/package.json @@ -48,6 +48,7 @@ "react-dom": "18.2.0", "sass": "^1.43.2", "swc-loader": "^0.2.6", + "swiper": "^11.1.4", "uuid": "^9.0.0" }, "browserslist": { diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js index a7aece48be3bcf..75fc1f2dcd0c5d 100644 --- a/docs-website/sidebars.js +++ b/docs-website/sidebars.js @@ -31,7 +31,11 @@ module.exports = { label: "Demo", href: "https://demo.datahubproject.io/", }, - "docs/what-is-datahub/customer-stories", + { + type: "link", + label: "Adoption Stories", + href: "/adoption-stories", + }, "docs/what-is-datahub/datahub-concepts", ], }, @@ -200,7 +204,7 @@ module.exports = { ], }, { - label: "Managed DataHub", + label: "DataHub Cloud", type: "category", collapsed: true, link: { @@ -254,6 +258,11 @@ module.exports = { id: "docs/managed-datahub/slack/saas-slack-app", className: "saasOnly", }, + { + type: "doc", + id: "docs/managed-datahub/slack/saas-slack-troubleshoot", + className: "saasOnly", + }, ], }, { @@ -280,7 +289,8 @@ module.exports = { className: "saasOnly", }, { - "Managed DataHub Release History": [ + "DataHub Cloud Release History": [ + "docs/managed-datahub/release-notes/v_0_3_4", "docs/managed-datahub/release-notes/v_0_3_3", "docs/managed-datahub/release-notes/v_0_3_2", "docs/managed-datahub/release-notes/v_0_3_1", @@ -810,7 +820,6 @@ module.exports = { }, { "API & SDK Guides": [ - "docs/api/tutorials/custom-properties", "docs/api/tutorials/datasets", "docs/api/tutorials/deprecation", "docs/api/tutorials/descriptions", diff --git a/docs-website/src/components/FeatureAvailability/index.js b/docs-website/src/components/FeatureAvailability/index.js index 27e7424432b599..b3e776952aa968 100644 --- a/docs-website/src/components/FeatureAvailability/index.js +++ b/docs-website/src/components/FeatureAvailability/index.js @@ -14,7 +14,7 @@ const FeatureAvailability = ({ saasOnly, ossOnly }) => (
- Managed DataHub {ossOnly ? : } + DataHub Cloud {ossOnly ? : }
diff --git a/docs-website/src/learn/_components/LearnListPage/index.jsx b/docs-website/src/learn/_components/LearnListPage/index.jsx index 4df87a340f21ee..1ceec9afa1e8a3 100644 --- a/docs-website/src/learn/_components/LearnListPage/index.jsx +++ b/docs-website/src/learn/_components/LearnListPage/index.jsx @@ -58,8 +58,9 @@ function BlogListPageContent(props) { For: {audiences.map((audience) => ( diff --git a/docs-website/src/learn/_components/LearnListPage/styles.module.scss b/docs-website/src/learn/_components/LearnListPage/styles.module.scss index d08b48a011de07..ce86e124afdb81 100644 --- a/docs-website/src/learn/_components/LearnListPage/styles.module.scss +++ b/docs-website/src/learn/_components/LearnListPage/styles.module.scss @@ -4,4 +4,10 @@ align-items: center; gap: 10px; flex-wrap: wrap; -} \ No newline at end of file + + .buttonActive { + background-color: var(--ifm-color-primary); + border: 1px solid var(--ifm-color-primary); + color: #ffffff; + } +} diff --git a/docs-website/src/learn/business-glossary.md b/docs-website/src/learn/business-glossary.md index d6b249617fc5ac..4568406a1667fc 100644 --- a/docs-website/src/learn/business-glossary.md +++ b/docs-website/src/learn/business-glossary.md @@ -91,7 +91,7 @@ Some companies use manual methods to track data terminology and manage access re ### Our Solution -Acryl DataHub offers comprehensive features designed to support the authoring of a unified business glossary for your organization: +DataHub Cloud offers comprehensive features designed to support the authoring of a unified business glossary for your organization:

diff --git a/docs-website/src/learn/business-metric.md b/docs-website/src/learn/business-metric.md index 39221a67d40abc..1378168f42195e 100644 --- a/docs-website/src/learn/business-metric.md +++ b/docs-website/src/learn/business-metric.md @@ -54,7 +54,7 @@ Some companies try to align metric definitions through emails and meetings. Whil ### Our Solution -Acryl DataHub offers comprehensive features designed to tackle the challenges of defining and standardizing business metrics: +DataHub Cloud offers comprehensive features designed to tackle the challenges of defining and standardizing business metrics:

@@ -72,13 +72,14 @@ Acryl DataHub offers comprehensive features designed to tackle the challenges of

- **[Approval Flows](https://datahubproject.io/docs/managed-datahub/approval-workflows):** Structured workflows for approving changes to metric definitions, maintaining accuracy and reliability. - - -![Untitled](https://prod-files-secure.s3.us-west-2.amazonaws.com/f818df0d-1067-44ab-99e1-8cf45d930c01/33ebd070-32a1-4875-b220-c31373f5eedf/Untitled.png) +

+ +
+ Lineage Tracking +

- **[Lineage Tracking](https://datahubproject.io/docs/generated/lineage/lineage-feature-guide):** Tools to track the origin and transformations of metrics, ensuring they align with standardized definitions. - - -![Screenshot 2024-07-10 at 12.07.28 PM.png](https://prod-files-secure.s3.us-west-2.amazonaws.com/f818df0d-1067-44ab-99e1-8cf45d930c01/39503957-ad64-4d2d-a5b2-b140abfc1f6c/Screenshot_2024-07-10_at_12.07.28_PM.png) By implementing these solutions, you can ensure that your business metrics are consistently defined and accurately used across all teams, supporting reliable analysis and decision-making. diff --git a/docs-website/src/learn/data-freshness.md b/docs-website/src/learn/data-freshness.md index e97e9b054b256d..528b2975f7528e 100644 --- a/docs-website/src/learn/data-freshness.md +++ b/docs-website/src/learn/data-freshness.md @@ -91,7 +91,7 @@ DataHub offers comprehensive features designed to tackle data freshness challeng

-**Freshness Monitoring & Alerting:** Automatically detect and alert when data freshness issues occur, to ensure timely updates by proactively monitoring key datasets for updates. Check out [Assertions](https://datahubproject.io/docs/managed-datahub/observe/assertions) and [Freshness Assertions](https://datahubproject.io/docs/managed-datahub/observe/freshness-assertions), Available in **Acryl Managed DataHub Only.** +**Freshness Monitoring & Alerting:** Automatically detect and alert when data freshness issues occur, to ensure timely updates by proactively monitoring key datasets for updates. Check out [Assertions](https://datahubproject.io/docs/managed-datahub/observe/assertions) and [Freshness Assertions](https://datahubproject.io/docs/managed-datahub/observe/freshness-assertions), Available in **DataHub Cloud Only.**

diff --git a/docs-website/src/learn/data-mesh.md b/docs-website/src/learn/data-mesh.md index f9a625d103ae71..038fcb971f5da2 100644 --- a/docs-website/src/learn/data-mesh.md +++ b/docs-website/src/learn/data-mesh.md @@ -90,7 +90,7 @@ While a centralized data lake or warehouse can simplify data governance by virtu ### Our Solution -Acryl DataHub offers a comprehensive set of features designed to support the implementation of a Data Mesh at your organization: +DataHub Cloud offers a comprehensive set of features designed to support the implementation of a Data Mesh at your organization: - **[Data Domains](https://datahubproject.io/docs/domains)**: Clearly define and manage data products within each business unit. - **[Data Products](https://datahubproject.io/docs/dataproducts):** Ensure each domain owns and manages its data products, promoting autonomy and agility. @@ -100,7 +100,7 @@ Acryl DataHub offers a comprehensive set of features designed to support the imp


- Data Contracts in Acryl DataHub UI + Data Contracts in DataHub Cloud UI

@@ -128,4 +128,4 @@ By implementing these solutions, you can effectively manage decentralized data, ## Conclusion -Implementing a Data Mesh can significantly improve your organization's ability to manage and leverage decentralized data. By understanding the benefits of data mesh and following best practices for implementation, you can overcome the limitations of centralized data systems and enhance your agility, scalability, and ability to generate insights. Acryl DataHub was built from the ground up to help you achieve this, providing the tools and features necessary to implement a large-scale Data Mesh successfully. \ No newline at end of file +Implementing a Data Mesh can significantly improve your organization's ability to manage and leverage decentralized data. By understanding the benefits of data mesh and following best practices for implementation, you can overcome the limitations of centralized data systems and enhance your agility, scalability, and ability to generate insights. DataHub Cloud was built from the ground up to help you achieve this, providing the tools and features necessary to implement a large-scale Data Mesh successfully. \ No newline at end of file diff --git a/docs-website/src/learn/data-pipeline.md b/docs-website/src/learn/data-pipeline.md index f5e5bb6615f48b..d57341c30a8c73 100644 --- a/docs-website/src/learn/data-pipeline.md +++ b/docs-website/src/learn/data-pipeline.md @@ -61,7 +61,7 @@ Some companies resort to manual debugging or use communication tools like Slack ### Our Solution -Acryl DataHub offers comprehensive features designed to optimize data pipelines: +DataHub Cloud offers comprehensive features designed to optimize data pipelines:

diff --git a/docs-website/src/pages/_components/CardCTAs/cardCTAs.module.scss b/docs-website/src/pages/_components/CardCTAs/cardCTAs.module.scss deleted file mode 100644 index fcd3666d03ddc9..00000000000000 --- a/docs-website/src/pages/_components/CardCTAs/cardCTAs.module.scss +++ /dev/null @@ -1,24 +0,0 @@ -.flexCol { - display: flex; -} - -.ctaCard { - flex-direction: row; - align-items: flex-start; - justify-content: space-between; - row-gap: 1rem; - padding: 1rem; - &:hover { - text-decoration: none; - border: 1px solid var(--ifm-color-primary); - background-color: var(--ifm-background-surface-color); - } - margin-bottom: 1rem; - flex: 1; -} - -.ctaHeading { - margin-bottom: 0; - display: flex; - align-items: center; -} diff --git a/docs-website/src/pages/_components/CardCTAs/index.js b/docs-website/src/pages/_components/CardCTAs/index.js deleted file mode 100644 index dc1b148d24bcd2..00000000000000 --- a/docs-website/src/pages/_components/CardCTAs/index.js +++ /dev/null @@ -1,52 +0,0 @@ -import React from "react"; -import clsx from "clsx"; -import styles from "./cardCTAs.module.scss"; -import useBaseUrl from "@docusaurus/useBaseUrl"; -import { ArrowRightOutlined } from "@ant-design/icons"; - -const cardsContent = [ - { - label: "Data Mesh", - title: "Data Products, Delivered", - url: "https://www.acryldata.io/blog/data-products-in-datahub-everything-you-need-to-know?utm_source=datahub&utm_medium=referral&utm_content=blog", - }, - { - label: "Data Contracts", - title: "Data Contracts: End-to-end Reliability in Data", - url: "https://www.acryldata.io/blog/data-contracts-in-datahub-combining-verifiability-with-holistic-data-management?utm_source=datahub&utm_medium=referral&utm_content=blog", - }, - { - label: "Shift Left", - title: "Data Governance and Lineage Impact Analysis", - url: "https://www.acryldata.io/blog/the-3-must-haves-of-metadata-management-part-2?utm_source=datahub&utm_medium=referral&utm_content=blog", - }, -]; - -const Card = ({ label, title, url }) => { - return ( -

- ); -}; - -const CardCTAs = () => - cardsContent?.length > 0 ? ( -
-
-
- {cardsContent.map((props, idx) => ( - - ))} -
-
-
- ) : null; - -export default CardCTAs; diff --git a/docs-website/src/pages/_components/Hero/hero.module.scss b/docs-website/src/pages/_components/Hero/hero.module.scss index 6e4a623f469d51..97bdceaef69366 100644 --- a/docs-website/src/pages/_components/Hero/hero.module.scss +++ b/docs-website/src/pages/_components/Hero/hero.module.scss @@ -42,58 +42,3 @@ } } } - -.quickLinks { - display: flex; - align-items: center; - justify-content: space-between; - padding: 1rem; - font-weight: bold; - margin-bottom: -2.5vh; - @media (min-width: 768px) { - flex-direction: row; - } - - > * { - padding: 0.5rem 1rem; - display: inline-block; - - @media (min-width: 768px) { - padding: 0 1rem; - } - } -} - -.quickLinksLabel { - display: flex; - align-items: center; - svg { - width: 24px; - height: 24px; - color: var(--ifm-text-color) !important; - margin-right: 0.5rem; - } -} - -.quickstartContent { - text-align: center; - padding: 2rem 0; - height: 100%; - margin: 2rem 0; - background: #34394d; - border-radius: var(--ifm-card-border-radius); -} - -.quickstartTitle { - color: #fafafa; -} - -.quickstartSubtitle { - font-size: 1.1rem; - color: gray; -} - -.quickstartCodeblock { - text-align: left; - padding: 0 20vh; -} diff --git a/docs-website/src/pages/_components/Hero/index.js b/docs-website/src/pages/_components/Hero/index.js index 17e5d0e7f4966a..12e41e2ecd1766 100644 --- a/docs-website/src/pages/_components/Hero/index.js +++ b/docs-website/src/pages/_components/Hero/index.js @@ -7,8 +7,8 @@ import { useColorMode } from "@docusaurus/theme-common"; import { QuestionCircleOutlined } from "@ant-design/icons"; import styles from "./hero.module.scss"; import CodeBlock from "@theme/CodeBlock"; -import CardCTAs from "../CardCTAs"; import TownhallButton from "../TownhallButton"; +import { Section } from "../Section"; const HeroAnnouncement = ({ message, linkUrl, linkText }) => (
@@ -50,33 +50,6 @@ const Hero = ({}) => {
- -
-

Get Started Now

-

Run the following command to get started with DataHub.

-
- - python3 -m pip install --upgrade pip wheel setuptools
- python3 -m pip install --upgrade acryl-datahub
- datahub docker quickstart - -
- - DataHub Quickstart Guide - - - Deploying With Kubernetes - -
-
-
- - Learn -
- What is DataHub? - How is DataHub architected? - See DataHub in action -
); diff --git a/docs-website/src/pages/_components/Logos/index.js b/docs-website/src/pages/_components/Logos/index.js index 3243617bcc40d6..565f6e9a46feec 100644 --- a/docs-website/src/pages/_components/Logos/index.js +++ b/docs-website/src/pages/_components/Logos/index.js @@ -1,204 +1,15 @@ -import React from "react"; import clsx from "clsx"; -import Tabs from "@theme/Tabs"; -import TabItem from "@theme/TabItem"; import Link from "@docusaurus/Link"; import useBaseUrl from "@docusaurus/useBaseUrl"; - +import React from "react"; +import { Swiper, SwiperSlide } from "swiper/react"; +import "swiper/css"; +import "swiper/css/pagination"; +import { Pagination } from "swiper/modules"; import styles from "./logos.module.scss"; +const companyIndexes = require("../../../../adoptionStoriesIndexes.json"); +const companies = companyIndexes.companies; -const companiesByIndustry = [ - { - name: "B2B & B2C", - companies: [ - { - name: "LinkedIn", - imageUrl: "/img/logos/companies/linkedin.svg", - imageSize: "medium", - }, - { - name: "Udemy", - imageUrl: "/img/logos/companies/udemy.png", - imageSize: "medium", - }, - { - name: "Airtel", - imageUrl: "/img/logos/companies/airtel.png", - imageSize: "large", - }, - { - name: "Coursera", - imageUrl: "/img/logos/companies/coursera.svg", - imageSize: "small", - }, - { - name: "Geotab", - imageUrl: "/img/logos/companies/geotab.jpg", - imageSize: "small", - }, - { - name: "ThoughtWorks", - imageUrl: "/img/logos/companies/thoughtworks.png", - imageSize: "medium", - }, - { - name: "Expedia Group", - imageUrl: "/img/logos/companies/expedia.svg", - imageSize: "medium", - }, - { - name: "Typeform", - imageUrl: "/img/logos/companies/typeform.svg", - imageSize: "medium", - }, - { - name: "Peloton", - imageUrl: "/img/logos/companies/peloton.png", - imageSize: "default", - }, - { - name: "Zynga", - imageUrl: "/img/logos/companies/zynga.png", - imageSize: "default", - }, - { - name: "Hurb", - imageUrl: "/img/logos/companies/hurb.png", - imageSize: "medium", - }, - { - name: "Razer", - imageUrl: "/img/logos/companies/razer.jpeg", - imageSize: "large", - }, - { - name: "ClassDojo", - imageUrl: "/img/logos/companies/classdojo.png", - imageSize: "medium", - }, - ], - }, - { - name: "Financial & Fintech", - companies: [ - { - name: "Saxo Bank", - imageUrl: "/img/logos/companies/saxobank.svg", - imageSize: "default", - }, - { - name: "Klarna", - imageUrl: "/img/logos/companies/klarna.svg", - imageSize: "medium", - }, - { - name: "N26", - imageUrl: "/img/logos/companies/n26.svg", - imageSize: "medium", - }, - { - name: "BankSalad", - imageUrl: "/img/logos/companies/banksalad.png", - imageSize: "default", - }, - { - name: "Uphold", - imageUrl: "/img/logos/companies/uphold.png", - imageSize: "default", - }, - { - name: "Stash", - imageUrl: "/img/logos/companies/stash.svg", - imageSize: "medium", - }, - { - name: "SumUp", - imageUrl: "/img/logos/companies/sumup.png", - imageSize: "medium", - }, - ], - }, - { - name: "E-Commerce", - companies: [ - { - name: "Adevinta", - imageUrl: "/img/logos/companies/adevinta.png", - imageSize: "medium", - }, - { - name: "VanMoof", - imageUrl: "/img/logos/companies/vanmoof.png", - imageSize: "small", - }, - { - name: "Grofers", - imageUrl: "/img/logos/companies/grofers.png", - imageSize: "medium", - }, - { - name: "SpotHero", - imageUrl: "/img/logos/companies/spothero.png", - imageSize: "default", - }, - { - name: "hipages", - imageUrl: "/img/logos/companies/hipages.png", - imageSize: "medium", - }, - { - name: "Wolt", - imageUrl: "/img/logos/companies/wolt.png", - imageSize: "default", - }, - { - name: "Showroomprive.com", - imageUrl: "/img/logos/companies/showroomprive.png", - imageSize: "small", - }, - ], - }, - { - name: "And More", - companies: [ - { - name: "Wikimedia Foundation", - imageUrl: "/img/logos/companies/wikimedia-foundation.png", - imageSize: "medium", - }, - { - name: "Cabify", - imageUrl: "/img/logos/companies/cabify.png", - imageSize: "medium", - }, - { - name: "Digital Turbine", - imageUrl: "/img/logos/companies/digitalturbine.svg", - imageSize: "medium", - }, - { - name: "Viasat", - imageUrl: "/img/logos/companies/viasat.png", - imageSize: "medium", - }, - { - name: "DFDS", - imageUrl: "/img/logos/companies/dfds.png", - imageSize: "medium", - }, - { - name: "Moloco", - imageUrl: "/img/logos/companies/moloco.png", - imageSize: "medium", - }, - { - name: "Optum", - imageUrl: "/img/logos/companies/optum.jpg", - imageSize: "medium", - }, - ], - }, -]; const platformLogos = [ { @@ -315,10 +126,19 @@ const platformLogos = [ ]; export const PlatformLogos = () => ( - +
{[...platformLogos, ...platformLogos].map((logo, idx) => ( - {logo.name} + {logo.name} ))}
@@ -326,22 +146,58 @@ export const PlatformLogos = () => ( export const CompanyLogos = () => (
- - {companiesByIndustry.map((industry, idx) => ( - -
- {industry.companies.map((company, idx) => ( + + {companies.map((company, idx) => ( + + {company.link ? ( + {company.name} - ))} -
- + + ) : ( + {company.name} + )} + ))} - +
); diff --git a/docs-website/src/pages/_components/Logos/logos.module.scss b/docs-website/src/pages/_components/Logos/logos.module.scss index b20cc9a48b247c..fd331bccb45563 100644 --- a/docs-website/src/pages/_components/Logos/logos.module.scss +++ b/docs-website/src/pages/_components/Logos/logos.module.scss @@ -1,3 +1,7 @@ +.pillTabs { + justify-content: center; +} + .marquee { width: 100%; overflow: hidden; @@ -35,7 +39,6 @@ } .companyWrapper { - background: #fff; display: flex; flex-wrap: wrap; align-items: center; @@ -45,16 +48,25 @@ filter: invert(1); mix-blend-mode: exclusion; } + + :global { + .swiper-wrapper { + display: flex; + align-items: center; + margin-bottom: 1rem; + .swiper-slide { + display: flex; + justify-content: center; + align-items: center; + height: 100%; + } + } + } } .companyLogoContainer { - display: flex; - align-items: center; justify-content: center; > div { - display: flex; - flex-direction: column; - align-items: center; ul[role="tablist"] { padding: 0 1rem; overflow-x: auto; @@ -79,23 +91,35 @@ } } +.companyLogoWithLink { + &:hover { + opacity: 1; + filter: grayscale(0%); + } +} + .companyLogo { - width: auto; + flex-shrink: 0; + width: 100%; + height: auto; mix-blend-mode: luminosity; opacity: 0.66; - margin: 2.5rem; - height: 60px; + filter: grayscale(100%); &.default { - height: 60px; + padding: 30px; } &.large { - height: 100px; + padding: 5px; } &.medium { - height: 30px; + padding: 15px; } &.small { - height: 20px; + padding: 10px; } } + +.swiper-pagination { + margin-top: 1rem; +} \ No newline at end of file diff --git a/docs-website/src/pages/_components/QuickstartContent/index.js b/docs-website/src/pages/_components/QuickstartContent/index.js new file mode 100644 index 00000000000000..8c942a6a2e440b --- /dev/null +++ b/docs-website/src/pages/_components/QuickstartContent/index.js @@ -0,0 +1,50 @@ +import React from "react"; +import clsx from "clsx"; +import Link from "@docusaurus/Link"; +import useBaseUrl from "@docusaurus/useBaseUrl"; +import Image from "@theme/IdealImage"; +import { useColorMode } from "@docusaurus/theme-common"; +import { QuestionCircleOutlined } from "@ant-design/icons"; +import styles from "./quickstartcontent.module.scss"; +import CodeBlock from "@theme/CodeBlock"; +import TownhallButton from "../TownhallButton"; +import { Section } from "../Section"; + + +const QuickstartContent = ({}) => { + const { colorMode } = useColorMode(); + return ( +
+
+
+

Get Started Now

+

Run the following command to get started with DataHub.

+
+ + python3 -m pip install --upgrade pip wheel setuptools
+ python3 -m pip install --upgrade acryl-datahub
+ datahub docker quickstart + +
+ + DataHub Quickstart Guide + + + Deploying With Kubernetes + +
+
+
+ + Learn +
+ What is DataHub? + How is DataHub architected? + See DataHub in action +
+
+
+ ); +}; + +export default QuickstartContent; diff --git a/docs-website/src/pages/_components/QuickstartContent/quickstartcontent.module.scss b/docs-website/src/pages/_components/QuickstartContent/quickstartcontent.module.scss new file mode 100644 index 00000000000000..e1badca6d2e348 --- /dev/null +++ b/docs-website/src/pages/_components/QuickstartContent/quickstartcontent.module.scss @@ -0,0 +1,67 @@ +.container { + margin-bottom: 2rem; +} + +.button { + text-decoration: none; + margin: 0.5rem 0 0 0; + white-space: nowrap; + @media (min-width: 690px) { + margin: 0 0 0 0.5rem; + } +} + +.quickstartContent { + text-align: center; + padding: 2rem 0; + height: 100%; + margin: 2rem 0; + background: #34394d; + border-radius: var(--ifm-card-border-radius); +} + +.quickstartTitle { + color: #fafafa; +} + +.quickstartSubtitle { + font-size: 1.1rem; + color: gray; +} + +.quickstartCodeblock { + text-align: left; + padding: 0 20vh; +} + +.quickLinks { + display: flex; + align-items: center; + justify-content: space-between; + padding: 1rem; + font-weight: bold; + margin-bottom: -2.5vh; + @media (min-width: 768px) { + flex-direction: row; + } + + > * { + padding: 0.5rem 1rem; + display: inline-block; + + @media (min-width: 768px) { + padding: 0 1rem; + } + } +} + +.quickLinksLabel { + display: flex; + align-items: center; + svg { + width: 24px; + height: 24px; + color: var(--ifm-text-color) !important; + margin-right: 0.5rem; + } +} diff --git a/docs-website/src/pages/_components/Quotes/index.js b/docs-website/src/pages/_components/Quotes/index.js index b66a04c2c6538e..664eba0b120519 100644 --- a/docs-website/src/pages/_components/Quotes/index.js +++ b/docs-website/src/pages/_components/Quotes/index.js @@ -32,7 +32,7 @@ const quotesContent = [ const Quote = ({ quote, company }) => { return ( -
+
{quote}
diff --git a/docs-website/src/pages/_components/Quotes/quotes.module.scss b/docs-website/src/pages/_components/Quotes/quotes.module.scss index 59573fa7a597a2..3bd895c4e3b243 100644 --- a/docs-website/src/pages/_components/Quotes/quotes.module.scss +++ b/docs-website/src/pages/_components/Quotes/quotes.module.scss @@ -8,6 +8,7 @@ .companyLogoWrapper { background: #fff; + height: 100px; html[data-theme="dark"] & { filter: invert(1); mix-blend-mode: exclusion; diff --git a/docs-website/src/pages/_components/Section/index.js b/docs-website/src/pages/_components/Section/index.js index 281b8e6928f3ae..6780273d313616 100644 --- a/docs-website/src/pages/_components/Section/index.js +++ b/docs-website/src/pages/_components/Section/index.js @@ -16,8 +16,8 @@ const PromoSection = () => (
-

Managed DataHub

-

Acryl Data provides a live demo of Managed DataHub every Tuesday and Thursday

+

DataHub Cloud

+

Acryl Data provides a live demo of DataHub Cloud every Tuesday and Thursday

Sign up for a live demo → diff --git a/docs-website/src/pages/adoption-stories/_components/LearnItemCard/index.jsx b/docs-website/src/pages/adoption-stories/_components/LearnItemCard/index.jsx new file mode 100644 index 00000000000000..67b94788d97800 --- /dev/null +++ b/docs-website/src/pages/adoption-stories/_components/LearnItemCard/index.jsx @@ -0,0 +1,25 @@ +import React from "react"; +import clsx from "clsx"; +import Link from "@docusaurus/Link"; +import styles from "./styles.module.scss"; + +const LearnItemCard = React.forwardRef(({ company, isSelected }, ref) => { + return ( +
+
+
+ {company.name} +
+
+
+
+ + Discover {company.name}'s Story + +
+
+
+ ); +}); + +export default LearnItemCard; diff --git a/docs-website/src/pages/adoption-stories/_components/LearnItemCard/styles.module.scss b/docs-website/src/pages/adoption-stories/_components/LearnItemCard/styles.module.scss new file mode 100644 index 00000000000000..881e90a7d09763 --- /dev/null +++ b/docs-website/src/pages/adoption-stories/_components/LearnItemCard/styles.module.scss @@ -0,0 +1,66 @@ +.featureCol { + display: flex; +} + +.card_date { + padding: 1rem 2rem; + font-size: 0.8rem; + font-style: italic; + color: gray; + margin-top: auto; +} + +.card_feature { + font-size: 2rem; + font-weight: 700; +} + +.card_button { + padding: 1rem; + text-align: center; +} + +.card { + color: var(--ifm-text-color); + text-decoration: none !important; + padding: 0rem; + margin-bottom: 2rem; + align-self: stretch; + flex-grow: 1; + &:hover { + opacity: 0.9; + } + + &.selected { + border-color: var(--ifm-color-primary); + box-shadow: rgba(0, 0, 0, 0.35) 0px 5px 15px; + scroll-margin-top: 100px; + } + + + hr { + margin: 0; + } +} + +.featureHeader { + h2 { + margin-bottom: 1rem !important; + font-size: 1.25rem; + } + padding: 1rem 2rem; +} + +.featureBody { + padding: 0 2rem; +} + +.card_image { + margin: 0; + margin-bottom: 0.5rem; + + img { + width: 100%; + height: auto; + } +} \ No newline at end of file diff --git a/docs-website/src/pages/adoption-stories/index.jsx b/docs-website/src/pages/adoption-stories/index.jsx new file mode 100644 index 00000000000000..27f4b876af20a6 --- /dev/null +++ b/docs-website/src/pages/adoption-stories/index.jsx @@ -0,0 +1,83 @@ +import React, { useState, useEffect, useRef } from "react"; +import Layout from "@theme/Layout"; +import BrowserOnly from "@docusaurus/BrowserOnly"; +import LearnItemCard from "./_components/LearnItemCard"; +import styles from "./styles.module.scss"; + +import customerStoriesIndexes from "../../../adoptionStoriesIndexes.json"; + +function AdoptionStoriesListPageContent() { + const companies = (customerStoriesIndexes?.companies || []).filter((company) => company.link); + const [activeFilters, setActiveFilters] = useState([]); + const categories = ["B2B & B2C", "E-Commerce", "Financial & Fintech", "And More"]; + const selectedCardRef = useRef(null); + + const filteredItems = activeFilters.length + ? companies.filter((company) => activeFilters.includes(company.category)) + : companies; + + const handleFilterToggle = (category) => { + if (activeFilters.includes(category)) { + setActiveFilters(activeFilters.filter((filter) => filter !== category)); + } else { + setActiveFilters([...new Set([...activeFilters, category])]); + } + }; + + useEffect(() => { + const selectedSlug = window.location.hash.substring(1); + if (selectedCardRef.current) { + selectedCardRef.current.scrollIntoView({ behavior: "smooth", block: "start", inline: "nearest" }); + } + }, [selectedCardRef]); + + return ( + +
+
+
+
+

DataHub Adoption Stories

+

Learn how the best data and AI teams are using DataHub +
+ Check out more stories on the DataHub Youtube. +

+
+
+
+ For: + {categories.map((category) => ( + + ))} +
+
+
+
+
+ {filteredItems.map((company) => ( + + ))} +
+
+ + ); +} + +export default function AdoptionStoriesListPage() { + return ( + + {() => } + + ); +} diff --git a/docs-website/src/pages/adoption-stories/styles.module.scss b/docs-website/src/pages/adoption-stories/styles.module.scss new file mode 100644 index 00000000000000..d08b48a011de07 --- /dev/null +++ b/docs-website/src/pages/adoption-stories/styles.module.scss @@ -0,0 +1,7 @@ +.filterBar { + display: flex; + justify-content: center; + align-items: center; + gap: 10px; + flex-wrap: wrap; +} \ No newline at end of file diff --git a/docs-website/src/pages/cloud/CompanyLogos/customersData.json b/docs-website/src/pages/cloud/CompanyLogos/customersData.json new file mode 100644 index 00000000000000..e8a7470eab4702 --- /dev/null +++ b/docs-website/src/pages/cloud/CompanyLogos/customersData.json @@ -0,0 +1,124 @@ +{ + "customers": [ + { + "link": { + "href": "https://www.depop.com/", + "blank": true + }, + "logo": { + "asset": { + "_ref": "https://cdn.sanity.io/images/cqo9wkgf/production/345d124a249d43bf0f20b608f8bfa2f7683311fa-360x180.png" + }, + "alt": "depop" + } + }, + { + "link": { + "href": "https://riskified.com", + "blank": true + }, + "logo": { + "asset": { + "_ref": "https://cdn.sanity.io/images/cqo9wkgf/production/c982e0459bed565273a9b696d9d40aed76f84b1e-360x180.png" + }, + "alt": "Riskified" + } + }, + { + "link": { + "href": "https://get.betterup.com/", + "blank": true + }, + "logo": { + "asset": { + "_ref": "https://cdn.sanity.io/images/cqo9wkgf/production/5988a55b3c090a12ddc3f3cae07b290ac3134771-360x180.png" + }, + "alt": "Betterup" + } + }, + { + "link": { + "href": "https://www.ovoenergy.com/", + "blank": true + }, + "logo": { + "asset": { + "_ref": "https://cdn.sanity.io/images/cqo9wkgf/production/5e7bd32dfbc769849dca136947ebd2fc2f5e91f3-540x270.png" + }, + "alt": "OVO Energy" + } + }, + { + "link": { + "href": "https://myob.com", + "blank": true + }, + "logo": { + "asset": { + "_ref": "https://cdn.sanity.io/images/cqo9wkgf/production/3d7c10e1bd7c7a062250e092d6d9d0553fb57790-360x180.png" + }, + "alt": "Myob" + } + }, + { + "link": { + "href": "https://www.dpgmediagroup.com/", + "blank": true + }, + "logo": { + "asset": { + "_ref": "https://cdn.sanity.io/images/cqo9wkgf/production/b446f595b4b13a72ee82a285924715f950e012ca-540x270.png" + }, + "alt": "DPG Megia" + } + }, + { + "link": { + "href": "https://www.notion.so/", + "blank": true + }, + "logo": { + "asset": { + "_ref": "https://cdn.sanity.io/images/cqo9wkgf/production/c2e84f93572cd1baf30ea7ab8da234ff44182eb6-540x270.png" + }, + "alt": "Notion" + } + }, + { + "link": { + "href": "https://www.sae.org/", + "blank": true + }, + "logo": { + "asset": { + "_ref": "https://cdn.sanity.io/images/cqo9wkgf/production/a9e8586635cb4039cbfc5836a6a5cacdeba9e6b3-540x270.png" + }, + "alt": "SAE International" + } + }, + { + "link": { + "href": "https://viator.com", + "blank": true + }, + "logo": { + "asset": { + "_ref": "https://cdn.sanity.io/images/cqo9wkgf/production/0c17141cad5baa053da18dffb17d9c182d242e69-1200x475.png" + }, + "alt": "Viator" + } + }, + { + "link": { + "href": "https://www.tripadvisor.co.uk/", + "blank": true + }, + "logo": { + "asset": { + "_ref": "https://cdn.sanity.io/images/cqo9wkgf/production/28255a28d261a074a83d1ee8632f0338bf5cf57e-1112x256.png" + }, + "alt": "Trip Advisor" + } + } + ] +} diff --git a/docs-website/src/pages/cloud/CompanyLogos/index.js b/docs-website/src/pages/cloud/CompanyLogos/index.js new file mode 100644 index 00000000000000..d395d5b50e48ac --- /dev/null +++ b/docs-website/src/pages/cloud/CompanyLogos/index.js @@ -0,0 +1,49 @@ +import React, { useEffect, useRef } from 'react'; +import Link from '@docusaurus/Link'; +import styles from './logos.module.scss'; +import customersData from './customersData.json'; +import clsx from 'clsx'; + +const ScrollingCustomers = ({ noOverlay = true, spacing, ...rest }) => { + const customers = customersData.customers; + const duplicatedLogos = [...customers, ...customers, ...customers]; + const scrollingRef = useRef(null); + + useEffect(() => { + if (scrollingRef.current) { + scrollingRef.current.style.setProperty('--customers-length', customers.length); + } + }, [customers.length]); + + return ( +
+
+ {duplicatedLogos.map((customer, index) => ( + + {customer.logo.alt} + + ))} +
+
+ ); +}; + +export default ScrollingCustomers; diff --git a/docs-website/src/pages/cloud/CompanyLogos/logos.module.scss b/docs-website/src/pages/cloud/CompanyLogos/logos.module.scss new file mode 100644 index 00000000000000..a6a9dba9d8d41f --- /dev/null +++ b/docs-website/src/pages/cloud/CompanyLogos/logos.module.scss @@ -0,0 +1,56 @@ +@keyframes scrollingCustomerAnimate { + 0% { + transform: translateX(0); + } + 100% { + transform: translateX(calc(var(--customers-length) * -220px)); + } +} + +@media (max-width: 767px) { + @keyframes scrollingCustomerAnimate { + 0% { + transform: translateX(0); + } + 100% { + transform: translateX(calc(var(--customers-length) * -166px)); + } + } +} + +.scrollingCustomers { + position: relative; + overflow: hidden; +} + + +.scrollingCustomers__inner { + display: flex; + padding: 1.25rem 0; + position: relative; + align-items: center; + animation: scrollingCustomerAnimate 15s linear infinite; +} + +.scrollingCustomers__inner img { + max-width: 160px !important; + min-width: unset; + filter: invert(1) brightness(0) contrast(100); // make image black +} + +@media (max-width: 767px) { + .scrollingCustomers__inner img { + width: 126px !important; + } +} + +.animateScrollingCustomers { + display: flex; + animation: scrollingCustomerAnimate 15s linear infinite; +} + +@media (max-width: 767px) { + .animateScrollingCustomers { + width: calc(var(--customers-length) * 126px); + } +} diff --git a/docs-website/src/pages/cloud/Enterprise/index.js b/docs-website/src/pages/cloud/Enterprise/index.js new file mode 100644 index 00000000000000..03cef99058ea2c --- /dev/null +++ b/docs-website/src/pages/cloud/Enterprise/index.js @@ -0,0 +1,48 @@ +import React from "react"; +import clsx from "clsx"; +import Link from "@docusaurus/Link"; +import styles from "./styles.module.scss"; + + +const featuresContent = [ + { + title: "99.5% Uptime SLA", + description: "We’ll focus on keeping DataHub running, so you can focus on the things that really matter." + }, + { + title: "In-VPC Ingestion and
Data Quality evaluation", + description: "So your actual data never leaves your network." + }, + { + title: "SOC-2 Compliant", + description: "An incredibly high level of security you can trust." + }, +]; + +const Feature = ({ title, description }) => { + return ( +
+

+

{description}

+
+ ); +}; + +const Features = () => + featuresContent?.length > 0 ? ( +
+
+
Enterprise Ready
+
+ {featuresContent.map((props, idx) => ( + + ))} +
+ + +4 benefits + +
+
+ ) : null; + +export default Features; diff --git a/docs-website/src/pages/cloud/Enterprise/styles.module.scss b/docs-website/src/pages/cloud/Enterprise/styles.module.scss new file mode 100644 index 00000000000000..9d95dcc3990b65 --- /dev/null +++ b/docs-website/src/pages/cloud/Enterprise/styles.module.scss @@ -0,0 +1,40 @@ + +.container { + padding: 2vh 6vh; + padding-bottom: 8vh; + display: flex; + justify-content: center; + align-items: center; +} +.wrapper { + display: flex; + justify-content: center; + flex-direction: column; + width: 90%; + max-width: 1200px; + +} + +.title { + font-size: 2.8rem; + font-weight: 600; + margin-bottom: 2rem; +} + +.moreBenefits { + font-weight: 400; + color: var(--ifm-color-primary); + + &:hover { + text-decoration: none; + opacity: 0.8; + } + +} + +@media (max-width: 767px) { + + .container { + padding: 0px 36px; + } +} \ No newline at end of file diff --git a/docs-website/src/pages/cloud/FeatureCards/index.js b/docs-website/src/pages/cloud/FeatureCards/index.js new file mode 100644 index 00000000000000..4a45cbcbe17174 --- /dev/null +++ b/docs-website/src/pages/cloud/FeatureCards/index.js @@ -0,0 +1,130 @@ +import React from "react"; +import clsx from "clsx"; +import styles from "./styles.module.scss"; +import Link from "@docusaurus/Link"; +import { CheckCircleOutlined } from "@ant-design/icons"; + +const data = { + sections: [ + { + title: "Data Discovery", + icon: "/img/assets/data-discovery.svg", + cloudPageLink: "https://www.acryldata.io/acryl-datahub", + cloudBenefits: [ + { text: "Enhanced search ranking", link: "" }, // → + { text: "Personalization for every persona", link: "" }, // → + { text: "A browser extension for BI Tools", link: "" }, // → + { text: "AI-Powered Documentation", link: "" }, // → + { text: "No-Code Automations", link: "" }, // → + ], + coreBenefits: [ + { text: "Integrations for 50+ data sources"}, + { text: "Lineage for tables, columns, and jobs"}, + ], + hiddenCoreBenefits: "+4 benefits", + hiddenCoreBenefitsLink: '/docs/managed-datahub/managed-datahub-overview#search-and-discovery', + }, + { + title: "Data Observability", + icon: "/img/assets/data-ob.svg", + cloudPageLink: "https://www.acryldata.io/observe", + cloudBenefits: [ + { text: "Continuous data quality monitors", link: "" }, // → + { text: "End-to-end data incident tracking & management", link: "" }, // → + { text: "AI-Driven anomaly detection", link: "" }, // → + { text: "Executive Data Health Dashboard", link: "" }, // → + { text: "On-demand data quality evaluation via APIs & UI", link: "" }, // → + ], + coreBenefits: [ + { text: "Surface data quality results"}, + { text: "Create and manage data contracts" }, + ], + hiddenCoreBenefits: "+1 benefit", + hiddenCoreBenefitsLink: '/docs/managed-datahub/managed-datahub-overview#data-observability', + }, + { + title: "Data Governance", + icon: "/img/assets/data-governance.svg", + cloudPageLink: "https://www.acryldata.io/acryl-datahub#governance", + cloudBenefits: [ + { text: "Human-assisted asset certification workflows", link: "" }, // → + { text: "Automations to enforce governance standards", link: "" }, // → + { text: "End-to-end glossary management workflows", link: "" }, // → + { text: "Ownership management workflows", link: "" }, // → + ], + coreBenefits: [ + { text: "Shift-left governance"}, + { text: "Business glossaries"}, + ], + hiddenCoreBenefits: "+1 benefit", + hiddenCoreBenefitsLink: '/docs/managed-datahub/managed-datahub-overview#data-governance', + }, + ], +}; + +const Features = () => ( +
+ {data.sections.map((section, sectionIndex) => ( +
+
+
+
+
+ {section.title} +
{section.title}
+
+
+
+
DataHub Cloud includes:
+
+ {section.cloudBenefits.map((benefit, index) => ( +
+ + +
{benefit.text}
+ +
+ ))} +
+ + Explore in DataHub Cloud + +
+
+
+
+

{`In addition to DataHub Core:`}

+
+ {section.coreBenefits.map((benefit, index) => ( +
+ +
{benefit.text}
+
+ ))} +
+ {section.hiddenCoreBenefits} +
+
+
+
+ ))} +
+); + +export default Features; diff --git a/docs-website/src/pages/cloud/FeatureCards/styles.module.scss b/docs-website/src/pages/cloud/FeatureCards/styles.module.scss new file mode 100644 index 00000000000000..cfba5e217374d8 --- /dev/null +++ b/docs-website/src/pages/cloud/FeatureCards/styles.module.scss @@ -0,0 +1,164 @@ +.container { + padding: 48px 20px; +} + +.rowItem { + justify-content: center; +} + +.cardGroup { + display: flex; + justify-content: space-between; + position: relative; + align-items: flex-start; + margin: 0rem 1.2rem 2rem 1.2rem; +} + +.cardGroupInverse { + flex-direction: row-reverse; + .coreBenefitCard { + margin-left: 15rem; + } +} + +.title { + display: flex; + align-items: center; + margin: 2rem; + + .icon { + width: 4rem; + height: 4rem; + margin-right: 1rem; + } + + .titleText { + font-size: 32px; + font-weight: 700; + } +} + +.card { + background: white; + padding: 40px; + margin: 0; +} + +.sectionTitle { + font-size: 24px; + font-weight: 600; + line-height: 33px; +} + +.benefitIcon { + width: 32px; + height: 32px; + align-items: center; +} + +.moreBenefits { + font-family: Manrope; + font-size: 20px; + font-weight: 400; + line-height: 30px; + text-align: left; + color: rgba(24, 144, 255, 1); + + &:hover { + text-decoration: none; + opacity: 0.8; + } +} + +.featureList { + display: grid; + margin-top: 1rem; + font-weight: 400; + font-size: 20px; + text-align: left; + line-height: 27px; +} + +.exploreButton { + margin-top: 24px; +} + +.cloudBenefitCard { + border-radius: 16px; + border: 2px solid rgba(24, 144, 255, 1); + box-shadow: 0px 4px 25px 0px rgba(24, 144, 255, 0.25); + background: linear-gradient(180deg, #ffffff 27%, #e4f7ff 87%); + min-width: 543px; + z-index: 2; + position: relative; +} + +.coreBenefitCard { + border-radius: 16px; + border: 1px solid rgba(227, 227, 227, 1); + min-width: 451px; + z-index: 1; + position: relative; + margin-top: 14rem; + margin-left: -48px; + padding-left: 72px; +} + +.reversedRow .coreBenefitCard { + margin-left: auto!important; + padding-left: auto!important; + margin-right: -48px; + padding-right: 72px; +} + +.coreBenefitCard h3 { + margin-bottom: 2rem; +} +.coreBenefit { + display: flex; + flex-direction: row; + margin-bottom: 1rem; +} + +.cloudBenefit { + color: rgba(24, 144, 255, 1); + margin-bottom: 1rem; +} +.cloudBenefitLink { + display: flex; + flex-direction: row; +} +.cloudBenefitLinkText { + display: block; +} + +.reversedRow { + flex-direction: row-reverse; +} + +.reversedRow .title { + justify-content: flex-end; +} + +@media (max-width: 768px) { + .cardGroup { + flex-direction: column; + align-items: center; + } + + .cardGroupInverse { + flex-direction: column; + } + + .cloudBenefitCard, .coreBenefitCard { + min-width: 100%; + margin-top: 20px; + } + + .coreBenefitCard { + margin-top: -40px; + margin-left: 0 !important; + margin-right: 0 !important; + padding-top: 80px; + } +} diff --git a/docs-website/src/pages/cloud/UnifiedTabs/index.js b/docs-website/src/pages/cloud/UnifiedTabs/index.js new file mode 100644 index 00000000000000..0c7ff50ce54471 --- /dev/null +++ b/docs-website/src/pages/cloud/UnifiedTabs/index.js @@ -0,0 +1,76 @@ +import React, { useState } from 'react'; +import { CaretUpFilled } from '@ant-design/icons'; +import styles from './styles.module.scss'; +import clsx from 'clsx'; + +const TabbedComponent = () => { + const [activeTab, setActiveTab] = useState(0); + + const tabs = [ + { + title: 'Data Discovery', + description: 'All the search and discovery features of DataHub Core you already love, enhanced.', + icon: "/img/assets/data-discovery.svg", + link: "https://www.acryldata.io/acryl-datahub", + image: 'https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/saas/demo/discovery.webm', + }, + { + title: 'Data Observability', + description: 'Detect, resolve, and prevent data quality issues before they impact your business. Unify data health signals from all your data quality tools, including dbt tests and more.', + icon: "/img/assets/data-ob.svg", + link: "https://www.acryldata.io/observe", + image: 'https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/saas/demo/observe.webm', + }, + { + title: 'Data Governance', + description: 'Powerful Automation, Reporting and Organizational tools to help you govern effectively.', + icon: "/img/assets/data-governance.svg", + link: "https://www.acryldata.io/acryl-datahub#governance", + image: 'https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/saas/demo/governance.webm', + }, + ]; + + return ( +
+
One platform to rule them all
+
+
+ {tabs.map((tab, index) => ( + +
+ + {activeTab === index && ( +
+

{tab.description}

+ Learn More → +
+ )} +
+ {activeTab === index && ( +
+
+
+
+ )} + + ))} +
+
+
+
+
+
+
+ ); +}; + +export default TabbedComponent; diff --git a/docs-website/src/pages/cloud/UnifiedTabs/styles.module.scss b/docs-website/src/pages/cloud/UnifiedTabs/styles.module.scss new file mode 100644 index 00000000000000..7549f743053552 --- /dev/null +++ b/docs-website/src/pages/cloud/UnifiedTabs/styles.module.scss @@ -0,0 +1,179 @@ +.tabbedComponent { + text-align: center; + padding: 20px; + padding-top: 48px; + padding-bottom: 48px; + display: flex; + flex-direction: column; + align-items: center; +} + +.title { + font-size: 2.5rem; + font-weight: 500; + margin-bottom: 2rem; + text-align: center; + position: relative; + padding: 0 2rem; + display: block; + width: 100%; + + &:before, &:after { + content: " "; + height: 2px; + width: calc((100vw - 468px - 120px)/2); + background: #EEE; + display: block; + position: absolute; + top: 50%; + } + + &:before { + left: 0; + } + + &:after { + right: 0; + } +} + +.container { + display: flex; + flex-direction: column; // Changed to column for mobile view + background: white; + border-radius: 1rem; + overflow: hidden; + box-shadow: 0 4px 4px rgba(0, 0, 0, 0.1); + margin: 0rem 5rem; + max-width: 1200px; + width: 100%; +} + +.tabs { + display: flex; + flex-direction: column; + justify-content: flex-start; +} + +.tab { + align-items: center; + margin: 0; + border-bottom: 1px solid #e0e0e0; + position: relative; + + &.activeTab { + border-bottom: none; + } +} + +.activeTab .tabButton { + border-bottom: 1px solid #f1f1f1; +} + +.tabButton { + padding: 1.5rem; + background: none; + border: none; + cursor: pointer; + display: flex; + align-items: center; + width: 100%; + justify-content: space-between; + + .tabButton:focus { + outline: none; + } + + .tabTitle { + font-size: 1.5rem; + font-weight: 700; + margin-left: 1rem; + flex-grow: 1; + text-align: left; + } + + .icon { + width: 2.3rem; + height: 2.3rem; + } +} + +.arrow { + margin-left: auto; + transition: transform 0.2s; + color: black; +} + +.upsideDown { + transform: rotate(180deg); +} + +.dropdown { + background-color: #ffffff; + margin-top: 5px; + text-align: left; + padding: 1rem 1.5rem; + + .learnMore { + font-weight: 600; + font-size: 15px; + line-height: 23px; + + &:hover { + text-decoration: none; + opacity: 0.8; + } + } +} + +.imageContainer { + display: none; + justify-content: center; + align-items: center; + background-color: transparent; + margin: 1rem 0; // Added margin for spacing + height: 380px; + border-radius: 24px; +} + +.tabImage { + width: 100%; + height: 100%; + display: flex; + padding: 4px; + align-items: center; + justify-content: center; +} +.tabImage video { + max-height: 100%; + max-width: 100%; + border-radius: 22px; + box-shadow: 0px 4px 12px rgba(0, 0, 0, 0.1); +} + +.mobileImageContainer { + @media (max-width: 767px) { + display: flex; + } +} + +.webImageContainer { + @media (min-width: 768px) { + display: flex; + flex-grow: 1; + } +} + +@media (min-width: 768px) { + .container { + flex-direction: row; // Change back to row for larger screens + padding: 40px 32px; + } + .tabs { + width: 440px; + } + + .imageContainer { + margin: 1rem; // Reset margin for larger screens + } +} diff --git a/docs-website/src/pages/cloud/index.js b/docs-website/src/pages/cloud/index.js new file mode 100644 index 00000000000000..5166d80bf3b7b0 --- /dev/null +++ b/docs-website/src/pages/cloud/index.js @@ -0,0 +1,90 @@ +import React from "react"; +import Layout from "@theme/Layout"; +import Link from "@docusaurus/Link"; +import useDocusaurusContext from "@docusaurus/useDocusaurusContext"; +import Enterprise from "./Enterprise"; +import { Section } from "../_components/Section"; +import ScrollingCustomers from "./CompanyLogos"; +import clsx from "clsx"; +import styles from "./styles.module.scss"; +import UnifiedTabs from "./UnifiedTabs"; +import FeatureCards from "./FeatureCards"; + + +function Home() { + const context = useDocusaurusContext(); + const { siteConfig = {} } = context; + // const { colorMode } = useColorMode(); + + + if (siteConfig.customFields.isSaas) { + window.location.replace("/docs"); + } + + return !siteConfig.customFields.isSaas ? ( + +
+
+
+
+

Try DataHub Cloud

+
+ Introducing DataHub as a Managed Service +
with Data Observability and Data Governance built-in.
+ {/* */} +
+ + Book Demo + + + Product Tour + +
+
+
+
+ +
+ +
+ +
+
+ +
+
+
+
+
+
+

Get your free trial.

+
Data Discovery, Data Quality and Data Governance unified.
+ + + Book Demo + + + Product Tour + +
+
+ An extension of the DataHub Core project.
+ View Cloud Docs. + +
+
+
+
+
+ + ) : null; +} + +export default Home; diff --git a/docs-website/src/pages/cloud/styles.module.scss b/docs-website/src/pages/cloud/styles.module.scss new file mode 100644 index 00000000000000..d1ac31f3ef8cc0 --- /dev/null +++ b/docs-website/src/pages/cloud/styles.module.scss @@ -0,0 +1,51 @@ +.link { + &:hover { + text-decoration: none; + opacity: 0.8; + } +} + +.bgSection { + background-color: #FAFAFA !important; + } + +.hero { + :global { + .button { + margin-right: 1rem; + } + } + .hero__title { + font-size: 4rem; + } + + .hero__secondtitle { + font-size: 2rem; + font-weight: 300; + margin-bottom: 2rem; + + } + + .hero__subtitle { + margin-bottom: 2rem; + font-size: 1.75rem; + line-height: 2.5rem; + } + + .buttonLightBlue { + color: #1990FF !important; + background: #EAF3FF; + border-color: #EAF3FF; + :hover { + background: #D6E7FF; + } + } + + .learnMore { + font-size: 1.25rem; + font-weight: 600; + margin-top: 0.5rem; + + } + } + diff --git a/docs-website/src/pages/docs/_components/CustomerCard/customercard.module.scss b/docs-website/src/pages/docs/_components/CustomerCard/customercard.module.scss deleted file mode 100644 index 349f705d25b10d..00000000000000 --- a/docs-website/src/pages/docs/_components/CustomerCard/customercard.module.scss +++ /dev/null @@ -1,56 +0,0 @@ -.card { - color: var(--ifm-hero-text-color); - padding: 0; - margin: 0rem 3rem 2rem 0rem; - text-decoration: none !important; - - .card_button { - padding: 0rem 0rem 0rem 1rem; - text-align: right; - } - - .card_img { - justify-content: center; - display: flex; - height: 250px; - margin: 0; - position: relative; - text-align: center; - } - - .card_body { - padding: 2rem 3rem 2rem 3rem; - - .card_description { - min-height: 20rem; - } - } - - .card_overlay_text { - position: absolute; - text-align: left; - width: 80%; - top: 50%; - left: 50%; - transform: translate(-50%, -50%); - color: white; - - .card_customer { - font-size: 3.2rem; - font-weight: 800; - line-height: 1.2; - - } - .card_title { - font-size: 1.2rem; - font-weight: 600; - } - - } - - img { - object-fit: cover; - filter: brightness(50%); - } - -} diff --git a/docs-website/src/pages/docs/_components/CustomerCard/index.jsx b/docs-website/src/pages/docs/_components/CustomerCard/index.jsx deleted file mode 100644 index 36c83226e1f732..00000000000000 --- a/docs-website/src/pages/docs/_components/CustomerCard/index.jsx +++ /dev/null @@ -1,30 +0,0 @@ -import React from "react"; -import clsx from "clsx"; -import styles from "./customercard.module.scss"; -import Link from "@docusaurus/Link"; - -const CustomerCard = ({ customer, title, imgUrl, description, to,}) => { - return ( -
-
-
- {customer} -
-
{customer}
-
{title}
-
-
-
-

{description}

-
- - Discover {customer}'s Story - -
-
-
-
- ); -}; - -export default CustomerCard; diff --git a/docs-website/src/pages/docs/_components/CustomerCardSection/index.jsx b/docs-website/src/pages/docs/_components/CustomerCardSection/index.jsx index ca34d89df8701d..505a2810c9433c 100644 --- a/docs-website/src/pages/docs/_components/CustomerCardSection/index.jsx +++ b/docs-website/src/pages/docs/_components/CustomerCardSection/index.jsx @@ -57,7 +57,7 @@ const customerCardContent = [ 3. Community-driven project which continually evolves with industry trends and best practices ), - to: "https://www.acryldata.io/blog/data-contracts-in-datahub-combining-verifiability-with-holistic-data-management?utm_source=datahub&utm_medium=referral&utm_content=blog", + to: "https://youtu.be/wsCFnElN_Wo?si=i-bNAQAsbHJq5O9-", }, { customer: "Airtel", @@ -75,7 +75,7 @@ const customerCardContent = [ DataHub to take their data mesh implementation to the next level. ), - to: "https://youtu.be/wsCFnElN_Wo?si=i-bNAQAsbHJq5O9-", + to: "https://www.youtube.com/watch?v=yr24mM91BN4", }, ]; @@ -93,4 +93,4 @@ const CustomerCardSection = () => { ); }; -export default CustomerCardSection; +export default CustomerCardSection; \ No newline at end of file diff --git a/docs-website/src/pages/docs/_components/QuickstartCards/index.jsx b/docs-website/src/pages/docs/_components/QuickstartCards/index.jsx index bcb77c043f1d0b..9f582967175edd 100644 --- a/docs-website/src/pages/docs/_components/QuickstartCards/index.jsx +++ b/docs-website/src/pages/docs/_components/QuickstartCards/index.jsx @@ -10,7 +10,7 @@ const quickstartContent = [ fontColor: '#091013', }, { - title: "Learn about Managed DataHub", + title: "Learn about DataHub Cloud", icon: "acryl-logo-transparent-mark", to: "managed-datahub/managed-datahub-overview", color: '#091013', diff --git a/docs-website/src/pages/index.js b/docs-website/src/pages/index.js index 68b177d10f7aff..2eed41b4ad1bd3 100644 --- a/docs-website/src/pages/index.js +++ b/docs-website/src/pages/index.js @@ -3,13 +3,14 @@ import Layout from "@theme/Layout"; import Link from "@docusaurus/Link"; import useDocusaurusContext from "@docusaurus/useDocusaurusContext"; import CodeBlock from "@theme/CodeBlock"; - +import useBaseUrl from "@docusaurus/useBaseUrl"; import Hero from "./_components/Hero"; import Features from "./_components/Features"; -import Quotes from "./_components/Quotes"; import { Section, PromoSection } from "./_components/Section"; -import { PlatformLogos, CompanyLogos } from "./_components/Logos"; +import { PlatformLogos } from "./_components/Logos"; import RoundedImage from "./_components/RoundedImage"; +import { CompanyLogos } from "./_components/Logos"; +import QuickstartContent from "./_components/QuickstartContent"; const example_recipe = ` source: @@ -38,6 +39,18 @@ function Home() { description="DataHub is a data discovery application built on an extensible data catalog that helps you tame the complexity of diverse data ecosystems." > +
+ +
+ + Check Out Adoption Stories → + +
+
+
@@ -157,10 +170,6 @@ function Home() {
-
- - -
) : null; } diff --git a/docs-website/static/img/adoption-stories/adoption-stories-adevinta.png b/docs-website/static/img/adoption-stories/adoption-stories-adevinta.png new file mode 100644 index 00000000000000..6c790995843c54 Binary files /dev/null and b/docs-website/static/img/adoption-stories/adoption-stories-adevinta.png differ diff --git a/docs-website/static/img/adoption-stories/adoption-stories-airtel.png b/docs-website/static/img/adoption-stories/adoption-stories-airtel.png new file mode 100644 index 00000000000000..ae5ebdedd47aa1 Binary files /dev/null and b/docs-website/static/img/adoption-stories/adoption-stories-airtel.png differ diff --git a/docs-website/static/img/adoption-stories/adoption-stories-coursera.png b/docs-website/static/img/adoption-stories/adoption-stories-coursera.png new file mode 100644 index 00000000000000..4f473874d0dc26 Binary files /dev/null and b/docs-website/static/img/adoption-stories/adoption-stories-coursera.png differ diff --git a/docs-website/static/img/adoption-stories/adoption-stories-geotab.png b/docs-website/static/img/adoption-stories/adoption-stories-geotab.png new file mode 100644 index 00000000000000..2b3c8a158273a9 Binary files /dev/null and b/docs-website/static/img/adoption-stories/adoption-stories-geotab.png differ diff --git a/docs-website/static/img/adoption-stories/adoption-stories-grofers.png b/docs-website/static/img/adoption-stories/adoption-stories-grofers.png new file mode 100644 index 00000000000000..51af8a3ad69d7b Binary files /dev/null and b/docs-website/static/img/adoption-stories/adoption-stories-grofers.png differ diff --git a/docs-website/static/img/adoption-stories/adoption-stories-hurb.png b/docs-website/static/img/adoption-stories/adoption-stories-hurb.png new file mode 100644 index 00000000000000..b7b8bae5d8c321 Binary files /dev/null and b/docs-website/static/img/adoption-stories/adoption-stories-hurb.png differ diff --git a/docs-website/static/img/adoption-stories/adoption-stories-mediamarkt-saturn.png b/docs-website/static/img/adoption-stories/adoption-stories-mediamarkt-saturn.png new file mode 100644 index 00000000000000..ac2f524a7a0e77 Binary files /dev/null and b/docs-website/static/img/adoption-stories/adoption-stories-mediamarkt-saturn.png differ diff --git a/docs-website/static/img/adoption-stories/adoption-stories-netflix.png b/docs-website/static/img/adoption-stories/adoption-stories-netflix.png new file mode 100644 index 00000000000000..de65a4c59419b5 Binary files /dev/null and b/docs-website/static/img/adoption-stories/adoption-stories-netflix.png differ diff --git a/docs-website/static/img/adoption-stories/adoption-stories-optum.png b/docs-website/static/img/adoption-stories/adoption-stories-optum.png new file mode 100644 index 00000000000000..051abaa96a0e01 Binary files /dev/null and b/docs-website/static/img/adoption-stories/adoption-stories-optum.png differ diff --git a/docs-website/static/img/adoption-stories/adoption-stories-pinterest.png b/docs-website/static/img/adoption-stories/adoption-stories-pinterest.png new file mode 100644 index 00000000000000..e005ea6d5750aa Binary files /dev/null and b/docs-website/static/img/adoption-stories/adoption-stories-pinterest.png differ diff --git a/docs-website/static/img/adoption-stories/adoption-stories-saxo-bank.png b/docs-website/static/img/adoption-stories/adoption-stories-saxo-bank.png new file mode 100644 index 00000000000000..333003d146cf5e Binary files /dev/null and b/docs-website/static/img/adoption-stories/adoption-stories-saxo-bank.png differ diff --git a/docs-website/static/img/adoption-stories/adoption-stories-viasat.png b/docs-website/static/img/adoption-stories/adoption-stories-viasat.png new file mode 100644 index 00000000000000..b6f633450296c6 Binary files /dev/null and b/docs-website/static/img/adoption-stories/adoption-stories-viasat.png differ diff --git a/docs-website/static/img/adoption-stories/adoption-stories-visa.png b/docs-website/static/img/adoption-stories/adoption-stories-visa.png new file mode 100644 index 00000000000000..11d732faf85fec Binary files /dev/null and b/docs-website/static/img/adoption-stories/adoption-stories-visa.png differ diff --git a/docs-website/static/img/adoption-stories/adoption-stories-wolt.png b/docs-website/static/img/adoption-stories/adoption-stories-wolt.png new file mode 100644 index 00000000000000..43501a1f2f6d57 Binary files /dev/null and b/docs-website/static/img/adoption-stories/adoption-stories-wolt.png differ diff --git a/docs-website/static/img/adoption-stories/adoption-stories-zynga.png b/docs-website/static/img/adoption-stories/adoption-stories-zynga.png new file mode 100644 index 00000000000000..94ee9e9b2fb8ee Binary files /dev/null and b/docs-website/static/img/adoption-stories/adoption-stories-zynga.png differ diff --git a/docs-website/static/img/adoption-stories/img.png b/docs-website/static/img/adoption-stories/img.png new file mode 100644 index 00000000000000..4d4971018c3982 Binary files /dev/null and b/docs-website/static/img/adoption-stories/img.png differ diff --git a/docs-website/static/img/assets/business.jpg b/docs-website/static/img/assets/business.jpg deleted file mode 100644 index f5a91928ee2ad8..00000000000000 Binary files a/docs-website/static/img/assets/business.jpg and /dev/null differ diff --git a/docs-website/static/img/assets/data-discovery.svg b/docs-website/static/img/assets/data-discovery.svg new file mode 100644 index 00000000000000..1a6c6f36a231ae --- /dev/null +++ b/docs-website/static/img/assets/data-discovery.svg @@ -0,0 +1,4 @@ + + + + diff --git a/docs-website/static/img/assets/data-governance.svg b/docs-website/static/img/assets/data-governance.svg new file mode 100644 index 00000000000000..b1db48e3e76bfe --- /dev/null +++ b/docs-website/static/img/assets/data-governance.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/docs-website/static/img/assets/data-ob.svg b/docs-website/static/img/assets/data-ob.svg new file mode 100644 index 00000000000000..d630b0a2333a2d --- /dev/null +++ b/docs-website/static/img/assets/data-ob.svg @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/docs-website/static/img/assets/netflix.jpg b/docs-website/static/img/assets/netflix.jpg deleted file mode 100644 index 8b555f5b63187f..00000000000000 Binary files a/docs-website/static/img/assets/netflix.jpg and /dev/null differ diff --git a/docs-website/static/img/assets/phonecall.jpg b/docs-website/static/img/assets/phonecall.jpg deleted file mode 100644 index 87e48f28213827..00000000000000 Binary files a/docs-website/static/img/assets/phonecall.jpg and /dev/null differ diff --git a/docs-website/static/img/assets/travel.jpg b/docs-website/static/img/assets/travel.jpg deleted file mode 100644 index de2697f5631217..00000000000000 Binary files a/docs-website/static/img/assets/travel.jpg and /dev/null differ diff --git a/docs-website/static/img/cloud-bg.png b/docs-website/static/img/cloud-bg.png new file mode 100644 index 00000000000000..392aec2ff936c5 Binary files /dev/null and b/docs-website/static/img/cloud-bg.png differ diff --git a/docs-website/static/img/logos/companies/mediamarkt-saturn.png b/docs-website/static/img/logos/companies/mediamarkt-saturn.png new file mode 100644 index 00000000000000..6e3a39e0ae34b1 Binary files /dev/null and b/docs-website/static/img/logos/companies/mediamarkt-saturn.png differ diff --git a/docs-website/static/img/logos/companies/netflix.png b/docs-website/static/img/logos/companies/netflix.png new file mode 100755 index 00000000000000..151775b3b17bc4 Binary files /dev/null and b/docs-website/static/img/logos/companies/netflix.png differ diff --git a/docs-website/static/img/logos/companies/pinterest.png b/docs-website/static/img/logos/companies/pinterest.png new file mode 100644 index 00000000000000..715c8c33fd85b4 Binary files /dev/null and b/docs-website/static/img/logos/companies/pinterest.png differ diff --git a/docs-website/static/img/logos/companies/visa.png b/docs-website/static/img/logos/companies/visa.png new file mode 100644 index 00000000000000..0a0198bfb76a28 Binary files /dev/null and b/docs-website/static/img/logos/companies/visa.png differ diff --git a/docs-website/static/img/logos/scrollingCompanies/acertus.webp b/docs-website/static/img/logos/scrollingCompanies/acertus.webp new file mode 100644 index 00000000000000..20ff1c6d7d554f Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/acertus.webp differ diff --git a/docs-website/static/img/logos/scrollingCompanies/autoscout24.webp b/docs-website/static/img/logos/scrollingCompanies/autoscout24.webp new file mode 100644 index 00000000000000..27b34c6f724dee Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/autoscout24.webp differ diff --git a/docs-website/static/img/logos/scrollingCompanies/betterup.webp b/docs-website/static/img/logos/scrollingCompanies/betterup.webp new file mode 100644 index 00000000000000..268e019de8fd4a Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/betterup.webp differ diff --git a/docs-website/static/img/logos/scrollingCompanies/depop.webp b/docs-website/static/img/logos/scrollingCompanies/depop.webp new file mode 100644 index 00000000000000..7c006bb8620607 Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/depop.webp differ diff --git a/docs-website/static/img/logos/scrollingCompanies/dpg-media.png b/docs-website/static/img/logos/scrollingCompanies/dpg-media.png new file mode 100644 index 00000000000000..40022ef0294d73 Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/dpg-media.png differ diff --git a/docs-website/static/img/logos/scrollingCompanies/dpg_media.webp b/docs-website/static/img/logos/scrollingCompanies/dpg_media.webp new file mode 100644 index 00000000000000..0d5c42847068a1 Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/dpg_media.webp differ diff --git a/docs-website/static/img/logos/scrollingCompanies/event-callout-img.png b/docs-website/static/img/logos/scrollingCompanies/event-callout-img.png new file mode 100644 index 00000000000000..032c34fb6a10ec Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/event-callout-img.png differ diff --git a/docs-website/static/img/logos/scrollingCompanies/graham-stirling.png b/docs-website/static/img/logos/scrollingCompanies/graham-stirling.png new file mode 100644 index 00000000000000..14bb23dcef0d94 Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/graham-stirling.png differ diff --git a/docs-website/static/img/logos/scrollingCompanies/icon-check.svg b/docs-website/static/img/logos/scrollingCompanies/icon-check.svg new file mode 100644 index 00000000000000..83a4d0983ebe0e --- /dev/null +++ b/docs-website/static/img/logos/scrollingCompanies/icon-check.svg @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + diff --git a/docs-website/static/img/logos/scrollingCompanies/log-saxo.svg b/docs-website/static/img/logos/scrollingCompanies/log-saxo.svg new file mode 100644 index 00000000000000..f6ab4ae071b10c --- /dev/null +++ b/docs-website/static/img/logos/scrollingCompanies/log-saxo.svg @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/docs-website/static/img/logos/scrollingCompanies/myob.png b/docs-website/static/img/logos/scrollingCompanies/myob.png new file mode 100644 index 00000000000000..c161532e650ba4 Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/myob.png differ diff --git a/docs-website/static/img/logos/scrollingCompanies/myob.webp b/docs-website/static/img/logos/scrollingCompanies/myob.webp new file mode 100644 index 00000000000000..509e3fba0804c5 Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/myob.webp differ diff --git a/docs-website/static/img/logos/scrollingCompanies/notion-logo.png b/docs-website/static/img/logos/scrollingCompanies/notion-logo.png new file mode 100644 index 00000000000000..f65884f53ead89 Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/notion-logo.png differ diff --git a/docs-website/static/img/logos/scrollingCompanies/notion.webp b/docs-website/static/img/logos/scrollingCompanies/notion.webp new file mode 100644 index 00000000000000..393756855d938c Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/notion.webp differ diff --git a/docs-website/static/img/logos/scrollingCompanies/ovo_energy.webp b/docs-website/static/img/logos/scrollingCompanies/ovo_energy.webp new file mode 100644 index 00000000000000..3e08a80f5c5f42 Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/ovo_energy.webp differ diff --git a/docs-website/static/img/logos/scrollingCompanies/regeneron.png b/docs-website/static/img/logos/scrollingCompanies/regeneron.png new file mode 100644 index 00000000000000..0a660b95c6d503 Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/regeneron.png differ diff --git a/docs-website/static/img/logos/scrollingCompanies/regeneron.webp b/docs-website/static/img/logos/scrollingCompanies/regeneron.webp new file mode 100644 index 00000000000000..45bed965f3a01c Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/regeneron.webp differ diff --git a/docs-website/static/img/logos/scrollingCompanies/riskified.png b/docs-website/static/img/logos/scrollingCompanies/riskified.png new file mode 100644 index 00000000000000..69b94b43fd56e8 Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/riskified.png differ diff --git a/docs-website/static/img/logos/scrollingCompanies/riskified.webp b/docs-website/static/img/logos/scrollingCompanies/riskified.webp new file mode 100644 index 00000000000000..a2a2f96d5ea7be Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/riskified.webp differ diff --git a/docs-website/static/img/logos/scrollingCompanies/robinhood.png b/docs-website/static/img/logos/scrollingCompanies/robinhood.png new file mode 100644 index 00000000000000..e75535a383f324 Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/robinhood.png differ diff --git a/docs-website/static/img/logos/scrollingCompanies/robinhood.webp b/docs-website/static/img/logos/scrollingCompanies/robinhood.webp new file mode 100644 index 00000000000000..661c25c0dd8302 Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/robinhood.webp differ diff --git a/docs-website/static/img/logos/scrollingCompanies/saxo_bank.webp b/docs-website/static/img/logos/scrollingCompanies/saxo_bank.webp new file mode 100644 index 00000000000000..a4c1aae73fe48b Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/saxo_bank.webp differ diff --git a/docs-website/static/img/logos/scrollingCompanies/screenshot.png b/docs-website/static/img/logos/scrollingCompanies/screenshot.png new file mode 100644 index 00000000000000..59d982c5aec6d5 Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/screenshot.png differ diff --git a/docs-website/static/img/logos/scrollingCompanies/twilio-bg.png b/docs-website/static/img/logos/scrollingCompanies/twilio-bg.png new file mode 100644 index 00000000000000..74dcbf88a35951 Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/twilio-bg.png differ diff --git a/docs-website/static/img/logos/scrollingCompanies/twilio.png b/docs-website/static/img/logos/scrollingCompanies/twilio.png new file mode 100644 index 00000000000000..f226674d0ffbce Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/twilio.png differ diff --git a/docs-website/static/img/logos/scrollingCompanies/twilio.webp b/docs-website/static/img/logos/scrollingCompanies/twilio.webp new file mode 100644 index 00000000000000..5ad47d5d5c87e2 Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/twilio.webp differ diff --git a/docs-website/static/img/logos/scrollingCompanies/xero.png b/docs-website/static/img/logos/scrollingCompanies/xero.png new file mode 100644 index 00000000000000..653ddfb2c76869 Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/xero.png differ diff --git a/docs-website/static/img/logos/scrollingCompanies/xero.webp b/docs-website/static/img/logos/scrollingCompanies/xero.webp new file mode 100644 index 00000000000000..9f2b4cc0cf0f9f Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/xero.webp differ diff --git a/docs-website/static/img/logos/scrollingCompanies/zendesk.webp b/docs-website/static/img/logos/scrollingCompanies/zendesk.webp new file mode 100644 index 00000000000000..e790fdc2af6eda Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/zendesk.webp differ diff --git a/docs-website/yarn.lock b/docs-website/yarn.lock index a93b0e74c327db..0970a59cbc00a3 100644 --- a/docs-website/yarn.lock +++ b/docs-website/yarn.lock @@ -1827,7 +1827,7 @@ "@docusaurus/theme-search-algolia" "2.4.3" "@docusaurus/types" "2.4.3" -"@docusaurus/react-loadable@5.5.2": +"@docusaurus/react-loadable@5.5.2", "react-loadable@npm:@docusaurus/react-loadable@5.5.2": version "5.5.2" resolved "https://registry.yarnpkg.com/@docusaurus/react-loadable/-/react-loadable-5.5.2.tgz#81aae0db81ecafbdaee3651f12804580868fa6ce" integrity sha512-A3dYjdBGuy0IGT+wyLIGIKLRE+sAk1iNk0f1HjNDysO7u8lhL4N3VEm+FAubmJbAztn94F7MxBTPmnixbiyFdQ== @@ -9705,14 +9705,6 @@ react-loadable-ssr-addon-v5-slorber@^1.0.1: dependencies: "@babel/runtime" "^7.10.3" -"react-loadable@npm:@docusaurus/react-loadable@5.5.2": - version "5.5.2" - resolved "https://registry.yarnpkg.com/@docusaurus/react-loadable/-/react-loadable-5.5.2.tgz#81aae0db81ecafbdaee3651f12804580868fa6ce" - integrity sha512-A3dYjdBGuy0IGT+wyLIGIKLRE+sAk1iNk0f1HjNDysO7u8lhL4N3VEm+FAubmJbAztn94F7MxBTPmnixbiyFdQ== - dependencies: - "@types/react" "*" - prop-types "^15.6.2" - react-markdown@^8.0.6: version "8.0.7" resolved "https://registry.yarnpkg.com/react-markdown/-/react-markdown-8.0.7.tgz#c8dbd1b9ba5f1c5e7e5f2a44de465a3caafdf89b" @@ -10866,6 +10858,11 @@ swc-loader@^0.2.6: dependencies: "@swc/counter" "^0.1.3" +swiper@^11.1.4: + version "11.1.4" + resolved "https://registry.yarnpkg.com/swiper/-/swiper-11.1.4.tgz#2f8e303e8bf9e5bc40a3885fc637ae60ff27996c" + integrity sha512-1n7kbYJB2dFEpUHRFszq7gys/ofIBrMNibwTiMvPHwneKND/t9kImnHt6CfGPScMHgI+dWMbGTycCKGMoOO1KA== + symbol-observable@^1.0.4: version "1.2.0" resolved "https://registry.yarnpkg.com/symbol-observable/-/symbol-observable-1.2.0.tgz#c22688aed4eab3cdc2dfeacbb561660560a00804" diff --git a/docs/_feature-guide-template.md b/docs/_feature-guide-template.md index 9c1aead5e13ab3..f03dffb957a796 100644 --- a/docs/_feature-guide-template.md +++ b/docs/_feature-guide-template.md @@ -5,9 +5,9 @@ import FeatureAvailability from '@site/src/components/FeatureAvailability'; diff --git a/docs/actions/actions/slack.md b/docs/actions/actions/slack.md index a89439825d2da1..73d990948812d5 100644 --- a/docs/actions/actions/slack.md +++ b/docs/actions/actions/slack.md @@ -136,9 +136,9 @@ In the next steps, we'll show you how to configure the Slack Action based on the ### Installation Instructions (Deployment specific) -#### Managed DataHub +#### DataHub Cloud -Head over to the [Configuring Notifications](../../managed-datahub/slack/saas-slack-setup.md#configuring-notifications) section in the Managed DataHub guide to configure Slack notifications for your Managed DataHub instance. +Head over to the [Configuring Notifications](../../managed-datahub/slack/saas-slack-setup.md#configuring-notifications) section in the DataHub Cloud guide to configure Slack notifications for your DataHub Cloud instance. #### Quickstart diff --git a/docs/actions/guides/developing-a-transformer.md b/docs/actions/guides/developing-a-transformer.md index a843dbc846cd51..6406cdfae6104b 100644 --- a/docs/actions/guides/developing-a-transformer.md +++ b/docs/actions/guides/developing-a-transformer.md @@ -23,7 +23,7 @@ print the configuration that is provided when it is created, and print any Event ```python # custom_transformer.py from datahub_actions.transform.transformer import Transformer -from datahub_actions.event.event import EventEnvelope +from datahub_actions.event.event_envelope import EventEnvelope from datahub_actions.pipeline.pipeline_context import PipelineContext from typing import Optional @@ -75,7 +75,7 @@ Next, install the package pip install -e . ``` -inside the module. (alt.`python setup.py`). +inside the module. (alt.`python setup.py`). Once we have done this, our class will be referencable via `custom_transformer_example.custom_transformer:CustomTransformer`. @@ -96,7 +96,7 @@ source: connection: bootstrap: ${KAFKA_BOOTSTRAP_SERVER:-localhost:9092} schema_registry_url: ${SCHEMA_REGISTRY_URL:-http://localhost:8081} -transform: +transform: - type: "custom_transformer_example.custom_transformer:CustomTransformer" config: # Some sample configuration which should be printed on create. @@ -130,4 +130,4 @@ it without defining the full module path. Prerequisites to consideration for inclusion in the core Transformer library include - **Testing** Define unit tests for your Transformer -- **Deduplication** Confirm that no existing Transformer serves the same purpose, or can be easily extended to serve the same purpose \ No newline at end of file +- **Deduplication** Confirm that no existing Transformer serves the same purpose, or can be easily extended to serve the same purpose diff --git a/docs/api/graphql/getting-started.md b/docs/api/graphql/getting-started.md index 98aeca196600d7..dfa556051bd4d1 100644 --- a/docs/api/graphql/getting-started.md +++ b/docs/api/graphql/getting-started.md @@ -27,6 +27,7 @@ For more information on, please refer to the following links." - [Querying for Domain of a Dataset](/docs/api/tutorials/domains.md#read-domains) - [Querying for Glossary Terms of a Dataset](/docs/api/tutorials/terms.md#read-terms) - [Querying for Deprecation of a dataset](/docs/api/tutorials/deprecation.md#read-deprecation) +- [Querying for all DataJobs that belong to a DataFlow](/docs/lineage/airflow.md#get-all-datajobs-associated-with-a-dataflow) ### Search diff --git a/docs/api/tutorials/assertions.md b/docs/api/tutorials/assertions.md index f89fe728f7e977..6837220a581c11 100644 --- a/docs/api/tutorials/assertions.md +++ b/docs/api/tutorials/assertions.md @@ -5,7 +5,7 @@ import TabItem from '@theme/TabItem'; -This guide specifically covers how to use the Assertion APIs for **Acryl Cloud** native assertions, including: +This guide specifically covers how to use the Assertion APIs for **DataHub Cloud** native assertions, including: - [Freshness Assertions](/docs/managed-datahub/observe/freshness-assertions.md) - [Volume Assertions](/docs/managed-datahub/observe/volume-assertions.md) @@ -15,7 +15,7 @@ This guide specifically covers how to use the Assertion APIs for **Acryl Cloud** ## Why Would You Use Assertions APIs? -The Assertions APIs allow you to create, schedule, run, and delete Assertions with Acryl Cloud. +The Assertions APIs allow you to create, schedule, run, and delete Assertions with DataHub Cloud. ### Goal Of This Guide diff --git a/docs/api/tutorials/data-contracts.md b/docs/api/tutorials/data-contracts.md index ac19920a5c4b7b..e977345273e223 100644 --- a/docs/api/tutorials/data-contracts.md +++ b/docs/api/tutorials/data-contracts.md @@ -5,7 +5,7 @@ import TabItem from '@theme/TabItem'; -This guide specifically covers how to use the Data Contract APIs with **Acryl Cloud**. +This guide specifically covers how to use the Data Contract APIs with **DataHub Cloud**. ## Why Would You Use Data Contract APIs? diff --git a/docs/api/tutorials/forms.md b/docs/api/tutorials/forms.md index f60699ffebab58..3f28353595be72 100644 --- a/docs/api/tutorials/forms.md +++ b/docs/api/tutorials/forms.md @@ -22,7 +22,7 @@ For detailed information, please refer to [Datahub Quickstart Guide](/docs/quick -Install the relevant CLI version. Forms are available as of CLI version `0.13.1`. The corresponding SaaS release version is `v0.2.16.5` +Install the relevant CLI version. Forms are available as of CLI version `0.13.1`. The corresponding DataHub Cloud release version is `v0.2.16.5` Connect to your instance via [init](https://datahubproject.io/docs/cli/#init): 1. Run `datahub init` to update the instance you want to load into diff --git a/docs/api/tutorials/operations.md b/docs/api/tutorials/operations.md index 70ede993ec95f6..e1d41f80e68366 100644 --- a/docs/api/tutorials/operations.md +++ b/docs/api/tutorials/operations.md @@ -7,7 +7,7 @@ import TabItem from '@theme/TabItem'; The Operations APIs allow you to report operational changes that were made to a given Dataset or Table using the 'Operation' concept. These operations may be viewed on the Dataset Profile (e.g. as last modified time), accessed via the DataHub GraphQL API, or -used to as inputs to Acryl Cloud [Freshness Assertions](/docs/managed-datahub/observe/freshness-assertions.md). +used as inputs to DataHub Cloud [Freshness Assertions](/docs/managed-datahub/observe/freshness-assertions.md). ### Goal Of This Guide diff --git a/docs/api/tutorials/structured-properties.md b/docs/api/tutorials/structured-properties.md index 940f4632f1d45f..c56a2848638fc2 100644 --- a/docs/api/tutorials/structured-properties.md +++ b/docs/api/tutorials/structured-properties.md @@ -32,7 +32,7 @@ Additionally, you need to have the following tools installed according to the me -Install the relevant CLI version. Forms are available as of CLI version `0.13.1`. The corresponding SaaS release version is `v0.2.16.5` +Install the relevant CLI version. Forms are available as of CLI version `0.13.1`. The corresponding DataHub Cloud release version is `v0.2.16.5` Connect to your instance via [init](https://datahubproject.io/docs/cli/#init): - Run `datahub init` to update the instance you want to load into. diff --git a/docs/assertions/open-assertions-spec.md b/docs/assertions/open-assertions-spec.md index 519e917c30587f..09dad4710a2e53 100644 --- a/docs/assertions/open-assertions-spec.md +++ b/docs/assertions/open-assertions-spec.md @@ -2,7 +2,7 @@ DataHub is developing an open-source Data Quality Assertions Specification & Compiler that will allow you to declare data quality checks / expectations / assertions using a simple, universal YAML-based format, and then compile this into artifacts that can be registered or directly executed by 3rd party Data Quality tools like [Snowflake DMFs](https://docs.snowflake.com/en/user-guide/data-quality-intro), -dbt tests, Great Expectations or Acryl Cloud natively. +dbt tests, Great Expectations or DataHub Cloud natively. Ultimately, our goal is to provide an framework-agnostic, highly-portable format for defining Data Quality checks, making it seamless to swap out the underlying assertion engine without service disruption for end consumers of the results of these data quality checks in catalogging tools like DataHub. diff --git a/docs/authentication/guides/add-users.md b/docs/authentication/guides/add-users.md index 86dac3ea328e53..30da5c9f229f94 100644 --- a/docs/authentication/guides/add-users.md +++ b/docs/authentication/guides/add-users.md @@ -60,7 +60,7 @@ and many more. This option is strongly recommended for production deployments of DataHub. -### Managed DataHub +### DataHub Cloud Single Sign-On can be configured and enabled by navigating to **Settings** > **SSO** > **OIDC**. Note that a user must have the **Manage Platform Settings** [Platform Privilege](../../authorization/access-policies-guide.md) diff --git a/docs/authorization/policies.md b/docs/authorization/policies.md index 9867ff6ab264dd..91b0241c7d5149 100644 --- a/docs/authorization/policies.md +++ b/docs/authorization/policies.md @@ -105,7 +105,7 @@ These privileges are for DataHub operators to access & manage the administrative | Manage Monitors[^2] | Allow actor to create, update, and delete any data asset monitors, including Custom SQL monitors. Grant with care. | [^1]: Only active if REST_API_AUTHORIZATION_ENABLED is true -[^2]: Managed DataHub only +[^2]: DataHub Cloud only ##### Common metadata privileges These privileges are to view & modify any entity within DataHub. @@ -143,10 +143,10 @@ These privileges are to view & modify any entity within DataHub. | Manage Tag Proposals[^2] | Allow actor to manage a proposal to add a tag to an asset. | | Manage Glossary Term Proposals[^2] | Allow actor to manage a proposal to add a glossary term to an asset. | | Manage Documentation Proposals[^2] | Allow actor to manage a proposal update an asset's documentation | -| Share Entity[^2] | Allow actor to share an entity with another Acryl instance. | +| Share Entity[^2] | Allow actor to share an entity with another DataHub Cloud instance. | [^1]: Only active if REST_API_AUTHORIZATION_ENABLED is true -[^2]: Managed DataHub only +[^2]: DataHub Cloud only ##### Specific entity-level privileges These privileges are not generalizable. diff --git a/docs/authorization/roles.md b/docs/authorization/roles.md index fe41cae2bc3cc3..7c7b4581faffca 100644 --- a/docs/authorization/roles.md +++ b/docs/authorization/roles.md @@ -73,9 +73,9 @@ in DataHub. ### Role Privileges -#### Self-Hosted DataHub and Managed DataHub +#### Self-Hosted DataHub and DataHub Cloud -These privileges are common to both Self-Hosted DataHub and Managed DataHub. +These privileges are common to both Self-Hosted DataHub and DataHub Cloud. ##### Platform Privileges @@ -146,9 +146,9 @@ These privileges are common to both Self-Hosted DataHub and Managed DataHub. | Explain ElasticSearch Query API | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | The ability to use the Operations API explain endpoint. | | Produce Platform Event API | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to produce Platform Events using the API. | -#### Managed DataHub +#### DataHub Cloud -These privileges are only relevant to Managed DataHub. +These privileges are only relevant to DataHub Cloud. ##### Platform Privileges @@ -178,7 +178,7 @@ These privileges are only relevant to Managed DataHub. | Manage Group Notification Settings | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to manage notification settings for a group. | | Manage Group Subscriptions | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to manage subscriptions for a group. | | Manage Data Contract Proposals | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to manage a proposal for a Data Contract | -| Share Entity | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to share an entity with another Acryl instance. | +| Share Entity | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to share an entity with another DataHub Cloud instance. | ## Additional Resources diff --git a/docs/cli.md b/docs/cli.md index fdcfa6616c9bfc..1f1e6dfa26be71 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -102,6 +102,7 @@ Command Options: --test-source-connection When set, ingestion will only test the source connection details from the recipe --no-progress If enabled, mute intermediate progress ingestion reports ``` + #### ingest --dry-run The `--dry-run` option of the `ingest` command performs all of the ingestion steps, except writing to the sink. This is useful to validate that the @@ -133,23 +134,8 @@ By default `--preview` creates 10 workunits. But if you wish to try producing mo datahub ingest -c ./examples/recipes/example_to_datahub_rest.dhub.yaml -n --preview --preview-workunits=20 ``` -#### ingest deploy - -The `ingest deploy` command instructs the cli to upload an ingestion recipe to DataHub to be run by DataHub's [UI Ingestion](./ui-ingestion.md). -This command can also be used to schedule the ingestion while uploading or even to update existing sources. It will upload to the remote instance the -CLI is connected to, not the sink of the recipe. Use `datahub init` to set the remote if not already set. - -To schedule a recipe called "test", to run at 5am everyday, London time with the recipe configured in a local `recipe.yaml` file: -````shell -datahub ingest deploy --name "test" --schedule "5 * * * *" --time-zone "Europe/London" -c recipe.yaml -```` - -To update an existing recipe please use the `--urn` parameter to specify the id of the recipe to update. - -**Note:** Updating a recipe will result in a replacement of the existing options with what was specified in the cli command. -I.e: Not specifying a schedule in the cli update command will remove the schedule from the recipe to be updated. - #### ingest --no-default-report + By default, the cli sends an ingestion report to DataHub, which allows you to see the result of all cli-based ingestion in the UI. This can be turned off with the `--no-default-report` flag. ```shell @@ -180,6 +166,52 @@ failure_log: filename: ./path/to/failure.json ``` +### ingest deploy + +The `ingest deploy` command instructs the cli to upload an ingestion recipe to DataHub to be run by DataHub's [UI Ingestion](./ui-ingestion.md). +This command can also be used to schedule the ingestion while uploading or even to update existing sources. It will upload to the remote instance the +CLI is connected to, not the sink of the recipe. Use `datahub init` to set the remote if not already set. + +This command will automatically create a new recipe if it doesn't exist, or update it if it does. +Note that this is a complete update, and will remove any options that were previously set. +I.e: Not specifying a schedule in the cli update command will remove the schedule from the recipe to be updated. + +**Basic example** + +To schedule a recipe called "Snowflake Integration", to run at 5am every day, London time with the recipe configured in a local `recipe.yaml` file: + +```shell +datahub ingest deploy --name "Snowflake Integration" --schedule "5 * * * *" --time-zone "Europe/London" -c recipe.yaml +``` + +By default, the ingestion recipe's identifier is generated by hashing the name. +You can override the urn generation by passing the `--urn` flag to the CLI. + +**Using `deployment` to avoid CLI args** + +As an alternative to configuring settings from the CLI, all of these settings can also be set in the `deployment` field of the recipe. + +```yml +# deployment_recipe.yml +deployment: + name: "Snowflake Integration" + schedule: "5 * * * *" + time_zone: "Europe/London" + +source: ... +``` + +```shell +datahub ingest deploy -c deployment_recipe.yml +``` + +This is particularly useful when you want all recipes to be stored in version control. + +```shell +# Deploy every yml recipe in a directory +ls recipe_directory/*.yml | xargs -n 1 -I {} datahub ingest deploy -c {} +``` + ### init The init command is used to tell `datahub` about where your DataHub instance is located. The CLI will point to localhost DataHub by default. @@ -242,8 +274,6 @@ The [metadata deletion guide](./how/delete-metadata.md) covers the various optio ### exists -**🤝 Version compatibility** : `acryl-datahub>=0.10.2.4` - The exists command can be used to check if an entity exists in DataHub. ```shell @@ -253,7 +283,6 @@ true false ``` - ### get The `get` command allows you to easily retrieve metadata from DataHub, by using the REST API. This works for both versioned aspects and timeseries aspects. For timeseries aspects, it fetches the latest value. @@ -314,6 +343,7 @@ Update succeeded with status 200 ``` #### put platform + **🤝 Version Compatibility:** `acryl-datahub>0.8.44.4` The **put platform** command instructs `datahub` to create or update metadata about a data platform. This is very useful if you are using a custom data platform, to set up its logo and display name for a native UI experience. @@ -346,6 +376,7 @@ datahub timeline --urn "urn:li:dataset:(urn:li:dataPlatform:mysql,User.UserAccou The `dataset` command allows you to interact with the dataset entity. The `get` operation can be used to read in a dataset into a yaml file. + ```shell datahub dataset get --urn "$URN" --to-file "$FILE_NAME" ``` @@ -358,7 +389,6 @@ datahub dataset upsert -f dataset.yaml An example of `dataset.yaml` would look like as in [dataset.yaml](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/cli_usage/dataset/dataset.yaml). - ### user (User Entity) The `user` command allows you to interact with the User entity. @@ -411,7 +441,6 @@ members: display_name: "Joe's Hub" ``` - ### dataproduct (Data Product Entity) **🤝 Version Compatibility:** `acryl-datahub>=0.10.2.4` @@ -566,14 +595,12 @@ Use this to delete a Data Product from DataHub. Default to `--soft` which preser # > datahub dataproduct delete --urn "urn:li:dataProduct:pet_of_the_week" --hard ``` - ## Miscellaneous Admin Commands ### lite (experimental) The lite group of commands allow you to run an embedded, lightweight DataHub instance for command line exploration of your metadata. This is intended more for developer tool oriented usage rather than as a production server instance for DataHub. See [DataHub Lite](./datahub_lite.md) for more information about how you can ingest metadata into DataHub Lite and explore your metadata easily. - ### telemetry To help us understand how people are using DataHub, we collect anonymous usage statistics on actions such as command invocations via Mixpanel. @@ -640,7 +667,6 @@ External Entities Affected: None Old Entities Migrated = {'urn:li:dataset:(urn:li:dataPlatform:hive,logging_events,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)'} ``` - ## Alternate Installation Options ### Using docker @@ -673,7 +699,7 @@ We use a plugin architecture so that you can install only the dependencies you a Please see our [Integrations page](https://datahubproject.io/integrations) if you want to filter on the features offered by each source. | Plugin Name | Install Command | Provides | -|------------------------------------------------------------------------------------------------| ---------------------------------------------------------- | --------------------------------------- | +| ---------------------------------------------------------------------------------------------- | ---------------------------------------------------------- | --------------------------------------- | | [metadata-file](./generated/ingestion/sources/metadata-file.md) | _included by default_ | File source and sink | | [athena](./generated/ingestion/sources/athena.md) | `pip install 'acryl-datahub[athena]'` | AWS Athena source | | [bigquery](./generated/ingestion/sources/bigquery.md) | `pip install 'acryl-datahub[bigquery]'` | BigQuery source | @@ -715,7 +741,7 @@ Please see our [Integrations page](https://datahubproject.io/integrations) if yo ### Sinks | Plugin Name | Install Command | Provides | -|-------------------------------------------------------------------| -------------------------------------------- | -------------------------- | +| ----------------------------------------------------------------- | -------------------------------------------- | -------------------------- | | [metadata-file](../metadata-ingestion/sink_docs/metadata-file.md) | _included by default_ | File source and sink | | [console](../metadata-ingestion/sink_docs/console.md) | _included by default_ | Console sink | | [datahub-rest](../metadata-ingestion/sink_docs/datahub.md) | `pip install 'acryl-datahub[datahub-rest]'` | DataHub sink over REST API | diff --git a/docs/deploy/aws.md b/docs/deploy/aws.md index d1003077e24861..67dd9a734e67f5 100644 --- a/docs/deploy/aws.md +++ b/docs/deploy/aws.md @@ -76,7 +76,7 @@ First, if you did not use eksctl to setup the kubernetes cluster, make sure to g Download the IAM policy document for allowing the controller to make calls to AWS APIs on your behalf. ``` -curl -o iam_policy.json https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.2.0/docs/install/iam_policy.json +curl -o iam_policy.json https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/main/docs/install/iam_policy.json ``` Create an IAM policy based on the policy document by running the following. @@ -148,13 +148,9 @@ datahub-frontend: alb.ingress.kubernetes.io/certificate-arn: <> alb.ingress.kubernetes.io/inbound-cidrs: 0.0.0.0/0 alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS":443}]' - alb.ingress.kubernetes.io/actions.ssl-redirect: '{"Type": "redirect", "RedirectConfig": { "Protocol": "HTTPS", "Port": "443", "StatusCode": "HTTP_301"}}' + alb.ingress.kubernetes.io/ssl-redirect: '443' hosts: - host: <> - redirectPaths: - - path: /* - name: ssl-redirect - port: use-annotation paths: - /* ``` diff --git a/docs/features.md b/docs/features.md index 7951eea1f909c8..bcf3ec3618f88b 100644 --- a/docs/features.md +++ b/docs/features.md @@ -28,7 +28,7 @@ To get started with DataHub, you can use a simple CLI command. Alternatively, yo - [Quickstart](quickstart.md) - [Self-hosted DataHub](deploy/kubernetes.md) -- [Managed DataHub](managed-datahub/managed-datahub-overview.md) +- [DataHub Cloud](managed-datahub/managed-datahub-overview.md) ### Ingestion diff --git a/docs/features/feature-guides/documentation-forms.md b/docs/features/feature-guides/documentation-forms.md index 8b2966810de7c0..b007892e660946 100644 --- a/docs/features/feature-guides/documentation-forms.md +++ b/docs/features/feature-guides/documentation-forms.md @@ -77,7 +77,7 @@ Provide a step-by-step guide to use feature, including relevant screenshots and/ ### Videos -**Asset Verification in Acryl Cloud** +**Asset Verification in DataHub Cloud**

diff --git a/docs/how/search.md b/docs/how/search.md index 7012f5321f2ff7..c809ab1efba12d 100644 --- a/docs/how/search.md +++ b/docs/how/search.md @@ -71,7 +71,7 @@ After creating a filter, you can choose whether results should or should not mat ### Results -Search results appear ranked by their relevance. In self-hosted DataHub ranking is based on how closely the query matched textual fields of an asset and its metadata. In Managed DataHub, ranking is based on a combination of textual relevance, usage (queries / views), and change frequency. +Search results appear ranked by their relevance. In self-hosted DataHub ranking is based on how closely the query matched textual fields of an asset and its metadata. In DataHub Cloud, ranking is based on a combination of textual relevance, usage (queries / views), and change frequency. With better metadata comes better results. Learn more about ingestion technical metadata in the [metadata ingestion](../../metadata-ingestion/README.md) guide. @@ -148,24 +148,42 @@ The same GraphQL API that powers the Search UI can be used for integrations and programmatic use-cases. ``` -# Example query -{ - searchAcrossEntities( - input: {types: [], query: "*", start: 0, count: 10, filters: [{field: "fieldTags", value: "urn:li:tag:Dimension"}]} +# Example query - search for datasets matching the example_query_text who have the Dimension tag applied to a schema field and are from the data platform looker +query searchEntities { + search( + input: { + type: DATASET, + query: "example_query_text", + orFilters: [ + { + and: [ + { + field: "fieldTags", + values: ["urn:li:tag:Dimension"] + }, + { + field: "platform", + values: ["urn:li:dataPlatform:looker"] + } + ] + } + ], + start: 0, + count: 10 + } ) { start count total searchResults { entity { + urn type ... on Dataset { - urn - type + name platform { name } - name } } } diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md index ffceb7a5d1b020..08ababcb5cfce9 100644 --- a/docs/how/updating-datahub.md +++ b/docs/how/updating-datahub.md @@ -55,6 +55,19 @@ New (optional fields `systemMetadata` and `headers`): "headers": {} } ``` +- #10858 Profiling configuration for Glue source has been updated. + +Previously, the configuration was: +```yaml +profiling: {} +``` + +Now, it needs to be: + +```yaml +profiling: + enabled: true +``` ### Potential Downtime @@ -66,7 +79,10 @@ New (optional fields `systemMetadata` and `headers`): ### Other Notable Changes - #10498 - Tableau ingestion can now be configured to ingest multiple sites at once and add the sites as containers. The feature is currently only available for Tableau Server. - +- #10466 - Extends configuration in `~/.datahubenv` to match `DatahubClientConfig` object definition. See full configuration in https://datahubproject.io/docs/python-sdk/clients/. The CLI should now respect the updated configurations specified in `~/.datahubenv` across its functions and utilities. This means that for systems where ssl certification is disabled, setting `disable_ssl_verification: true` in `~./datahubenv` will apply to all CLI calls. +- #11002 - We will not auto-generate a `~/.datahubenv` file. You must either run `datahub init` to create that file, or set environment variables so that the config is loaded. +- #11023 - Added a new parameter to datahub's `put` cli command: `--run-id`. This parameter is useful to associate a given write to an ingestion process. A use-case can be mimick transformers when a transformer for aspect being written does not exist. +- #11051 - Ingestion reports will now trim the summary text to a maximum of 800k characters to avoid generating `dataHubExecutionRequestResult` that are too large for GMS to handle. ## 0.13.3 ### Breaking Changes diff --git a/docs/incidents/incidents.md b/docs/incidents/incidents.md index 41b4df10b78281..c1412d0bebd043 100644 --- a/docs/incidents/incidents.md +++ b/docs/incidents/incidents.md @@ -417,9 +417,9 @@ Authorization: Bearer Also, remember that you can play with an interactive version of the GraphQL API at `https://your-account-id.acryl.io/api/graphiql` ::: -## Enabling Slack Notifications (Acryl Cloud Only) +## Enabling Slack Notifications (DataHub Cloud Only) -In Acryl Cloud, you can configure your to send Slack notifications to a specific channel when incidents are raised or their status is changed. +In DataHub Cloud, you can configure your to send Slack notifications to a specific channel when incidents are raised or their status is changed. These notifications are also able to tag the immediate asset's owners, along with the owners of downstream assets consuming it. diff --git a/docs/lineage/airflow.md b/docs/lineage/airflow.md index 62715ed506ffe9..2d7707637e2d1c 100644 --- a/docs/lineage/airflow.md +++ b/docs/lineage/airflow.md @@ -17,7 +17,7 @@ There's two actively supported implementations of the plugin, with different Air | Approach | Airflow Version | Notes | | --------- | --------------- | --------------------------------------------------------------------------- | -| Plugin v2 | 2.3+ | Recommended. Requires Python 3.8+ | +| Plugin v2 | 2.3.4+ | Recommended. Requires Python 3.8+ | | Plugin v1 | 2.1+ | No automatic lineage extraction; may not extract lineage if the task fails. | If you're using Airflow older than 2.1, it's possible to use the v1 plugin with older versions of `acryl-datahub-airflow-plugin`. See the [compatibility section](#compatibility) for more details. @@ -45,7 +45,7 @@ Set up a DataHub connection in Airflow, either via command line or the Airflow U airflow connections add --conn-type 'datahub-rest' 'datahub_rest_default' --conn-host 'http://datahub-gms:8080' --conn-password '' ``` -If you are using hosted Acryl Datahub then please use `https://YOUR_PREFIX.acryl.io/gms` as the `--conn-host` parameter. +If you are using DataHub Cloud then please use `https://YOUR_PREFIX.acryl.io/gms` as the `--conn-host` parameter. #### Airflow UI @@ -66,7 +66,7 @@ enabled = True # default ``` | Name | Default value | Description | -|----------------------------|----------------------|------------------------------------------------------------------------------------------| +| -------------------------- | -------------------- | ---------------------------------------------------------------------------------------- | | enabled | true | If the plugin should be enabled. | | conn_id | datahub_rest_default | The name of the datahub rest connection. | | cluster | prod | name of the airflow cluster, this is equivalent to the `env` of the instance | @@ -132,7 +132,7 @@ conn_id = datahub_rest_default # or datahub_kafka_default ``` | Name | Default value | Description | -|----------------------------|----------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| -------------------------- | -------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | enabled | true | If the plugin should be enabled. | | conn_id | datahub_rest_default | The name of the datahub connection you set in step 1. | | cluster | prod | name of the airflow cluster | @@ -240,6 +240,7 @@ See this [example PR](https://github.com/datahub-project/datahub/pull/10452) whi There might be a case where the DAGs are removed from the Airflow but the corresponding pipelines and tasks are still there in the Datahub, let's call such pipelines ans tasks, `obsolete pipelines and tasks` Following are the steps to cleanup them from the datahub: + - create a DAG named `Datahub_Cleanup`, i.e. ```python @@ -263,8 +264,31 @@ with DAG( ) ``` + - ingest this DAG, and it will remove all the obsolete pipelines and tasks from the Datahub based on the `cluster` value set in the `airflow.cfg` +## Get all dataJobs associated with a dataFlow + +If you are looking to find all tasks (aka DataJobs) that belong to a specific pipeline (aka DataFlow), you can use the following GraphQL query: + +```graphql +query { + dataFlow(urn: "urn:li:dataFlow:(airflow,db_etl,prod)") { + childJobs: relationships( + input: { types: ["IsPartOf"], direction: INCOMING, start: 0, count: 100 } + ) { + total + relationships { + entity { + ... on DataJob { + urn + } + } + } + } + } +} +``` ## Emit Lineage Directly diff --git a/docs/managed-datahub/approval-workflows.md b/docs/managed-datahub/approval-workflows.md index 75cab458d285d9..e28ba1b87d7eaa 100644 --- a/docs/managed-datahub/approval-workflows.md +++ b/docs/managed-datahub/approval-workflows.md @@ -6,7 +6,7 @@ import FeatureAvailability from '@site/src/components/FeatureAvailability'; ## Overview -Keeping all your metadata properly classified can be hard work when you only have a limited number of trusted data stewards. With Managed DataHub, you can source proposals of Tags and Glossary Terms associated to datasets or dataset columns. These proposals may come from users with limited context or programatic processes using hueristics. Then, data stewards and data owners can go through them and only approve proposals they consider correct. This reduces the burden of your stewards and owners while increasing coverage. +Keeping all your metadata properly classified can be hard work when you only have a limited number of trusted data stewards. With DataHub Cloud, you can source proposals of Tags and Glossary Terms associated to datasets or dataset columns. These proposals may come from users with limited context or programatic processes using hueristics. Then, data stewards and data owners can go through them and only approve proposals they consider correct. This reduces the burden of your stewards and owners while increasing coverage. Approval workflows also cover the Business Glossary itself. This allows you to source Glossary Terms and Glossary Term description changes from across your organization while limiting who has final control over what gets in. diff --git a/docs/managed-datahub/chrome-extension.md b/docs/managed-datahub/chrome-extension.md index a4560bc8cc09ba..a98aa6cb7e3040 100644 --- a/docs/managed-datahub/chrome-extension.md +++ b/docs/managed-datahub/chrome-extension.md @@ -1,12 +1,12 @@ --- -description: Learn how to upload and use the Acryl DataHub Chrome extension (beta) locally before it's available on the Chrome store. +description: Learn how to upload and use the DataHub Cloud Chrome extension (beta) locally before it's available on the Chrome store. --- -# Acryl DataHub Chrome Extension +# DataHub Cloud Chrome Extension ## Installing the Extension -In order to use the Acryl DataHub Chrome extension, you need to download it onto your browser from the Chrome web store [here](https://chrome.google.com/webstore/detail/datahub-chrome-extension/aoenebhmfokhglijmoacfjcnebdpchfj). +In order to use the DataHub Cloud Chrome extension, you need to download it onto your browser from the Chrome web store [here](https://chrome.google.com/webstore/detail/datahub-chrome-extension/aoenebhmfokhglijmoacfjcnebdpchfj).

@@ -18,7 +18,7 @@ Simply click "Add to Chrome" then "Add extension" on the ensuing popup. ## Configuring the Extension -Once you have your extension installed, you'll need to configure it to work with your Acryl DataHub deployment. +Once you have your extension installed, you'll need to configure it to work with your DataHub Cloud deployment. 1. Click the extension button on the right of your browser's address bar to view all of your installed extensions. Click on the newly installed DataHub extension. @@ -40,7 +40,7 @@ If your organization uses standard SaaS domains for Looker, you should be ready ### Additional Configurations -Some organizations have custom SaaS domains for Looker and some Acryl DataHub deployments utilize **Platform Instances** and set custom **Environments** when creating DataHub assets. If any of these situations applies to you, please follow the next few steps to finish configuring your extension. +Some organizations have custom SaaS domains for Looker and some DataHub Cloud deployments utilize **Platform Instances** and set custom **Environments** when creating DataHub assets. If any of these situations applies to you, please follow the next few steps to finish configuring your extension. 1. Click on the extension button and select your DataHub extension to open the popup again. Now click the settings icon in order to open the configurations page. @@ -62,13 +62,13 @@ Some organizations have custom SaaS domains for Looker and some Acryl DataHub de Once you have everything configured on your extension, it's time to use it! -1. First ensure that you are logged in to your Acryl DataHub instance. +1. First ensure that you are logged in to your DataHub Cloud instance. 2. Navigate to Looker or Tableau and log in to view your data assets. 3. Navigate to a page where DataHub can provide insights on your data assets (Dashboards and Explores). -4. Click the Acryl DataHub extension button on the bottom right of your page to open a drawer where you can now see additional information about this asset right from your DataHub instance. +4. Click the DataHub Cloud extension button on the bottom right of your page to open a drawer where you can now see additional information about this asset right from your DataHub instance.

@@ -78,7 +78,7 @@ Once you have everything configured on your extension, it's time to use it! ## Advanced: Self-Hosted DataHub -If you are using the Acryl DataHub Chrome extension for your self-hosted DataHub instance, everything above is applicable. However, there is one additional step you must take in order to set up your instance to be compatible with the extension. +If you are using the DataHub Cloud Chrome extension for your self-hosted DataHub instance, everything above is applicable. However, there is one additional step you must take in order to set up your instance to be compatible with the extension. ### Configure Auth Cookies diff --git a/docs/managed-datahub/datahub-api/graphql-api/getting-started.md b/docs/managed-datahub/datahub-api/graphql-api/getting-started.md index 736bf6fea6811d..5993e2dfd773dd 100644 --- a/docs/managed-datahub/datahub-api/graphql-api/getting-started.md +++ b/docs/managed-datahub/datahub-api/graphql-api/getting-started.md @@ -4,7 +4,7 @@ description: Getting started with the DataHub GraphQL API. # Getting Started -The Acryl DataHub GraphQL API is an extension of the open source [DataHub GraphQL API.](docs/api/graphql/overview.md) +The DataHub Cloud GraphQL API is an extension of the open source [DataHub GraphQL API.](docs/api/graphql/overview.md) For a full reference to the Queries & Mutations available for consumption, check out [Queries](graphql/queries.md) & [Mutations](graphql/mutations.md). @@ -39,7 +39,7 @@ e.g. to connect to ingestion endpoint for doing ingestion programmatically you c The entire GraphQL API can be explored & [introspected](https://graphql.org/learn/introspection/) using GraphiQL, an interactive query tool which allows you to navigate the entire Acryl GraphQL schema as well as craft & issue using an intuitive UI. -[GraphiQL](https://www.gatsbyjs.com/docs/how-to/querying-data/running-queries-with-graphiql/) is available for each Acryl DataHub deployment, locating at `https://your-account.acryl.io/api/graphiql`. +[GraphiQL](https://www.gatsbyjs.com/docs/how-to/querying-data/running-queries-with-graphiql/) is available for each DataHub Cloud deployment, locating at `https://your-account.acryl.io/api/graphiql`. ### Querying the API diff --git a/docs/managed-datahub/managed-datahub-overview.md b/docs/managed-datahub/managed-datahub-overview.md index 4efc96eaf17a7c..f4d155ff0dbe10 100644 --- a/docs/managed-datahub/managed-datahub-overview.md +++ b/docs/managed-datahub/managed-datahub-overview.md @@ -1,8 +1,8 @@ -# How Acryl DataHub compares to DataHub +# How DataHub Cloud compares to DataHub DataHub is the #1 open source data catalog for developers. -Acryl DataHub takes DataHub to the next level by offering features that allow +DataHub Cloud takes DataHub to the next level by offering features that allow you to roll out the product to the entire organization beyond your central data platform team. @@ -12,7 +12,7 @@ checklists and we hope you love them too! ## Search and Discovery Features aimed at making it easy to discover data assets at your organization and understand relationships between them. -| Feature | DataHub | Acryl DataHub | +| Feature | DataHub | DataHub Cloud | | ---------------------------------------------- | ------- | ------------- | | Integrations for 50+ data sources | ✅ | ✅ | | Table level, Column-level, Job-level lineage | ✅ | ✅ | @@ -32,7 +32,7 @@ Features aimed at making it easy to discover data assets at your organization an Features that help you govern the crown jewels of your organization, and trim out the datasets that seem to grow like weeds when no one’s looking. -| Feature | DataHub | Acryl DataHub | +| Feature | DataHub | DataHub Cloud | | ---------------------------------------------- | ------- | ------------- | | Shift-Left governance | ✅ | ✅ | | Dataset ownership management | ✅ | ✅ | @@ -48,7 +48,7 @@ Features that help you ensure your data pipelines are producing high quality assets, and if they’re not, making sure you and impacted users are the first to know. -| Feature | DataHub | Acryl DataHub | +| Feature | DataHub | DataHub Cloud | | ---------------------------------------------- | ------- | ------------- | | Surface data quality results | ✅ | ✅ | | Create data contracts | ✅ | ✅ | @@ -68,7 +68,7 @@ know. ## Enterprise Grade Features needed to roll out at scale to large enterprises. -| Feature | DataHub | Acryl DataHub | +| Feature | DataHub | DataHub Cloud | | ---------------------------------------------- | ------- | ------------- | | Battle-tested open source metadata platform | ✅ | ✅ | | Metadata change events as a real-time stream | ✅ | ✅ | @@ -82,7 +82,7 @@ Features needed to roll out at scale to large enterprises. ## Implementation and Support Features related to ease of deployment and maintenance. -| Feature | DataHub | Acryl DataHub | +| Feature | DataHub | DataHub Cloud | | ---------------------------------------------- | ------- | ------------- | | Community support | ✅ | ✅ | | Your own engineering team | ✅ | ❌ (They can instead focus on high-value work like contributing features to the open source product, or build amazing data applications with the APIs!)| @@ -102,7 +102,7 @@ the form using the link below, and someone from the Acryl team will reach out to set up a chat. - Learn about Acryl DataHub + Learn about DataHub Cloud