Skip to content

Commit

Permalink
Merge branch 'master' into feat/dremio-connector-source
Browse files Browse the repository at this point in the history
  • Loading branch information
sagar-salvi-apptware authored Oct 28, 2024
2 parents 9de8be0 + a6e1624 commit 102b55f
Show file tree
Hide file tree
Showing 152 changed files with 50,689 additions and 971 deletions.
6 changes: 5 additions & 1 deletion .github/workflows/build-and-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,11 @@ jobs:
- uses: actions/setup-python@v5
with:
python-version: "3.10"
cache: pip
- uses: actions/cache@v4
with:
path: |
~/.cache/uv
key: ${{ runner.os }}-uv-${{ hashFiles('**/requirements.txt') }}
- name: Set up JDK 17
uses: actions/setup-java@v4
with:
Expand Down
6 changes: 5 additions & 1 deletion .github/workflows/documentation.yml
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,11 @@ jobs:
- uses: actions/setup-python@v5
with:
python-version: "3.10"
cache: pip
- uses: actions/cache@v4
with:
path: |
~/.cache/uv
key: ${{ runner.os }}-uv-${{ hashFiles('**/requirements.txt') }}
- name: Install Python dependencies
run: ./metadata-ingestion/scripts/install_deps.sh
- name: Build Docs
Expand Down
5 changes: 0 additions & 5 deletions .github/workflows/metadata-ingestion.yml
Original file line number Diff line number Diff line change
Expand Up @@ -62,16 +62,11 @@ jobs:
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
cache: "pip"
- uses: actions/cache@v4
with:
path: |
~/.cache/uv
key: ${{ runner.os }}-uv-${{ hashFiles('**/requirements.txt') }}
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
cache: "pip"
- name: Install dependencies
run: ./metadata-ingestion/scripts/install_deps.sh
- name: Install package
Expand Down
2 changes: 2 additions & 0 deletions datahub-frontend/app/auth/AuthModule.java
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import com.linkedin.util.Configuration;
import config.ConfigurationProvider;
import controllers.SsoCallbackController;
import io.datahubproject.metadata.context.ValidationContext;
import java.nio.charset.StandardCharsets;
import java.util.Collections;

Expand Down Expand Up @@ -187,6 +188,7 @@ protected OperationContext provideOperationContext(
.authorizationContext(AuthorizationContext.builder().authorizer(Authorizer.EMPTY).build())
.searchContext(SearchContext.EMPTY)
.entityRegistryContext(EntityRegistryContext.builder().build(EmptyEntityRegistry.EMPTY))
.validationContext(ValidationContext.builder().alternateValidation(false).build())
.build(systemAuthentication);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,8 @@ public class ContainerType
Constants.DEPRECATION_ASPECT_NAME,
Constants.DATA_PRODUCTS_ASPECT_NAME,
Constants.STRUCTURED_PROPERTIES_ASPECT_NAME,
Constants.FORMS_ASPECT_NAME);
Constants.FORMS_ASPECT_NAME,
Constants.ACCESS_ASPECT_NAME);

private static final Set<String> FACET_FIELDS = ImmutableSet.of("origin", "platform");
private static final String ENTITY_NAME = "container";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import static com.linkedin.metadata.Constants.*;

import com.linkedin.common.Access;
import com.linkedin.common.DataPlatformInstance;
import com.linkedin.common.Deprecation;
import com.linkedin.common.Forms;
Expand Down Expand Up @@ -30,6 +31,7 @@
import com.linkedin.datahub.graphql.types.domain.DomainAssociationMapper;
import com.linkedin.datahub.graphql.types.form.FormsMapper;
import com.linkedin.datahub.graphql.types.glossary.mappers.GlossaryTermsMapper;
import com.linkedin.datahub.graphql.types.rolemetadata.mappers.AccessMapper;
import com.linkedin.datahub.graphql.types.structuredproperty.StructuredPropertiesMapper;
import com.linkedin.datahub.graphql.types.tag.mappers.GlobalTagsMapper;
import com.linkedin.domain.Domains;
Expand Down Expand Up @@ -105,6 +107,11 @@ public static Container map(
context, new GlossaryTerms(envelopedTerms.getValue().data()), entityUrn));
}

final EnvelopedAspect accessAspect = aspects.get(ACCESS_ASPECT_NAME);
if (accessAspect != null) {
result.setAccess(AccessMapper.map(new Access(accessAspect.getValue().data()), entityUrn));
}

final EnvelopedAspect envelopedInstitutionalMemory =
aspects.get(Constants.INSTITUTIONAL_MEMORY_ASPECT_NAME);
if (envelopedInstitutionalMemory != null) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ public class DatasetType
EMBED_ASPECT_NAME,
DATA_PRODUCTS_ASPECT_NAME,
BROWSE_PATHS_V2_ASPECT_NAME,
ACCESS_DATASET_ASPECT_NAME,
ACCESS_ASPECT_NAME,
STRUCTURED_PROPERTIES_ASPECT_NAME,
FORMS_ASPECT_NAME,
SUB_TYPES_ASPECT_NAME);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ public Dataset apply(
(dataset, dataMap) ->
dataset.setBrowsePathV2(BrowsePathsV2Mapper.map(context, new BrowsePathsV2(dataMap))));
mappingHelper.mapToResult(
ACCESS_DATASET_ASPECT_NAME,
ACCESS_ASPECT_NAME,
((dataset, dataMap) ->
dataset.setAccess(AccessMapper.map(new Access(dataMap), entityUrn))));
mappingHelper.mapToResult(
Expand Down
5 changes: 5 additions & 0 deletions datahub-graphql-core/src/main/resources/entity.graphql
Original file line number Diff line number Diff line change
Expand Up @@ -2837,6 +2837,11 @@ type Container implements Entity {
"""
exists: Boolean

"""
The Roles and the properties to access the container
"""
access: Access

"""
Experimental API.
For fetching extra entities that do not have custom UI code yet
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import io.datahubproject.metadata.context.OperationContextConfig;
import io.datahubproject.metadata.context.RetrieverContext;
import io.datahubproject.metadata.context.ServicesRegistryContext;
import io.datahubproject.metadata.context.ValidationContext;
import io.datahubproject.metadata.services.RestrictedService;
import java.util.List;
import javax.annotation.Nonnull;
Expand Down Expand Up @@ -161,7 +162,8 @@ protected OperationContext javaSystemOperationContext(
@Nonnull final GraphService graphService,
@Nonnull final SearchService searchService,
@Qualifier("baseElasticSearchComponents")
BaseElasticSearchComponentsFactory.BaseElasticSearchComponents components) {
BaseElasticSearchComponentsFactory.BaseElasticSearchComponents components,
@Nonnull final ConfigurationProvider configurationProvider) {

EntityServiceAspectRetriever entityServiceAspectRetriever =
EntityServiceAspectRetriever.builder()
Expand All @@ -186,6 +188,10 @@ protected OperationContext javaSystemOperationContext(
.aspectRetriever(entityServiceAspectRetriever)
.graphRetriever(systemGraphRetriever)
.searchRetriever(searchServiceSearchRetriever)
.build(),
ValidationContext.builder()
.alternateValidation(
configurationProvider.getFeatureFlags().isAlternateMCPValidation())
.build());

entityServiceAspectRetriever.setSystemOperationContext(systemOperationContext);
Expand Down
23 changes: 22 additions & 1 deletion datahub-web-react/src/app/entity/container/ContainerEntity.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import { DocumentationTab } from '../shared/tabs/Documentation/DocumentationTab'
import { SidebarAboutSection } from '../shared/containers/profile/sidebar/AboutSection/SidebarAboutSection';
import { SidebarOwnerSection } from '../shared/containers/profile/sidebar/Ownership/sidebar/SidebarOwnerSection';
import { getDataForEntityType } from '../shared/containers/profile/utils';
import { useGetContainerQuery } from '../../../graphql/container.generated';
import { useGetContainerQuery, GetContainerQuery } from '../../../graphql/container.generated';
import { ContainerEntitiesTab } from './ContainerEntitiesTab';
import { SidebarTagsSection } from '../shared/containers/profile/sidebar/SidebarTagsSection';
import { PropertiesTab } from '../shared/tabs/Properties/PropertiesTab';
Expand All @@ -17,6 +17,8 @@ import { capitalizeFirstLetterOnly } from '../../shared/textUtil';
import DataProductSection from '../shared/containers/profile/sidebar/DataProduct/DataProductSection';
import { getDataProduct } from '../shared/utils';
import EmbeddedProfile from '../shared/embed/EmbeddedProfile';
import AccessManagement from '../shared/tabs/Dataset/AccessManagement/AccessManagement';
import { useAppConfig } from '../../useAppConfig';

/**
* Definition of the DataHub Container entity.
Expand Down Expand Up @@ -65,6 +67,8 @@ export class ContainerEntity implements Entity<Container> {

useEntityQuery = useGetContainerQuery;

appconfig = useAppConfig;

renderProfile = (urn: string) => (
<EntityProfile
urn={urn}
Expand All @@ -85,6 +89,23 @@ export class ContainerEntity implements Entity<Container> {
name: 'Properties',
component: PropertiesTab,
},
{
name: 'Access Management',
component: AccessManagement,
display: {
visible: (_, container: GetContainerQuery) => {
return (
this.appconfig().config.featureFlags.showAccessManagement &&
!!container?.container?.access
);
},
enabled: (_, container: GetContainerQuery) => {
const accessAspect = container?.container?.access;
const rolesList = accessAspect?.roles;
return !!accessAspect && !!rolesList && rolesList.length > 0;
},
},
},
]}
sidebarSections={this.getSidebarSections()}
/>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@ import styled from 'styled-components';
import { Button, Table } from 'antd';
import { SpinProps } from 'antd/es/spin';
import { LoadingOutlined } from '@ant-design/icons';
import { useBaseEntity } from '../../../EntityContext';
import { GetDatasetQuery, useGetExternalRolesQuery } from '../../../../../../graphql/dataset.generated';
import { useEntityData } from '../../../EntityContext';
import { useGetExternalRolesQuery } from '../../../../../../graphql/dataset.generated';
import { handleAccessRoles } from './utils';
import AccessManagerDescription from './AccessManagerDescription';

Expand Down Expand Up @@ -60,11 +60,12 @@ const AccessButton = styled(Button)`
`;

export default function AccessManagement() {
const baseEntity = useBaseEntity<GetDatasetQuery>();
const { entityData } = useEntityData();
const entityUrn = (entityData as any)?.urn;

const { data: externalRoles, loading: isLoading } = useGetExternalRolesQuery({
variables: { urn: baseEntity?.dataset?.urn as string },
skip: !baseEntity?.dataset?.urn,
variables: { urn: entityUrn as string },
skip: !entityUrn,
});

const columns = [
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ const Title = styled.div`
font-weight: bold;
`;

const Item = styled.div`
const Item = styled.pre`
padding: 6px;
font-size: 12px;
border-radius: 2px;
Expand Down
7 changes: 7 additions & 0 deletions datahub-web-react/src/graphql/container.graphql
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,13 @@ query getContainer($urn: String!) {
status {
removed
}
access {
roles {
role {
urn
}
}
}
autoRenderAspects: aspects(input: { autoRenderOnly: true }) {
...autoRenderAspectFields
}
Expand Down
14 changes: 8 additions & 6 deletions docker/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ import com.avast.gradle.dockercompose.tasks.ComposeDownForced
apply from: "../gradle/versioning/versioning.gradle"

ext {
compose_base = "profiles/docker-compose.yml"

backend_profile_modules = [
':docker:elasticsearch-setup',
':docker:mysql-setup',
Expand All @@ -23,7 +25,7 @@ ext {

debug_modules = quickstart_modules - [':metadata-jobs:mce-consumer-job',
':metadata-jobs:mae-consumer-job']
compose_args = ['-f', 'profiles/docker-compose.yml']
compose_args = ['-f', compose_base]
debug_reloadable = [
'datahub-gms-debug',
'system-update-debug',
Expand Down Expand Up @@ -68,7 +70,7 @@ dockerCompose {
environment.put 'DATAHUB_VERSION', "v${version}"
environment.put 'DATAHUB_TELEMETRY_ENABLED', 'false' // disabled when built locally

useComposeFiles = ['profiles/docker-compose.yml']
useComposeFiles = [compose_base]
projectName = 'datahub'
projectNamePrefix = ''
buildBeforeUp = false
Expand All @@ -86,7 +88,7 @@ dockerCompose {
environment.put 'DATAHUB_VERSION', "v${version}"
environment.put 'DATAHUB_TELEMETRY_ENABLED', 'false' // disabled when built locally

useComposeFiles = ['profiles/docker-compose.yml']
useComposeFiles = [compose_base]
projectName = 'datahub'
projectNamePrefix = ''
buildBeforeUp = false
Expand All @@ -113,7 +115,7 @@ dockerCompose {
// disabled for spark-lineage smoke-test
environment.put 'DATAHUB_LOCAL_COMMON_ENV', "${rootProject.project(':metadata-integration:java:spark-lineage-legacy').projectDir}/spark-smoke-test/smoke-gms.env"

useComposeFiles = ['profiles/docker-compose.yml']
useComposeFiles = [compose_base]
projectName = 'datahub'
projectNamePrefix = ''
buildBeforeUp = false
Expand All @@ -133,7 +135,7 @@ dockerCompose {
}
environment.put 'DATAHUB_TELEMETRY_ENABLED', 'false' // disabled when built locally

useComposeFiles = ['profiles/docker-compose.yml']
useComposeFiles = [compose_base]
projectName = 'datahub'
projectNamePrefix = ''
buildBeforeUp = false
Expand All @@ -146,7 +148,7 @@ dockerCompose {
isRequiredBy(tasks.named('quickstartStorage'))
composeAdditionalArgs = ['--profile', 'quickstart-storage']

useComposeFiles = ['profiles/docker-compose.yml']
useComposeFiles = [compose_base]
projectName = 'datahub'
projectNamePrefix = ''
buildBeforeUp = false
Expand Down
2 changes: 1 addition & 1 deletion docker/kafka-setup/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
ARG KAFKA_DOCKER_VERSION=7.4.6
ARG KAFKA_DOCKER_VERSION=7.7.1

# Defining custom repo urls for use in enterprise environments. Re-used between stages below.
ARG ALPINE_REPO_URL=http://dl-cdn.alpinelinux.org/alpine
Expand Down
2 changes: 2 additions & 0 deletions docker/profiles/docker-compose.gms.yml
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@ x-datahub-gms-service: &datahub-gms-service
environment: &datahub-gms-env
<<: [*primary-datastore-mysql-env, *graph-datastore-search-env, *search-datastore-env, *datahub-quickstart-telemetry-env, *kafka-env]
ELASTICSEARCH_QUERY_CUSTOM_CONFIG_FILE: ${ELASTICSEARCH_QUERY_CUSTOM_CONFIG_FILE:-search_config.yaml}
ALTERNATE_MCP_VALIDATION: ${ALTERNATE_MCP_VALIDATION:-true}
healthcheck:
test: curl -sS --fail http://datahub-gms:${DATAHUB_GMS_PORT:-8080}/health
start_period: 90s
Expand Down Expand Up @@ -182,6 +183,7 @@ x-datahub-mce-consumer-service: &datahub-mce-consumer-service
- ${DATAHUB_LOCAL_MCE_ENV:-empty2.env}
environment: &datahub-mce-consumer-env
<<: [*primary-datastore-mysql-env, *graph-datastore-search-env, *search-datastore-env, *datahub-quickstart-telemetry-env, *kafka-env]
ALTERNATE_MCP_VALIDATION: ${ALTERNATE_MCP_VALIDATION:-true}

x-datahub-mce-consumer-service-dev: &datahub-mce-consumer-service-dev
<<: *datahub-mce-consumer-service
Expand Down
2 changes: 1 addition & 1 deletion docs-website/docusaurus.config.js
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ module.exports = {
announcementBar: {
id: "announcement-2",
content:
'<div style="display: flex; justify-content: center; align-items: center;width: 100%;"><!--img src="/img/acryl-logo-white-mark.svg" / --><div style="font-size: .8rem; font-weight: 600; background-color: white; color: #111; padding: 0px 8px; border-radius: 4px; margin-right:12px;">NEW</div><p><span>Join us at Metadata & AI Summit, Oct. 29 & 30!</span></p><a href="http://www.acryldata.io/conference?utm_source=datahub_web&utm_medium=metadata_ai_2024&utm_campaign=home_banner" target="_blank" class="button">Register</a></div>',
'<div style="display: flex; justify-content: center; align-items: center;width: 100%;"><!--img src="/img/acryl-logo-white-mark.svg" / --><div style="font-size: .8rem; font-weight: 600; background-color: white; color: #111; padding: 0px 8px; border-radius: 4px; margin-right:12px;">NEW</div><p>Join us at Metadata & AI Summit, Oct. 29 & 30!</p><a href="http://www.acryldata.io/conference?utm_source=datahub_web&utm_medium=metadata_ai_2024&utm_campaign=home_banner" target="_blank" class="button">Register<span> →</span></a></div>',
backgroundColor: "#111",
textColor: "#ffffff",
isCloseable: false,
Expand Down
7 changes: 7 additions & 0 deletions docs-website/sidebars.js
Original file line number Diff line number Diff line change
Expand Up @@ -101,12 +101,19 @@ module.exports = {
{
label: "Automations",
type: "category",
collapsed: false,
items: [
{
label: "Documentation Propagation",
type: "doc",
id: "docs/automations/docs-propagation",
},
{
label: "BigQuery Metadata Sync",
type: "doc",
id: "docs/automations/bigquery-metadata-sync",
className: "saasOnly",
},
{
label: "Snowflake Tag Sync",
type: "doc",
Expand Down
Loading

0 comments on commit 102b55f

Please sign in to comment.