diff --git a/.github/actions/ci-optimization/action.yml b/.github/actions/ci-optimization/action.yml index ff901b5de04b65..0d435963382675 100644 --- a/.github/actions/ci-optimization/action.yml +++ b/.github/actions/ci-optimization/action.yml @@ -57,7 +57,8 @@ runs: - "metadata-ingestion-modules/**" - "metadata-ingestion/**" - "metadata-models/**" - - "docker/datahub-ingestion**" + - "docker/datahub-ingestion-base/**" + - "docker/datahub-ingestion/**" ingestion-base: - "docker/datahub-ingestion-base/**" docker: diff --git a/.github/actions/docker-custom-build-and-push/action.yml b/.github/actions/docker-custom-build-and-push/action.yml index 3805b3501ccec0..ccaff510c120aa 100644 --- a/.github/actions/docker-custom-build-and-push/action.yml +++ b/.github/actions/docker-custom-build-and-push/action.yml @@ -31,16 +31,21 @@ inputs: description: "Main tag to use for the Docker image" required: true flavor: - description: 'Image flavor (e.g., slim, full)' + description: "Image flavor (e.g., slim, full)" required: false target: description: "Sets the target stage to build" required: false + depot-project: + # Setting this will use native arm64 docker builds instead of QEMU emulation. + # This speeds up builds by 2-3x. + description: "Depot project id" + required: false + outputs: image_tag: description: "Docker image tags" value: ${{ steps.docker_meta.outputs.tags }} - # image_name: ${{ env.DATAHUB_GMS_IMAGE }} runs: using: "composite" @@ -53,15 +58,27 @@ runs: images: ${{ inputs.images }} flavor: | latest=false - suffix=${{ inputs.flavor && format('-{0}', inputs.flavor) || '' }} tags: | type=raw,value=${{ inputs.image_tag }} - type=raw,value=head,enable={{is_default_branch}} - type=sha,prefix=,format=short + type=raw,value=head,suffix=${{ inputs.flavor && format('-{0}', inputs.flavor) || '' }},enable={{is_default_branch}} + type=sha,prefix=,format=short,suffix=${{ inputs.flavor && format('-{0}', inputs.flavor) || '' }} + + - name: Single Tag + id: single_tag + shell: bash + run: | + IMAGES=""" + ${{ inputs.images }} + """ + TAGS=""" + ${{ inputs.image_tag }} + """ + echo "SINGLE_IMAGE=$(echo $IMAGES | tr '\n' ' ' | awk -F' |,' '{ print $1 }')" >> "$GITHUB_OUTPUT" + echo "SINGLE_TAG=$(echo $IMAGES | tr '\n' ' ' | awk -F' |,' '{ print $1 }'):$(echo $TAGS | tr '\n' ' ' | awk -F' |,' '{ print $1 }')" >> "$GITHUB_OUTPUT" # Code for testing the build when not pushing to Docker Hub. - name: Build and Load image for testing (if not publishing) - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 if: ${{ inputs.publish != 'true' }} with: context: ${{ inputs.context }} @@ -74,20 +91,11 @@ runs: target: ${{ inputs.target }} load: true push: false - cache-from: type=registry,ref=${{ steps.docker_meta.outputs.tags }} - cache-to: type=inline - - name: Single Tag - if: ${{ inputs.publish != 'true' }} - shell: bash - run: | - IMAGES=""" - ${{ inputs.images }} - """ - TAGS=""" - ${{ inputs.image_tag }} - """ - echo "SINGLE_TAG=$(echo $IMAGES | tr '\n' ' ' | awk -F' |,' '{ print $1 }'):$(echo $TAGS | tr '\n' ' ' | awk -F' |,' '{ print $1 }')" >> $GITHUB_OUTPUT - id: single_tag + cache-from: | + type=registry,ref=${{ steps.single_tag.outputs.SINGLE_IMAGE }}:head${{ inputs.flavor && format('-{0}', inputs.flavor) || '' }} + type=registry,ref=${{ steps.docker_meta.outputs.tags }} + cache-to: | + type=inline - name: Upload image locally for testing (if not publishing) uses: ishworkh/docker-image-artifact-upload@v1 if: ${{ inputs.publish != 'true' }} @@ -97,19 +105,42 @@ runs: # Code for building multi-platform images and pushing to Docker Hub. - name: Set up QEMU uses: docker/setup-qemu-action@v3 - if: ${{ inputs.publish == 'true' }} + if: ${{ inputs.publish == 'true' && inputs.depot-project == '' }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - if: ${{ inputs.publish == 'true' }} + if: ${{ inputs.publish == 'true' && inputs.depot-project == '' }} + - name: Setup Depot CLI + uses: depot/setup-action@v1 + if: ${{ inputs.publish == 'true' && inputs.depot-project != '' }} - name: Login to DockerHub uses: docker/login-action@v3 if: ${{ inputs.publish == 'true' }} with: username: ${{ inputs.username }} password: ${{ inputs.password }} + + # Depot variant. - name: Build and Push Multi-Platform image - uses: docker/build-push-action@v5 - if: ${{ inputs.publish == 'true' }} + uses: depot/build-push-action@v1 + if: ${{ inputs.publish == 'true' && inputs.depot-project != '' }} + with: + project: ${{ inputs.depot-project }} + context: ${{ inputs.context }} + file: ${{ inputs.file }} + platforms: ${{ inputs.platforms }} + build-args: ${{ inputs.build-args }} + tags: ${{ steps.docker_meta.outputs.tags }} + target: ${{ inputs.target }} + push: true + cache-from: | + type=registry,ref=${{ steps.single_tag.outputs.SINGLE_IMAGE }}:head${{ inputs.flavor && format('-{0}', inputs.flavor) || '' }} + type=registry,ref=${{ steps.docker_meta.outputs.tags }} + cache-to: | + type=inline + + - name: Build and Push Multi-Platform image + uses: docker/build-push-action@v6 + if: ${{ inputs.publish == 'true' && inputs.depot-project == '' }} with: context: ${{ inputs.context }} file: ${{ inputs.file }} @@ -118,7 +149,10 @@ runs: tags: ${{ steps.docker_meta.outputs.tags }} target: ${{ inputs.target }} push: true - cache-from: type=registry,ref=${{ steps.docker_meta.outputs.tags }} - cache-to: type=inline + cache-from: | + type=registry,ref=${{ steps.single_tag.outputs.SINGLE_IMAGE }}:head${{ inputs.flavor && format('-{0}', inputs.flavor) || '' }} + type=registry,ref=${{ steps.docker_meta.outputs.tags }} + cache-to: | + type=inline # TODO add code for vuln scanning? diff --git a/.github/workflows/airflow-plugin.yml b/.github/workflows/airflow-plugin.yml index 912dd985f66c06..e75bafdac96284 100644 --- a/.github/workflows/airflow-plugin.yml +++ b/.github/workflows/airflow-plugin.yml @@ -43,16 +43,16 @@ jobs: extra_pip_requirements: "apache-airflow~=2.4.3" extra_pip_extras: plugin-v2,test-airflow24 - python-version: "3.10" - extra_pip_requirements: 'apache-airflow~=2.6.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.3/constraints-3.10.txt' + extra_pip_requirements: "apache-airflow~=2.6.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.3/constraints-3.10.txt" extra_pip_extras: plugin-v2 - python-version: "3.10" - extra_pip_requirements: 'apache-airflow~=2.7.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-3.10.txt' + extra_pip_requirements: "apache-airflow~=2.7.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-3.10.txt" extra_pip_extras: plugin-v2 - python-version: "3.10" - extra_pip_requirements: 'apache-airflow~=2.8.1 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.1/constraints-3.10.txt' + extra_pip_requirements: "apache-airflow~=2.8.1 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.1/constraints-3.10.txt" extra_pip_extras: plugin-v2 - python-version: "3.11" - extra_pip_requirements: 'apache-airflow~=2.9.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.11.txt' + extra_pip_requirements: "apache-airflow~=2.9.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.11.txt" extra_pip_extras: plugin-v2 fail-fast: false steps: @@ -73,7 +73,7 @@ jobs: run: ./gradlew -Pextra_pip_requirements='${{ matrix.extra_pip_requirements }}' -Pextra_pip_extras='${{ matrix.extra_pip_extras }}' :metadata-ingestion-modules:airflow-plugin:build - name: pip freeze show list installed if: always() - run: source metadata-ingestion-modules/airflow-plugin/venv/bin/activate && pip freeze + run: source metadata-ingestion-modules/airflow-plugin/venv/bin/activate && uv pip freeze - uses: actions/upload-artifact@v3 if: ${{ always() && matrix.python-version == '3.10' && matrix.extra_pip_requirements == 'apache-airflow>=2.7.0' }} with: diff --git a/.github/workflows/dagster-plugin.yml b/.github/workflows/dagster-plugin.yml index f0b9038b610d22..37b6c93ec841ab 100644 --- a/.github/workflows/dagster-plugin.yml +++ b/.github/workflows/dagster-plugin.yml @@ -55,7 +55,7 @@ jobs: run: ./gradlew -Pextra_pip_requirements='${{ matrix.extraPythonRequirement }}' :metadata-ingestion-modules:dagster-plugin:lint :metadata-ingestion-modules:dagster-plugin:testQuick - name: pip freeze show list installed if: always() - run: source metadata-ingestion-modules/dagster-plugin/venv/bin/activate && pip freeze + run: source metadata-ingestion-modules/dagster-plugin/venv/bin/activate && uv pip freeze - uses: actions/upload-artifact@v3 if: ${{ always() && matrix.python-version == '3.10' && matrix.extraPythonRequirement == 'dagster>=1.3.3' }} with: diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml index 6c97cd4e263307..9e9a0ed9884b46 100644 --- a/.github/workflows/docker-unified.yml +++ b/.github/workflows/docker-unified.yml @@ -33,6 +33,10 @@ env: DATAHUB_INGESTION_BASE_IMAGE: "acryldata/datahub-ingestion-base" DATAHUB_INGESTION_IMAGE: "acryldata/datahub-ingestion" +permissions: + contents: read + id-token: write + jobs: setup: runs-on: ubuntu-latest @@ -68,23 +72,23 @@ jobs: id: tag run: | source .github/scripts/docker_helpers.sh - echo "short_sha=${SHORT_SHA}" >> $GITHUB_OUTPUT - echo "tag=$(get_tag)" >> $GITHUB_OUTPUT - echo "slim_tag=$(get_tag_slim)" >> $GITHUB_OUTPUT - echo "full_tag=$(get_tag_full)" >> $GITHUB_OUTPUT - echo "unique_tag=$(get_unique_tag)" >> $GITHUB_OUTPUT - echo "unique_slim_tag=$(get_unique_tag_slim)" >> $GITHUB_OUTPUT - echo "unique_full_tag=$(get_unique_tag_full)" >> $GITHUB_OUTPUT - echo "python_release_version=$(get_python_docker_release_v)" >> $GITHUB_OUTPUT - echo "branch_name=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT - echo "repository_name=${GITHUB_REPOSITORY#*/}" >> $GITHUB_OUTPUT + echo "short_sha=${SHORT_SHA}" >> "$GITHUB_OUTPUT" + echo "tag=$(get_tag)" >> "$GITHUB_OUTPUT" + echo "slim_tag=$(get_tag_slim)" >> "$GITHUB_OUTPUT" + echo "full_tag=$(get_tag_full)" >> "$GITHUB_OUTPUT" + echo "unique_tag=$(get_unique_tag)" >> "$GITHUB_OUTPUT" + echo "unique_slim_tag=$(get_unique_tag_slim)" >> "$GITHUB_OUTPUT" + echo "unique_full_tag=$(get_unique_tag_full)" >> "$GITHUB_OUTPUT" + echo "python_release_version=$(get_python_docker_release_v)" >> "$GITHUB_OUTPUT" + echo "branch_name=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> "$GITHUB_OUTPUT" + echo "repository_name=${GITHUB_REPOSITORY#*/}" >> "$GITHUB_OUTPUT" - name: Check whether docker login is possible id: docker-login env: ENABLE_DOCKER_LOGIN: ${{ secrets.ACRYL_DOCKER_PASSWORD != '' }} run: | echo "Enable Docker Login: ${{ env.ENABLE_DOCKER_LOGIN }}" - echo "docker-login=${{ env.ENABLE_DOCKER_LOGIN }}" >> $GITHUB_OUTPUT + echo "docker-login=${{ env.ENABLE_DOCKER_LOGIN }}" >> "$GITHUB_OUTPUT" - name: Check whether publishing enabled id: publish env: @@ -95,7 +99,7 @@ jobs: }} run: | echo "Enable publish: ${{ env.ENABLE_PUBLISH }}" - echo "publish=${{ env.ENABLE_PUBLISH }}" >> $GITHUB_OUTPUT + echo "publish=${{ env.ENABLE_PUBLISH }}" >> "$GITHUB_OUTPUT" - name: Check whether PR publishing enabled id: pr-publish env: @@ -106,7 +110,7 @@ jobs: }} run: | echo "Enable PR publish: ${{ env.ENABLE_PUBLISH }}" - echo "publish=${{ env.ENABLE_PUBLISH }}" >> $GITHUB_OUTPUT + echo "publish=${{ env.ENABLE_PUBLISH }}" >> "$GITHUB_OUTPUT" - uses: ./.github/actions/ci-optimization id: ci-optimize - uses: actions/setup-python@v5 @@ -543,9 +547,10 @@ jobs: context: . file: ./docker/datahub-ingestion-base/Dockerfile platforms: linux/amd64,linux/arm64/v8 + depot-project: ${{ vars.DEPOT_PROJECT_ID }} - name: Compute DataHub Ingestion (Base) Tag id: tag - run: echo "tag=${{ needs.setup.outputs.ingestion_base_change == 'true' && needs.setup.outputs.unique_tag || 'head' }}" >> $GITHUB_OUTPUT + run: echo "tag=${{ needs.setup.outputs.ingestion_base_change == 'true' && needs.setup.outputs.unique_tag || 'head' }}" >> "$GITHUB_OUTPUT" datahub_ingestion_base_slim_build: name: Build and Push DataHub Ingestion (Base-Slim) Docker Image runs-on: ubuntu-latest @@ -575,6 +580,7 @@ jobs: images: | ${{ env.DATAHUB_INGESTION_BASE_IMAGE }} image_tag: ${{ needs.setup.outputs.slim_tag }} + flavor: slim username: ${{ secrets.ACRYL_DOCKER_USERNAME }} password: ${{ secrets.ACRYL_DOCKER_PASSWORD }} build-args: | @@ -584,9 +590,10 @@ jobs: context: . file: ./docker/datahub-ingestion-base/Dockerfile platforms: linux/amd64,linux/arm64/v8 + depot-project: ${{ vars.DEPOT_PROJECT_ID }} - name: Compute DataHub Ingestion (Base-Slim) Tag id: tag - run: echo "tag=${{ needs.setup.outputs.ingestion_base_change == 'true' && needs.setup.outputs.unique_slim_tag || 'head-slim' }}" >> $GITHUB_OUTPUT + run: echo "tag=${{ needs.setup.outputs.ingestion_base_change == 'true' && needs.setup.outputs.unique_slim_tag || 'head-slim' }}" >> "$GITHUB_OUTPUT" datahub_ingestion_base_full_build: name: Build and Push DataHub Ingestion (Base-Full) Docker Image runs-on: ubuntu-latest @@ -627,7 +634,7 @@ jobs: platforms: linux/amd64,linux/arm64/v8 - name: Compute DataHub Ingestion (Base-Full) Tag id: tag - run: echo "tag=${{ needs.setup.outputs.ingestion_base_change == 'true' && needs.setup.outputs.unique_full_tag || 'head' }}" >> $GITHUB_OUTPUT + run: echo "tag=${{ needs.setup.outputs.ingestion_base_change == 'true' && needs.setup.outputs.unique_full_tag || 'head' }}" >> "$GITHUB_OUTPUT" datahub_ingestion_slim_build: name: Build and Push DataHub Ingestion Docker Images @@ -673,15 +680,17 @@ jobs: RELEASE_VERSION=${{ needs.setup.outputs.python_release_version }} APP_ENV=slim image_tag: ${{ needs.setup.outputs.slim_tag }} + flavor: slim username: ${{ secrets.ACRYL_DOCKER_USERNAME }} password: ${{ secrets.ACRYL_DOCKER_PASSWORD }} publish: ${{ needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }} context: . file: ./docker/datahub-ingestion/Dockerfile platforms: linux/amd64,linux/arm64/v8 + depot-project: ${{ vars.DEPOT_PROJECT_ID }} - name: Compute Tag id: tag - run: echo "tag=${{ needs.setup.outputs.ingestion_change == 'true' && needs.setup.outputs.unique_slim_tag || 'head-slim' }}" >> $GITHUB_OUTPUT + run: echo "tag=${{ needs.setup.outputs.ingestion_change == 'true' && needs.setup.outputs.unique_slim_tag || 'head-slim' }}" >> "$GITHUB_OUTPUT" datahub_ingestion_slim_scan: permissions: contents: read # for actions/checkout to fetch code @@ -711,6 +720,7 @@ jobs: severity: "CRITICAL,HIGH" ignore-unfixed: true vuln-type: "os,library" + timeout: 15m - name: Upload Trivy scan results to GitHub Security tab uses: github/codeql-action/upload-sarif@v2 with: @@ -765,9 +775,10 @@ jobs: context: . file: ./docker/datahub-ingestion/Dockerfile platforms: linux/amd64,linux/arm64/v8 + depot-project: ${{ vars.DEPOT_PROJECT_ID }} - name: Compute Tag (Full) id: tag - run: echo "tag=${{ needs.setup.outputs.ingestion_change == 'true' && needs.setup.outputs.unique_tag || 'head' }}" >> $GITHUB_OUTPUT + run: echo "tag=${{ needs.setup.outputs.ingestion_change == 'true' && needs.setup.outputs.unique_tag || 'head' }}" >> "$GITHUB_OUTPUT" datahub_ingestion_full_scan: permissions: contents: read # for actions/checkout to fetch code @@ -797,6 +808,7 @@ jobs: severity: "CRITICAL,HIGH" ignore-unfixed: true vuln-type: "os,library" + timeout: 15m - name: Upload Trivy scan results to GitHub Security tab uses: github/codeql-action/upload-sarif@v2 with: @@ -811,13 +823,13 @@ jobs: - id: set-matrix run: | if [ '${{ needs.setup.outputs.frontend_only }}' == 'true' ]; then - echo 'matrix=["cypress_suite1","cypress_rest"]' >> $GITHUB_OUTPUT + echo 'matrix=["cypress_suite1","cypress_rest"]' >> "$GITHUB_OUTPUT" elif [ '${{ needs.setup.outputs.ingestion_only }}' == 'true' ]; then - echo 'matrix=["no_cypress_suite0","no_cypress_suite1"]' >> $GITHUB_OUTPUT + echo 'matrix=["no_cypress_suite0","no_cypress_suite1"]' >> "$GITHUB_OUTPUT" elif [[ '${{ needs.setup.outputs.backend_change }}' == 'true' || '${{ needs.setup.outputs.smoke_test_change }}' == 'true' ]]; then - echo 'matrix=["no_cypress_suite0","no_cypress_suite1","cypress_suite1","cypress_rest"]' >> $GITHUB_OUTPUT + echo 'matrix=["no_cypress_suite0","no_cypress_suite1","cypress_suite1","cypress_rest"]' >> "$GITHUB_OUTPUT" else - echo 'matrix=[]' >> $GITHUB_OUTPUT + echo 'matrix=[]' >> "$GITHUB_OUTPUT" fi smoke_test: @@ -956,6 +968,13 @@ jobs: docker pull '${{ env.DATAHUB_ELASTIC_SETUP_IMAGE }}:head' docker tag '${{ env.DATAHUB_ELASTIC_SETUP_IMAGE }}:head' '${{ env.DATAHUB_ELASTIC_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }}' fi + if [ '${{ needs.setup.outputs.integrations_service_change }}' == 'false' ]; then + echo 'datahub-integration-service head images' + docker pull '${{ env.DATAHUB_INTEGRATIONS_IMAGE }}:head' + docker tag '${{ env.DATAHUB_INTEGRATIONS_IMAGE }}:head' '${{ env.DATAHUB_INTEGRATIONS_IMAGE }}:${{ needs.setup.outputs.unique_tag }}' + fi + - name: CI Slim Head Images + run: | if [ '${{ needs.setup.outputs.ingestion_change }}' == 'false' ]; then echo 'datahub-ingestion head-slim images' docker pull '${{ env.DATAHUB_INGESTION_IMAGE }}:head-slim' @@ -971,7 +990,7 @@ jobs: DATAHUB_VERSION: ${{ needs.setup.outputs.unique_tag }} DATAHUB_ACTIONS_IMAGE: ${{ env.DATAHUB_INGESTION_IMAGE }} ACTIONS_VERSION: ${{ needs.datahub_ingestion_slim_build.outputs.tag || 'head-slim' }} - ACTIONS_EXTRA_PACKAGES: "acryl-datahub-actions[executor]==0.0.13 acryl-datahub-actions==0.0.13 acryl-datahub==0.10.5" + ACTIONS_EXTRA_PACKAGES: "acryl-datahub-actions[executor] acryl-datahub-actions" ACTIONS_CONFIG: "https://raw.githubusercontent.com/acryldata/datahub-actions/main/docker/config/executor.yaml" run: | ./smoke-test/run-quickstart.sh diff --git a/.github/workflows/gx-plugin.yml b/.github/workflows/gx-plugin.yml index 06a10883e70f54..99121f81099f2d 100644 --- a/.github/workflows/gx-plugin.yml +++ b/.github/workflows/gx-plugin.yml @@ -57,7 +57,7 @@ jobs: run: ./gradlew -Pextra_pip_requirements='${{ matrix.extraPythonRequirement }}' :metadata-ingestion-modules:gx-plugin:lint :metadata-ingestion-modules:gx-plugin:testQuick - name: pip freeze show list installed if: always() - run: source metadata-ingestion-modules/gx-plugin/venv/bin/activate && pip freeze + run: source metadata-ingestion-modules/gx-plugin/venv/bin/activate && uv pip freeze - uses: actions/upload-artifact@v3 if: ${{ always() && matrix.python-version == '3.11' && matrix.extraPythonRequirement == 'great-expectations~=0.17.0' }} with: diff --git a/.github/workflows/metadata-ingestion.yml b/.github/workflows/metadata-ingestion.yml index cfb3693d89381f..c718febca398a9 100644 --- a/.github/workflows/metadata-ingestion.yml +++ b/.github/workflows/metadata-ingestion.yml @@ -83,7 +83,7 @@ jobs: - name: Debug info if: always() run: | - source metadata-ingestion/venv/bin/activate && pip freeze + source metadata-ingestion/venv/bin/activate && uv pip freeze set -x df -hl docker image ls diff --git a/.github/workflows/prefect-plugin.yml b/.github/workflows/prefect-plugin.yml index 09af0ad3f354a3..b0af00f92b7727 100644 --- a/.github/workflows/prefect-plugin.yml +++ b/.github/workflows/prefect-plugin.yml @@ -43,7 +43,7 @@ jobs: with: distribution: "zulu" java-version: 17 - - uses: gradle/gradle-build-action@v2 + - uses: gradle/actions/setup-gradle@v3 - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: @@ -55,7 +55,7 @@ jobs: run: ./gradlew :metadata-ingestion-modules:prefect-plugin:lint :metadata-ingestion-modules:prefect-plugin:testQuick - name: pip freeze show list installed if: always() - run: source metadata-ingestion-modules/prefect-plugin/venv/bin/activate && pip freeze + run: source metadata-ingestion-modules/prefect-plugin/venv/bin/activate && uv pip freeze - uses: actions/upload-artifact@v3 if: ${{ always() && matrix.python-version == '3.10'}} with: diff --git a/README.md b/README.md index a0a41b692676ea..1089c4dbc055c4 100644 --- a/README.md +++ b/README.md @@ -133,6 +133,7 @@ Here are the companies that have officially adopted DataHub. Please feel free to - [Haibo Technology](https://www.botech.com.cn) - [hipages](https://hipages.com.au/) - [inovex](https://www.inovex.de/) +- [Inter&Co](https://inter.co/) - [IOMED](https://iomed.health) - [Klarna](https://www.klarna.com) - [LinkedIn](http://linkedin.com) diff --git a/build.gradle b/build.gradle index fbced335ddc2e7..016bfcbe62c41e 100644 --- a/build.gradle +++ b/build.gradle @@ -34,8 +34,8 @@ buildscript { // Releases: https://github.com/linkedin/rest.li/blob/master/CHANGELOG.md ext.pegasusVersion = '29.57.0' ext.mavenVersion = '3.6.3' - ext.springVersion = '6.1.6' - ext.springBootVersion = '3.2.6' + ext.springVersion = '6.1.13' + ext.springBootVersion = '3.2.9' ext.springKafkaVersion = '3.1.6' ext.openTelemetryVersion = '1.18.0' ext.neo4jVersion = '5.14.0' @@ -52,7 +52,7 @@ buildscript { ext.hadoop3Version = '3.3.6' ext.kafkaVersion = '5.5.15' ext.hazelcastVersion = '5.3.6' - ext.ebeanVersion = '12.16.1' + ext.ebeanVersion = '15.5.2' ext.googleJavaFormatVersion = '1.18.1' ext.openLineageVersion = '1.19.0' ext.logbackClassicJava8 = '1.2.12' @@ -104,8 +104,8 @@ project.ext.spec = [ project.ext.externalDependency = [ 'akkaHttp': 'com.typesafe.akka:akka-http-core_2.12:10.2.10', - 'antlr4Runtime': 'org.antlr:antlr4-runtime:4.7.2', - 'antlr4': 'org.antlr:antlr4:4.7.2', + 'antlr4Runtime': 'org.antlr:antlr4-runtime:4.9.3', + 'antlr4': 'org.antlr:antlr4:4.9.3', 'assertJ': 'org.assertj:assertj-core:3.11.1', 'avro': 'org.apache.avro:avro:1.11.3', 'avroCompiler': 'org.apache.avro:avro-compiler:1.11.3', @@ -129,8 +129,10 @@ project.ext.externalDependency = [ 'dropwizardMetricsCore': 'io.dropwizard.metrics:metrics-core:4.2.3', 'dropwizardMetricsJmx': 'io.dropwizard.metrics:metrics-jmx:4.2.3', 'ebean': 'io.ebean:ebean:' + ebeanVersion, + 'ebeanTest': 'io.ebean:ebean-test:' + ebeanVersion, 'ebeanAgent': 'io.ebean:ebean-agent:' + ebeanVersion, 'ebeanDdl': 'io.ebean:ebean-ddl-generator:' + ebeanVersion, + 'ebeanQueryBean': 'io.ebean:querybean-generator:' + ebeanVersion, 'elasticSearchRest': 'org.opensearch.client:opensearch-rest-high-level-client:' + elasticsearchVersion, 'elasticSearchJava': 'org.opensearch.client:opensearch-java:2.6.0', 'findbugsAnnotations': 'com.google.code.findbugs:annotations:3.0.1', @@ -220,7 +222,7 @@ project.ext.externalDependency = [ 'playServer': "com.typesafe.play:play-server_2.12:$playVersion", 'playTest': "com.typesafe.play:play-test_2.12:$playVersion", 'playFilters': "com.typesafe.play:filters-helpers_2.12:$playVersion", - 'pac4j': 'org.pac4j:pac4j-oidc:4.5.7', + 'pac4j': 'org.pac4j:pac4j-oidc:4.5.8', 'playPac4j': 'org.pac4j:play-pac4j_2.12:9.0.2', 'postgresql': 'org.postgresql:postgresql:42.3.9', 'protobuf': 'com.google.protobuf:protobuf-java:3.19.6', @@ -265,7 +267,7 @@ project.ext.externalDependency = [ 'testContainersOpenSearch': 'org.opensearch:opensearch-testcontainers:2.0.0', 'typesafeConfig':'com.typesafe:config:1.4.1', 'wiremock':'com.github.tomakehurst:wiremock:2.10.0', - 'zookeeper': 'org.apache.zookeeper:zookeeper:3.7.2', + 'zookeeper': 'org.apache.zookeeper:zookeeper:3.6.2', 'wire': 'com.squareup.wire:wire-compiler:3.7.1', 'charle': 'com.charleskorn.kaml:kaml:0.53.0', 'common': 'commons-io:commons-io:2.7', @@ -359,6 +361,9 @@ configure(subprojects.findAll {! it.name.startsWith('spark-lineage')}) { exclude group: "org.slf4j", module: "slf4j-log4j12" exclude group: "org.slf4j", module: "slf4j-nop" exclude group: "org.slf4j", module: "slf4j-ext" + + resolutionStrategy.force externalDependency.antlr4Runtime + resolutionStrategy.force externalDependency.antlr4 } } diff --git a/datahub-frontend/app/auth/AuthModule.java b/datahub-frontend/app/auth/AuthModule.java index 32dfba00d47dbf..d0d17fda263926 100644 --- a/datahub-frontend/app/auth/AuthModule.java +++ b/datahub-frontend/app/auth/AuthModule.java @@ -25,7 +25,7 @@ import java.util.Collections; import io.datahubproject.metadata.context.ActorContext; -import io.datahubproject.metadata.context.AuthorizerContext; +import io.datahubproject.metadata.context.AuthorizationContext; import io.datahubproject.metadata.context.EntityRegistryContext; import io.datahubproject.metadata.context.OperationContext; import io.datahubproject.metadata.context.OperationContextConfig; @@ -183,10 +183,10 @@ protected OperationContext provideOperationContext( return OperationContext.builder() .operationContextConfig(systemConfig) .systemActorContext(systemActorContext) + // Authorizer.EMPTY is fine since it doesn't actually apply to system auth + .authorizationContext(AuthorizationContext.builder().authorizer(Authorizer.EMPTY).build()) .searchContext(SearchContext.EMPTY) .entityRegistryContext(EntityRegistryContext.builder().build(EmptyEntityRegistry.EMPTY)) - // Authorizer.EMPTY doesn't actually apply to system auth - .authorizerContext(AuthorizerContext.builder().authorizer(Authorizer.EMPTY).build()) .build(systemAuthentication); } diff --git a/datahub-frontend/app/client/KafkaTrackingProducer.java b/datahub-frontend/app/client/KafkaTrackingProducer.java index b7173684b63500..058e75100c24ac 100644 --- a/datahub-frontend/app/client/KafkaTrackingProducer.java +++ b/datahub-frontend/app/client/KafkaTrackingProducer.java @@ -1,5 +1,6 @@ package client; +import com.linkedin.metadata.config.kafka.KafkaConfiguration; import com.linkedin.metadata.config.kafka.ProducerConfiguration; import com.typesafe.config.Config; import config.ConfigurationProvider; @@ -46,7 +47,7 @@ public KafkaTrackingProducer( if (_isEnabled) { _logger.debug("Analytics tracking is enabled"); - _producer = createKafkaProducer(config, configurationProvider.getKafka().getProducer()); + _producer = createKafkaProducer(config, configurationProvider.getKafka()); lifecycle.addStopHook( () -> { @@ -69,7 +70,8 @@ public void send(ProducerRecord record) { } private static KafkaProducer createKafkaProducer( - Config config, ProducerConfiguration producerConfiguration) { + Config config, KafkaConfiguration kafkaConfiguration) { + final ProducerConfiguration producerConfiguration = kafkaConfiguration.getProducer(); final Properties props = new Properties(); props.put(ProducerConfig.CLIENT_ID_CONFIG, "datahub-frontend"); props.put( @@ -78,12 +80,9 @@ private static KafkaProducer createKafkaProducer( props.put( ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, config.getString("analytics.kafka.bootstrap.server")); - props.put( - ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, - "org.apache.kafka.common.serialization.StringSerializer"); // Actor urn. - props.put( - ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, - "org.apache.kafka.common.serialization.StringSerializer"); // JSON object. + // key: Actor urn. + // value: JSON object. + props.putAll(kafkaConfiguration.getSerde().getUsageEvent().getProducerProperties(null)); props.put(ProducerConfig.MAX_REQUEST_SIZE_CONFIG, producerConfiguration.getMaxRequestSize()); props.put(ProducerConfig.COMPRESSION_TYPE_CONFIG, producerConfiguration.getCompressionType()); diff --git a/datahub-frontend/app/controllers/Application.java b/datahub-frontend/app/controllers/Application.java index d17e600aadc072..017847367de053 100644 --- a/datahub-frontend/app/controllers/Application.java +++ b/datahub-frontend/app/controllers/Application.java @@ -9,12 +9,15 @@ import akka.util.ByteString; import auth.Authenticator; import com.datahub.authentication.AuthenticationConstants; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.linkedin.util.Pair; import com.typesafe.config.Config; import java.io.InputStream; import java.net.URI; import java.time.Duration; +import java.time.Instant; import java.util.List; import java.util.Map; import java.util.Optional; @@ -33,6 +36,7 @@ import play.libs.ws.StandaloneWSClient; import play.libs.ws.ahc.StandaloneAhcWSClient; import play.mvc.Controller; +import play.mvc.Http.Cookie; import play.mvc.Http; import play.mvc.ResponseHeader; import play.mvc.Result; @@ -132,6 +136,9 @@ public CompletableFuture proxy(String path, Http.Request request) headers.put(Http.HeaderNames.X_FORWARDED_PROTO, List.of(schema)); } + // Get the current time to measure the duration of the request + Instant start = Instant.now(); + return _ws.url( String.format( "%s://%s:%s%s", protocol, metadataServiceHost, metadataServicePort, resolvedUri)) @@ -160,6 +167,15 @@ AuthenticationConstants.LEGACY_X_DATAHUB_ACTOR_HEADER, getDataHubActorHeader(req .execute() .thenApply( apiResponse -> { + // Log the query if it takes longer than the configured threshold and verbose logging is enabled + boolean verboseGraphQLLogging = _config.getBoolean("graphql.verbose.logging"); + int verboseGraphQLLongQueryMillis = _config.getInt("graphql.verbose.slowQueryMillis"); + Instant finish = Instant.now(); + long timeElapsed = Duration.between(start, finish).toMillis(); + if (verboseGraphQLLogging && timeElapsed >= verboseGraphQLLongQueryMillis) { + logSlowQuery(request, resolvedUri, timeElapsed); + } + final ResponseHeader header = new ResponseHeader( apiResponse.getStatus(), @@ -359,4 +375,34 @@ private String mapPath(@Nonnull final String path) { // Otherwise, return original path return path; } + + + /** + * Called if verbose logging is enabled and request takes longer that the slow query milliseconds defined in the config + * @param request GraphQL request that was made + * @param resolvedUri URI that was requested + * @param duration How long the query took to complete + */ + private void logSlowQuery(Http.Request request, String resolvedUri, float duration) { + StringBuilder jsonBody = new StringBuilder(); + Optional actorCookie = request.getCookie("actor"); + String actorValue = actorCookie.isPresent() ? actorCookie.get().value() : "N/A"; + + try { + ObjectMapper mapper = new ObjectMapper(); + JsonNode jsonNode = request.body().asJson(); + ((ObjectNode) jsonNode).remove("query"); + jsonBody.append(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(jsonNode)); + } + catch (Exception e) { + _logger.info("GraphQL Request Received: {}, Unable to parse JSON body", resolvedUri); + } + String jsonBodyStr = jsonBody.toString(); + _logger.info("Slow GraphQL Request Received: {}, Request query string: {}, Request actor: {}, Request JSON: {}, Request completed in {} ms", + resolvedUri, + request.queryString(), + actorValue, + jsonBodyStr, + duration); + } } diff --git a/datahub-frontend/app/utils/SearchUtil.java b/datahub-frontend/app/utils/SearchUtil.java index 803c70a63646a0..cfbf11c2ffa85a 100644 --- a/datahub-frontend/app/utils/SearchUtil.java +++ b/datahub-frontend/app/utils/SearchUtil.java @@ -20,7 +20,7 @@ private SearchUtil() { @Nonnull public static String escapeForwardSlash(@Nonnull String input) { if (input.contains("/")) { - input = input.replace("/", "\\\\/"); + input = input.replace("/", "\\/"); } return input; } diff --git a/datahub-frontend/conf/application.conf b/datahub-frontend/conf/application.conf index 63ff2c9166fbc9..be57a33b13564d 100644 --- a/datahub-frontend/conf/application.conf +++ b/datahub-frontend/conf/application.conf @@ -298,4 +298,10 @@ entityClient.numRetries = ${?ENTITY_CLIENT_NUM_RETRIES} entityClient.restli.get.batchSize = 50 entityClient.restli.get.batchSize = ${?ENTITY_CLIENT_RESTLI_GET_BATCH_SIZE} entityClient.restli.get.batchConcurrency = 2 -entityClient.restli.get.batchConcurrency = ${?ENTITY_CLIENT_RESTLI_GET_BATCH_CONCURRENCY} \ No newline at end of file +entityClient.restli.get.batchConcurrency = ${?ENTITY_CLIENT_RESTLI_GET_BATCH_CONCURRENCY} + +# Enable verbose authentication logging +graphql.verbose.logging = false +graphql.verbose.logging = ${?GRAPHQL_VERBOSE_LOGGING} +graphql.verbose.slowQueryMillis = 2500 +graphql.verbose.slowQueryMillis = ${?GRAPHQL_VERBOSE_LONG_QUERY_MILLIS} \ No newline at end of file diff --git a/datahub-frontend/test/utils/SearchUtilTest.java b/datahub-frontend/test/utils/SearchUtilTest.java index 6767fa56374692..2efc4420c7f766 100644 --- a/datahub-frontend/test/utils/SearchUtilTest.java +++ b/datahub-frontend/test/utils/SearchUtilTest.java @@ -8,10 +8,10 @@ public class SearchUtilTest { @Test public void testEscapeForwardSlash() { // escape "/" - assertEquals("\\\\/foo\\\\/bar", SearchUtil.escapeForwardSlash("/foo/bar")); + assertEquals("\\/foo\\/bar", SearchUtil.escapeForwardSlash("/foo/bar")); // "/" is escaped but "*" is not escaped and is treated as regex. Since currently we want to // retain the regex behaviour with "*" - assertEquals("\\\\/foo\\\\/bar\\\\/*", SearchUtil.escapeForwardSlash("/foo/bar/*")); + assertEquals("\\/foo\\/bar\\/*", SearchUtil.escapeForwardSlash("/foo/bar/*")); assertEquals("", ""); assertEquals("foo", "foo"); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java index b470da3c7c74aa..d38c1030b61be8 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java @@ -293,6 +293,7 @@ import com.linkedin.datahub.graphql.resolvers.step.BatchGetStepStatesResolver; import com.linkedin.datahub.graphql.resolvers.step.BatchUpdateStepStatesResolver; import com.linkedin.datahub.graphql.resolvers.structuredproperties.CreateStructuredPropertyResolver; +import com.linkedin.datahub.graphql.resolvers.structuredproperties.DeleteStructuredPropertyResolver; import com.linkedin.datahub.graphql.resolvers.structuredproperties.RemoveStructuredPropertiesResolver; import com.linkedin.datahub.graphql.resolvers.structuredproperties.UpdateStructuredPropertyResolver; import com.linkedin.datahub.graphql.resolvers.structuredproperties.UpsertStructuredPropertiesResolver; @@ -724,7 +725,7 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) { * Returns a {@link Supplier} responsible for creating a new {@link DataLoader} from a {@link * LoadableType}. */ - public Map>> loaderSuppliers( + public static Map>> loaderSuppliers( final Collection> loadableTypes) { return loadableTypes.stream() .collect( @@ -1034,6 +1035,7 @@ private void configureQueryResolvers(final RuntimeWiring.Builder builder) { .dataFetcher("assertion", getResolver(assertionType)) .dataFetcher("form", getResolver(formType)) .dataFetcher("view", getResolver(dataHubViewType)) + .dataFetcher("structuredProperty", getResolver(structuredPropertyType)) .dataFetcher("listPolicies", new ListPoliciesResolver(this.entityClient)) .dataFetcher("getGrantedPrivileges", new GetGrantedPrivilegesResolver()) .dataFetcher("listUsers", new ListUsersResolver(this.entityClient)) @@ -1135,16 +1137,16 @@ private DataFetcher getEntityResolver() { }); } - private DataFetcher getResolver(LoadableType loadableType) { - return getResolver(loadableType, this::getUrnField); + private static DataFetcher getResolver(LoadableType loadableType) { + return getResolver(loadableType, GmsGraphQLEngine::getUrnField); } - private DataFetcher getResolver( + private static DataFetcher getResolver( LoadableType loadableType, Function keyProvider) { return new LoadableTypeResolver<>(loadableType, keyProvider); } - private String getUrnField(DataFetchingEnvironment env) { + private static String getUrnField(DataFetchingEnvironment env) { return env.getArgument(URN_FIELD_NAME); } @@ -1343,6 +1345,9 @@ private void configureMutationResolvers(final RuntimeWiring.Builder builder) { .dataFetcher( "updateStructuredProperty", new UpdateStructuredPropertyResolver(this.entityClient)) + .dataFetcher( + "deleteStructuredProperty", + new DeleteStructuredPropertyResolver(this.entityClient)) .dataFetcher("raiseIncident", new RaiseIncidentResolver(this.entityClient)) .dataFetcher( "updateIncidentStatus", @@ -2116,6 +2121,9 @@ private void configureStructuredPropertyResolvers(final RuntimeWiring.Builder bu .getAllowedTypes().stream() .map(entityTypeType.getKeyProvider()) .collect(Collectors.toList())))); + builder.type( + "StructuredPropertyEntity", + typeWiring -> typeWiring.dataFetcher("exists", new EntityExistsResolver(entityService))); } /** @@ -3025,7 +3033,7 @@ private void configureTestResultResolvers(final RuntimeWiring.Builder builder) { }))); } - private DataLoader> createDataLoader( + private static DataLoader> createDataLoader( final LoadableType graphType, final QueryContext queryContext) { BatchLoaderContextProvider contextProvider = () -> queryContext; DataLoaderOptions loaderOptions = diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GraphQLEngine.java index dd8eabd3ce06fd..97e282c106a9a2 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GraphQLEngine.java @@ -28,7 +28,6 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import org.dataloader.DataLoader; -import org.dataloader.DataLoaderRegistry; /** * Simple wrapper around a {@link GraphQL} instance providing APIs for building an engine and @@ -100,7 +99,7 @@ public ExecutionResult execute( /* * Init DataLoaderRegistry - should be created for each request. */ - DataLoaderRegistry register = createDataLoaderRegistry(_dataLoaderSuppliers, context); + LazyDataLoaderRegistry register = new LazyDataLoaderRegistry(context, _dataLoaderSuppliers); /* * Construct execution input @@ -218,14 +217,4 @@ public GraphQLEngine build() { graphQLQueryIntrospectionEnabled); } } - - private DataLoaderRegistry createDataLoaderRegistry( - final Map>> dataLoaderSuppliers, - final QueryContext context) { - final DataLoaderRegistry registry = new DataLoaderRegistry(); - for (String key : dataLoaderSuppliers.keySet()) { - registry.register(key, dataLoaderSuppliers.get(key).apply(context)); - } - return registry; - } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/LazyDataLoaderRegistry.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/LazyDataLoaderRegistry.java new file mode 100644 index 00000000000000..1a1d2d5f71f336 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/LazyDataLoaderRegistry.java @@ -0,0 +1,53 @@ +package com.linkedin.datahub.graphql; + +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import lombok.extern.slf4j.Slf4j; +import org.dataloader.DataLoader; +import org.dataloader.DataLoaderRegistry; + +/** + * The purpose of this class is to avoid loading 42+ dataLoaders when many of the graphql queries do + * not use all of them. + */ +@Slf4j +public class LazyDataLoaderRegistry extends DataLoaderRegistry { + private final QueryContext queryContext; + private final Map>> dataLoaderSuppliers; + + public LazyDataLoaderRegistry( + QueryContext queryContext, + Map>> dataLoaderSuppliers) { + super(); + this.queryContext = queryContext; + this.dataLoaderSuppliers = new ConcurrentHashMap<>(dataLoaderSuppliers); + } + + @Override + public DataLoader getDataLoader(String key) { + return super.computeIfAbsent( + key, + k -> { + Function> supplier = dataLoaderSuppliers.get(key); + if (supplier == null) { + throw new IllegalArgumentException("No DataLoader registered for key: " + key); + } + return supplier.apply(queryContext); + }); + } + + @Override + public Set getKeys() { + return Stream.concat(dataLoaders.keySet().stream(), dataLoaderSuppliers.keySet().stream()) + .collect(Collectors.toSet()); + } + + @Override + public DataLoaderRegistry combine(DataLoaderRegistry registry) { + throw new UnsupportedOperationException(); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/QueryContext.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/QueryContext.java index 7dffd90cf2d7cc..5ad82b5d703757 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/QueryContext.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/QueryContext.java @@ -3,6 +3,7 @@ import com.datahub.authentication.Actor; import com.datahub.authentication.Authentication; import com.datahub.plugins.auth.authorization.Authorizer; +import com.linkedin.metadata.config.DataHubAppConfiguration; import io.datahubproject.metadata.context.OperationContext; /** Provided as input to GraphQL resolvers; used to carry information about GQL request context. */ @@ -31,4 +32,6 @@ default String getActorUrn() { * @return Returns the operational context */ OperationContext getOperationContext(); + + DataHubAppConfiguration getDataHubAppConfig(); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java index 4fb49d79a0aa70..ca60acaa805387 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java @@ -11,7 +11,6 @@ import com.datahub.authorization.ConjunctivePrivilegeGroup; import com.datahub.authorization.DisjunctivePrivilegeGroup; import com.datahub.authorization.EntitySpec; -import com.datahub.plugins.auth.authorization.Authorizer; import com.google.common.collect.ImmutableList; import com.linkedin.common.urn.Urn; import com.linkedin.datahub.graphql.QueryContext; @@ -21,7 +20,6 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.List; -import java.util.Set; import javax.annotation.Nonnull; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.reflect.ConstructorUtils; @@ -40,29 +38,25 @@ public class AuthorizationUtils { public static boolean canManageUsersAndGroups(@Nonnull QueryContext context) { return AuthUtil.isAuthorizedEntityType( - context.getActorUrn(), - context.getAuthorizer(), + context.getOperationContext(), MANAGE, List.of(CORP_USER_ENTITY_NAME, CORP_GROUP_ENTITY_NAME)); } public static boolean canManagePolicies(@Nonnull QueryContext context) { return AuthUtil.isAuthorizedEntityType( - context.getActorUrn(), context.getAuthorizer(), MANAGE, List.of(POLICY_ENTITY_NAME)); + context.getOperationContext(), MANAGE, List.of(POLICY_ENTITY_NAME)); } public static boolean canGeneratePersonalAccessToken(@Nonnull QueryContext context) { return AuthUtil.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - PoliciesConfig.GENERATE_PERSONAL_ACCESS_TOKENS_PRIVILEGE) - || AuthUtil.isAuthorized( - context.getAuthorizer(), context.getActorUrn(), MANAGE_ACCESS_TOKENS); + context.getOperationContext(), PoliciesConfig.GENERATE_PERSONAL_ACCESS_TOKENS_PRIVILEGE) + || AuthUtil.isAuthorized(context.getOperationContext(), MANAGE_ACCESS_TOKENS); } public static boolean canManageTokens(@Nonnull QueryContext context) { return AuthUtil.isAuthorizedEntityType( - context.getActorUrn(), context.getAuthorizer(), MANAGE, List.of(ACCESS_TOKEN_ENTITY_NAME)); + context.getOperationContext(), MANAGE, List.of(ACCESS_TOKEN_ENTITY_NAME)); } /** @@ -78,13 +72,12 @@ public static boolean canCreateDomains(@Nonnull QueryContext context) { new ConjunctivePrivilegeGroup( ImmutableList.of(PoliciesConfig.MANAGE_DOMAINS_PRIVILEGE.getType())))); - return AuthUtil.isAuthorized( - context.getAuthorizer(), context.getActorUrn(), orPrivilegeGroups, null); + return AuthUtil.isAuthorized(context.getOperationContext(), orPrivilegeGroups, null); } public static boolean canManageDomains(@Nonnull QueryContext context) { return AuthUtil.isAuthorized( - context.getAuthorizer(), context.getActorUrn(), PoliciesConfig.MANAGE_DOMAINS_PRIVILEGE); + context.getOperationContext(), PoliciesConfig.MANAGE_DOMAINS_PRIVILEGE); } /** @@ -100,25 +93,22 @@ public static boolean canCreateTags(@Nonnull QueryContext context) { new ConjunctivePrivilegeGroup( ImmutableList.of(PoliciesConfig.MANAGE_TAGS_PRIVILEGE.getType())))); - return AuthUtil.isAuthorized( - context.getAuthorizer(), context.getActorUrn(), orPrivilegeGroups, null); + return AuthUtil.isAuthorized(context.getOperationContext(), orPrivilegeGroups, null); } public static boolean canManageTags(@Nonnull QueryContext context) { return AuthUtil.isAuthorized( - context.getAuthorizer(), context.getActorUrn(), PoliciesConfig.MANAGE_TAGS_PRIVILEGE); + context.getOperationContext(), PoliciesConfig.MANAGE_TAGS_PRIVILEGE); } public static boolean canDeleteEntity(@Nonnull Urn entityUrn, @Nonnull QueryContext context) { return AuthUtil.isAuthorizedEntityUrns( - context.getAuthorizer(), context.getActorUrn(), DELETE, List.of(entityUrn)); + context.getOperationContext(), DELETE, List.of(entityUrn)); } public static boolean canManageUserCredentials(@Nonnull QueryContext context) { return AuthUtil.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - PoliciesConfig.MANAGE_USER_CREDENTIALS_PRIVILEGE); + context.getOperationContext(), PoliciesConfig.MANAGE_USER_CREDENTIALS_PRIVILEGE); } public static boolean canEditGroupMembers( @@ -130,12 +120,7 @@ public static boolean canEditGroupMembers( new ConjunctivePrivilegeGroup( ImmutableList.of(PoliciesConfig.EDIT_GROUP_MEMBERS_PRIVILEGE.getType())))); - return isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - CORP_GROUP_ENTITY_NAME, - groupUrnStr, - orPrivilegeGroups); + return isAuthorized(context, CORP_GROUP_ENTITY_NAME, groupUrnStr, orPrivilegeGroups); } public static boolean canCreateGlobalAnnouncements(@Nonnull QueryContext context) { @@ -149,27 +134,21 @@ public static boolean canCreateGlobalAnnouncements(@Nonnull QueryContext context ImmutableList.of( PoliciesConfig.MANAGE_GLOBAL_ANNOUNCEMENTS_PRIVILEGE.getType())))); - return AuthUtil.isAuthorized( - context.getAuthorizer(), context.getActorUrn(), orPrivilegeGroups, null); + return AuthUtil.isAuthorized(context.getOperationContext(), orPrivilegeGroups, null); } public static boolean canManageGlobalAnnouncements(@Nonnull QueryContext context) { return AuthUtil.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - PoliciesConfig.MANAGE_GLOBAL_ANNOUNCEMENTS_PRIVILEGE); + context.getOperationContext(), PoliciesConfig.MANAGE_GLOBAL_ANNOUNCEMENTS_PRIVILEGE); } public static boolean canManageGlobalViews(@Nonnull QueryContext context) { - return AuthUtil.isAuthorized( - context.getAuthorizer(), context.getActorUrn(), PoliciesConfig.MANAGE_GLOBAL_VIEWS); + return AuthUtil.isAuthorized(context.getOperationContext(), PoliciesConfig.MANAGE_GLOBAL_VIEWS); } public static boolean canManageOwnershipTypes(@Nonnull QueryContext context) { return AuthUtil.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - PoliciesConfig.MANAGE_GLOBAL_OWNERSHIP_TYPES); + context.getOperationContext(), PoliciesConfig.MANAGE_GLOBAL_OWNERSHIP_TYPES); } public static boolean canEditProperties(@Nonnull Urn targetUrn, @Nonnull QueryContext context) { @@ -183,11 +162,7 @@ public static boolean canEditProperties(@Nonnull Urn targetUrn, @Nonnull QueryCo ImmutableList.of(PoliciesConfig.EDIT_ENTITY_PROPERTIES_PRIVILEGE.getType())))); return AuthorizationUtils.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - targetUrn.getEntityType(), - targetUrn.toString(), - orPrivilegeGroups); + context, targetUrn.getEntityType(), targetUrn.toString(), orPrivilegeGroups); } public static boolean canEditEntityQueries( @@ -202,11 +177,7 @@ public static boolean canEditEntityQueries( .allMatch( entityUrn -> isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - entityUrn.getEntityType(), - entityUrn.toString(), - orPrivilegeGroups)); + context, entityUrn.getEntityType(), entityUrn.toString(), orPrivilegeGroups)); } public static boolean canCreateQuery( @@ -251,28 +222,7 @@ public static boolean canView(@Nonnull OperationContext opContext, @Nonnull Urn && !opContext.isSystemAuth() && VIEW_RESTRICTED_ENTITY_TYPES.contains(urn.getEntityType())) { - return opContext - .getViewAuthorizationContext() - .map( - viewAuthContext -> { - - // check cache - if (viewAuthContext.canView(Set.of(urn))) { - return true; - } - - if (!canViewEntity( - opContext.getSessionAuthentication().getActor().toUrnStr(), - opContext.getAuthorizerContext().getAuthorizer(), - urn)) { - return false; - } - - // cache viewable urn - viewAuthContext.addViewableUrns(Set.of(urn)); - return true; - }) - .orElse(false); + return canViewEntity(opContext, urn); } return true; } @@ -386,38 +336,32 @@ public static T restrictEntity(@Nonnull Object entity, Class clazz) { public static boolean canManageStructuredProperties(@Nonnull QueryContext context) { return AuthUtil.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - PoliciesConfig.MANAGE_STRUCTURED_PROPERTIES_PRIVILEGE); + context.getOperationContext(), PoliciesConfig.MANAGE_STRUCTURED_PROPERTIES_PRIVILEGE); } public static boolean canManageForms(@Nonnull QueryContext context) { return AuthUtil.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - PoliciesConfig.MANAGE_DOCUMENTATION_FORMS_PRIVILEGE); + context.getOperationContext(), PoliciesConfig.MANAGE_DOCUMENTATION_FORMS_PRIVILEGE); } public static boolean canManageFeatures(@Nonnull QueryContext context) { return AuthUtil.isAuthorized( - context.getAuthorizer(), context.getActorUrn(), PoliciesConfig.MANAGE_FEATURES_PRIVILEGE); + context.getOperationContext(), PoliciesConfig.MANAGE_FEATURES_PRIVILEGE); } public static boolean isAuthorized( - @Nonnull Authorizer authorizer, - @Nonnull String actor, + @Nonnull QueryContext context, @Nonnull String resourceType, @Nonnull String resource, @Nonnull DisjunctivePrivilegeGroup privilegeGroup) { final EntitySpec resourceSpec = new EntitySpec(resourceType, resource); - return AuthUtil.isAuthorized(authorizer, actor, privilegeGroup, resourceSpec); + return AuthUtil.isAuthorized(context.getOperationContext(), privilegeGroup, resourceSpec); } public static boolean isViewDatasetUsageAuthorized( final QueryContext context, final Urn resourceUrn) { return AuthUtil.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), + context.getOperationContext(), PoliciesConfig.VIEW_DATASET_USAGE_PRIVILEGE, new EntitySpec(resourceUrn.getEntityType(), resourceUrn.toString())); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/MeResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/MeResolver.java index 1f4ebbb88bf1a6..b1101ae3ee8657 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/MeResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/MeResolver.java @@ -108,20 +108,19 @@ public CompletableFuture get(DataFetchingEnvironment environm /** Returns true if the authenticated user has privileges to view analytics. */ private boolean canViewAnalytics(final QueryContext context) { - return isAuthorized(context.getActorUrn(), context.getAuthorizer(), ANALYTICS, READ); + return isAuthorized(context.getOperationContext(), ANALYTICS, READ); } /** Returns true if the authenticated user has privileges to manage policies analytics. */ private boolean canManagePolicies(final QueryContext context) { return isAuthorizedEntityType( - context.getActorUrn(), context.getAuthorizer(), MANAGE, List.of(POLICY_ENTITY_NAME)); + context.getOperationContext(), MANAGE, List.of(POLICY_ENTITY_NAME)); } /** Returns true if the authenticated user has privileges to manage users & groups. */ private boolean canManageUsersGroups(final QueryContext context) { return isAuthorizedEntityType( - context.getActorUrn(), - context.getAuthorizer(), + context.getOperationContext(), MANAGE, List.of(CORP_USER_ENTITY_NAME, CORP_GROUP_ENTITY_NAME)); } @@ -129,46 +128,37 @@ private boolean canManageUsersGroups(final QueryContext context) { /** Returns true if the authenticated user has privileges to generate personal access tokens */ private boolean canGeneratePersonalAccessToken(final QueryContext context) { return isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - PoliciesConfig.GENERATE_PERSONAL_ACCESS_TOKENS_PRIVILEGE); + context.getOperationContext(), PoliciesConfig.GENERATE_PERSONAL_ACCESS_TOKENS_PRIVILEGE); } /** Returns true if the authenticated user has privileges to view tests. */ private boolean canViewTests(final QueryContext context) { - return isAuthorized( - context.getAuthorizer(), context.getActorUrn(), PoliciesConfig.VIEW_TESTS_PRIVILEGE); + return isAuthorized(context.getOperationContext(), PoliciesConfig.VIEW_TESTS_PRIVILEGE); } /** Returns true if the authenticated user has privileges to manage (add or remove) tests. */ private boolean canManageTests(final QueryContext context) { - return isAuthorized( - context.getAuthorizer(), context.getActorUrn(), PoliciesConfig.MANAGE_TESTS_PRIVILEGE); + return isAuthorized(context.getOperationContext(), PoliciesConfig.MANAGE_TESTS_PRIVILEGE); } /** Returns true if the authenticated user has privileges to manage domains */ private boolean canManageDomains(final QueryContext context) { - return isAuthorized( - context.getAuthorizer(), context.getActorUrn(), PoliciesConfig.MANAGE_DOMAINS_PRIVILEGE); + return isAuthorized(context.getOperationContext(), PoliciesConfig.MANAGE_DOMAINS_PRIVILEGE); } /** Returns true if the authenticated user has privileges to manage access tokens */ private boolean canManageTokens(final QueryContext context) { - return isAuthorized( - context.getAuthorizer(), context.getActorUrn(), PoliciesConfig.MANAGE_ACCESS_TOKENS); + return isAuthorized(context.getOperationContext(), PoliciesConfig.MANAGE_ACCESS_TOKENS); } /** Returns true if the authenticated user has privileges to manage glossaries */ private boolean canManageGlossaries(final QueryContext context) { - return isAuthorized( - context.getAuthorizer(), context.getActorUrn(), PoliciesConfig.MANAGE_GLOSSARIES_PRIVILEGE); + return isAuthorized(context.getOperationContext(), PoliciesConfig.MANAGE_GLOSSARIES_PRIVILEGE); } /** Returns true if the authenticated user has privileges to manage user credentials */ private boolean canManageUserCredentials(@Nonnull QueryContext context) { return isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - PoliciesConfig.MANAGE_USER_CREDENTIALS_PRIVILEGE); + context.getOperationContext(), PoliciesConfig.MANAGE_USER_CREDENTIALS_PRIVILEGE); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ResolverUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ResolverUtils.java index 3617eb47259797..5f873b4bebab32 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ResolverUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ResolverUtils.java @@ -6,8 +6,6 @@ import com.datahub.authentication.Authentication; import com.fasterxml.jackson.core.StreamReadConstraints; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.ImmutableSet; -import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; import com.linkedin.data.template.StringArray; import com.linkedin.datahub.graphql.QueryContext; @@ -15,15 +13,12 @@ import com.linkedin.datahub.graphql.generated.AndFilterInput; import com.linkedin.datahub.graphql.generated.FacetFilterInput; import com.linkedin.datahub.graphql.resolvers.search.SearchUtils; -import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.query.filter.Condition; import com.linkedin.metadata.query.filter.ConjunctiveCriterion; import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; import com.linkedin.metadata.query.filter.Criterion; import com.linkedin.metadata.query.filter.CriterionArray; import com.linkedin.metadata.query.filter.Filter; -import com.linkedin.metadata.search.utils.ESUtils; -import com.linkedin.metadata.search.utils.QueryUtils; import com.linkedin.metadata.service.ViewService; import com.linkedin.view.DataHubViewInfo; import graphql.schema.DataFetchingEnvironment; @@ -41,8 +36,6 @@ public class ResolverUtils { - private static final Set KEYWORD_EXCLUDED_FILTERS = - ImmutableSet.of("runId", "_entityType"); private static final ObjectMapper MAPPER = new ObjectMapper(); static { @@ -72,7 +65,7 @@ public static T bindArgument(Object argument, Class clazz) { @Nonnull public static String escapeForwardSlash(@Nonnull String input) { if (input.contains("/")) { - input = input.replace("/", "\\\\/"); + input = input.replace("/", "\\/"); } return input; } @@ -113,11 +106,10 @@ public static Map buildFacetFilters( return facetFilters; } - public static List criterionListFromAndFilter( - List andFilters, @Nullable AspectRetriever aspectRetriever) { + public static List criterionListFromAndFilter(List andFilters) { return andFilters != null && !andFilters.isEmpty() ? andFilters.stream() - .map(filter -> criterionFromFilter(filter, aspectRetriever)) + .map(filter -> criterionFromFilter(filter)) .collect(Collectors.toList()) : Collections.emptyList(); } @@ -126,14 +118,13 @@ public static List criterionListFromAndFilter( // conjunctive criterion // arrays, rather than just one for the AND case. public static ConjunctiveCriterionArray buildConjunctiveCriterionArrayWithOr( - @Nonnull List orFilters, @Nullable AspectRetriever aspectRetriever) { + @Nonnull List orFilters) { return new ConjunctiveCriterionArray( orFilters.stream() .map( orFilter -> { CriterionArray andCriterionForOr = - new CriterionArray( - criterionListFromAndFilter(orFilter.getAnd(), aspectRetriever)); + new CriterionArray(criterionListFromAndFilter(orFilter.getAnd())); return new ConjunctiveCriterion().setAnd(andCriterionForOr); }) .collect(Collectors.toList())); @@ -141,9 +132,7 @@ public static ConjunctiveCriterionArray buildConjunctiveCriterionArrayWithOr( @Nullable public static Filter buildFilter( - @Nullable List andFilters, - @Nullable List orFilters, - @Nullable AspectRetriever aspectRetriever) { + @Nullable List andFilters, @Nullable List orFilters) { if ((andFilters == null || andFilters.isEmpty()) && (orFilters == null || orFilters.isEmpty())) { return null; @@ -152,34 +141,21 @@ public static Filter buildFilter( // Or filters are the new default. We will check them first. // If we have OR filters, we need to build a series of CriterionArrays if (orFilters != null && !orFilters.isEmpty()) { - return new Filter().setOr(buildConjunctiveCriterionArrayWithOr(orFilters, aspectRetriever)); + return new Filter().setOr(buildConjunctiveCriterionArrayWithOr(orFilters)); } // If or filters are not set, someone may be using the legacy and filters - final List andCriterions = criterionListFromAndFilter(andFilters, aspectRetriever); + final List andCriterions = criterionListFromAndFilter(andFilters); return new Filter() .setOr( new ConjunctiveCriterionArray( new ConjunctiveCriterion().setAnd(new CriterionArray(andCriterions)))); } - public static Criterion criterionFromFilter( - final FacetFilterInput filter, @Nullable AspectRetriever aspectRetriever) { - return criterionFromFilter(filter, false, aspectRetriever); - } - // Translates a FacetFilterInput (graphql input class) into Criterion (our internal model) - public static Criterion criterionFromFilter( - final FacetFilterInput filter, - final Boolean skipKeywordSuffix, - @Nullable AspectRetriever aspectRetriever) { + public static Criterion criterionFromFilter(final FacetFilterInput filter) { Criterion result = new Criterion(); - - if (skipKeywordSuffix) { - result.setField(filter.getField()); - } else { - result.setField(getFilterField(filter.getField(), skipKeywordSuffix, aspectRetriever)); - } + result.setField(filter.getField()); // `value` is deprecated in place of `values`- this is to support old query patterns. If values // is provided, @@ -212,37 +188,6 @@ public static Criterion criterionFromFilter( return result; } - private static String getFilterField( - final String originalField, - final boolean skipKeywordSuffix, - @Nullable AspectRetriever aspectRetriever) { - if (KEYWORD_EXCLUDED_FILTERS.contains(originalField)) { - return originalField; - } - return ESUtils.toKeywordField(originalField, skipKeywordSuffix, aspectRetriever); - } - - public static Filter buildFilterWithUrns(@Nonnull Set urns, @Nullable Filter inputFilters) { - Criterion urnMatchCriterion = - new Criterion() - .setField("urn") - .setValue("") - .setValues( - new StringArray(urns.stream().map(Object::toString).collect(Collectors.toList()))); - if (inputFilters == null) { - return QueryUtils.newFilter(urnMatchCriterion); - } - - // Add urn match criterion to each or clause - if (inputFilters.getOr() != null && !inputFilters.getOr().isEmpty()) { - for (ConjunctiveCriterion conjunctiveCriterion : inputFilters.getOr()) { - conjunctiveCriterion.getAnd().add(urnMatchCriterion); - } - return inputFilters; - } - return QueryUtils.newFilter(urnMatchCriterion); - } - public static Filter viewFilter( OperationContext opContext, ViewService viewService, String viewUrn) { if (viewUrn == null) { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/assertion/AssertionRunEventResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/assertion/AssertionRunEventResolver.java index 0e9d2cea611416..f7e86f1f2f3452 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/assertion/AssertionRunEventResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/assertion/AssertionRunEventResolver.java @@ -12,6 +12,7 @@ import com.linkedin.datahub.graphql.generated.AssertionRunStatus; import com.linkedin.datahub.graphql.generated.FacetFilterInput; import com.linkedin.datahub.graphql.generated.FilterInput; +import com.linkedin.datahub.graphql.resolvers.ResolverUtils; import com.linkedin.datahub.graphql.types.dataset.mappers.AssertionRunEventMapper; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.Constants; @@ -147,7 +148,7 @@ public static Filter buildFilter( .setAnd( new CriterionArray( facetFilters.stream() - .map(filter -> criterionFromFilter(filter, true, aspectRetriever)) + .map(ResolverUtils::criterionFromFilter) .collect(Collectors.toList()))))); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/assertion/AssertionUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/assertion/AssertionUtils.java index 757ff38de60065..a632ab5487000e 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/assertion/AssertionUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/assertion/AssertionUtils.java @@ -18,10 +18,6 @@ public static boolean isAuthorizedToEditAssertionFromAssertee( new ConjunctivePrivilegeGroup( ImmutableList.of(PoliciesConfig.EDIT_ENTITY_ASSERTIONS_PRIVILEGE.getType())))); return AuthorizationUtils.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - asserteeUrn.getEntityType(), - asserteeUrn.toString(), - orPrivilegeGroups); + context, asserteeUrn.getEntityType(), asserteeUrn.toString(), orPrivilegeGroups); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/assertion/DeleteAssertionResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/assertion/DeleteAssertionResolver.java index 1cf233221d4d33..d36611da0dc4db 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/assertion/DeleteAssertionResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/assertion/DeleteAssertionResolver.java @@ -113,11 +113,7 @@ private boolean isAuthorizedToDeleteAssertionFromAssertee( new ConjunctivePrivilegeGroup( ImmutableList.of(PoliciesConfig.EDIT_ENTITY_ASSERTIONS_PRIVILEGE.getType())))); return AuthorizationUtils.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - asserteeUrn.getEntityType(), - asserteeUrn.toString(), - orPrivilegeGroups); + context, asserteeUrn.getEntityType(), asserteeUrn.toString(), orPrivilegeGroups); } private Urn getAsserteeUrnFromInfo(final AssertionInfo info) { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/assertion/UpsertCustomAssertionResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/assertion/UpsertCustomAssertionResolver.java index 026f486e32c116..29a9fff5e013a0 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/assertion/UpsertCustomAssertionResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/assertion/UpsertCustomAssertionResolver.java @@ -101,7 +101,7 @@ private CustomAssertionInfo createCustomAssertionInfo( if (input.getFieldPath() != null) { customAssertionInfo.setField( - SchemaFieldUtils.generateSchemaFieldUrn(entityUrn.toString(), input.getFieldPath())); + SchemaFieldUtils.generateSchemaFieldUrn(entityUrn, input.getFieldPath())); } return customAssertionInfo; } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/auth/ListAccessTokensResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/auth/ListAccessTokensResolver.java index e0ecebbbc7bc2e..29d44b526692f5 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/auth/ListAccessTokensResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/auth/ListAccessTokensResolver.java @@ -71,10 +71,7 @@ public CompletableFuture get(DataFetchingEnvironment envi .withSearchFlags(flags -> flags.setFulltext(true)), Constants.ACCESS_TOKEN_ENTITY_NAME, "", - buildFilter( - filters, - Collections.emptyList(), - context.getOperationContext().getAspectRetriever()), + buildFilter(filters, Collections.emptyList()), sortCriteria, start, count); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/businessattribute/BusinessAttributeAuthorizationUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/businessattribute/BusinessAttributeAuthorizationUtils.java index 041f5e9ade77f0..364a5c982b0e03 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/businessattribute/BusinessAttributeAuthorizationUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/businessattribute/BusinessAttributeAuthorizationUtils.java @@ -20,8 +20,7 @@ public static boolean canCreateBusinessAttribute(@Nonnull QueryContext context) new ConjunctivePrivilegeGroup( ImmutableList.of( PoliciesConfig.MANAGE_BUSINESS_ATTRIBUTE_PRIVILEGE.getType())))); - return AuthUtil.isAuthorized( - context.getAuthorizer(), context.getActorUrn(), orPrivilegeGroups, null); + return AuthUtil.isAuthorized(context.getOperationContext(), orPrivilegeGroups, null); } public static boolean canManageBusinessAttribute(@Nonnull QueryContext context) { @@ -31,7 +30,6 @@ public static boolean canManageBusinessAttribute(@Nonnull QueryContext context) new ConjunctivePrivilegeGroup( ImmutableList.of( PoliciesConfig.MANAGE_BUSINESS_ATTRIBUTE_PRIVILEGE.getType())))); - return AuthUtil.isAuthorized( - context.getAuthorizer(), context.getActorUrn(), orPrivilegeGroups, null); + return AuthUtil.isAuthorized(context.getOperationContext(), orPrivilegeGroups, null); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/chart/BrowseV2Resolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/chart/BrowseV2Resolver.java index b54ca398aef980..18ee5f595ce582 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/chart/BrowseV2Resolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/chart/BrowseV2Resolver.java @@ -74,9 +74,7 @@ public CompletableFuture get(DataFetchingEnvironment environmen ? BROWSE_PATH_V2_DELIMITER + String.join(BROWSE_PATH_V2_DELIMITER, input.getPath()) : ""; - final Filter inputFilter = - ResolverUtils.buildFilter( - null, input.getOrFilters(), context.getOperationContext().getAspectRetriever()); + final Filter inputFilter = ResolverUtils.buildFilter(null, input.getOrFilters()); BrowseResultV2 browseResults = _entityClient.browseV2( diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/connection/ConnectionUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/connection/ConnectionUtils.java index bcdd6460ae75ed..30c2fb672a0c30 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/connection/ConnectionUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/connection/ConnectionUtils.java @@ -14,9 +14,7 @@ public class ConnectionUtils { */ public static boolean canManageConnections(@Nonnull QueryContext context) { return AuthUtil.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - PoliciesConfig.MANAGE_CONNECTIONS_PRIVILEGE); + context.getOperationContext(), PoliciesConfig.MANAGE_CONNECTIONS_PRIVILEGE); } private ConnectionUtils() {} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/datacontract/DataContractUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/datacontract/DataContractUtils.java index 3dd7cd9df63838..a04024a29dc97b 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/datacontract/DataContractUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/datacontract/DataContractUtils.java @@ -21,11 +21,7 @@ public static boolean canEditDataContract(@Nonnull QueryContext context, Urn ent PoliciesConfig.EDIT_ENTITY_DATA_CONTRACT_PRIVILEGE.getType())))); return AuthorizationUtils.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - entityUrn.getEntityType(), - entityUrn.toString(), - orPrivilegeGroups); + context, entityUrn.getEntityType(), entityUrn.toString(), orPrivilegeGroups); } private DataContractUtils() {} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataproduct/DataProductAuthorizationUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataproduct/DataProductAuthorizationUtils.java index f6fe11a587a39b..a8357fc0a1a3ff 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataproduct/DataProductAuthorizationUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataproduct/DataProductAuthorizationUtils.java @@ -30,11 +30,7 @@ public static boolean isAuthorizedToUpdateDataProductsForEntity( PoliciesConfig.EDIT_ENTITY_DATA_PRODUCTS_PRIVILEGE.getType())))); return AuthorizationUtils.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - entityUrn.getEntityType(), - entityUrn.toString(), - orPrivilegeGroups); + context, entityUrn.getEntityType(), entityUrn.toString(), orPrivilegeGroups); } public static boolean isAuthorizedToManageDataProducts( @@ -47,11 +43,7 @@ public static boolean isAuthorizedToManageDataProducts( ImmutableList.of(PoliciesConfig.MANAGE_DATA_PRODUCTS_PRIVILEGE.getType())))); return AuthorizationUtils.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - domainUrn.getEntityType(), - domainUrn.toString(), - orPrivilegeGroups); + context, domainUrn.getEntityType(), domainUrn.toString(), orPrivilegeGroups); } public static boolean isAuthorizedToEditDataProduct( @@ -60,10 +52,6 @@ public static boolean isAuthorizedToEditDataProduct( new DisjunctivePrivilegeGroup(ImmutableList.of(ALL_PRIVILEGES_GROUP)); return AuthorizationUtils.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - dataProductUrn.getEntityType(), - dataProductUrn.toString(), - orPrivilegeGroups); + context, dataProductUrn.getEntityType(), dataProductUrn.toString(), orPrivilegeGroups); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataproduct/ListDataProductAssetsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataproduct/ListDataProductAssetsResolver.java index 320d89cdec164a..e59f7b3116acdb 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataproduct/ListDataProductAssetsResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataproduct/ListDataProductAssetsResolver.java @@ -1,7 +1,7 @@ package com.linkedin.datahub.graphql.resolvers.dataproduct; import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument; -import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.buildFilterWithUrns; +import static com.linkedin.metadata.search.utils.QueryUtils.buildFilterWithUrns; import com.google.common.collect.ImmutableList; import com.linkedin.common.urn.Urn; @@ -11,6 +11,8 @@ import com.linkedin.datahub.graphql.concurrency.GraphQLConcurrencyUtils; import com.linkedin.datahub.graphql.generated.DataProduct; import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.datahub.graphql.generated.ExtraProperty; +import com.linkedin.datahub.graphql.generated.FacetFilterInput; import com.linkedin.datahub.graphql.generated.SearchAcrossEntitiesInput; import com.linkedin.datahub.graphql.generated.SearchResults; import com.linkedin.datahub.graphql.resolvers.ResolverUtils; @@ -30,8 +32,12 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Optional; +import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -44,6 +50,7 @@ public class ListDataProductAssetsResolver implements DataFetcher> { + private static final String OUTPUT_PORTS_FILTER_FIELD = "isOutputPort"; private static final int DEFAULT_START = 0; private static final int DEFAULT_COUNT = 10; @@ -63,6 +70,7 @@ public CompletableFuture get(DataFetchingEnvironment environment) // 1. Get urns of assets belonging to Data Product using an aspect query List assetUrns = new ArrayList<>(); + Set outputPorts = Collections.EMPTY_SET; try { final EntityResponse entityResponse = _entityClient.getV2( @@ -86,6 +94,11 @@ public CompletableFuture get(DataFetchingEnvironment environment) dataProductProperties.getAssets().stream() .map(DataProductAssociation::getDestinationUrn) .collect(Collectors.toList())); + outputPorts = + dataProductProperties.getAssets().stream() + .filter(DataProductAssociation::isOutputPort) + .map(dpa -> dpa.getDestinationUrn().toString()) + .collect(Collectors.toSet()); } } } catch (Exception e) { @@ -117,6 +130,7 @@ public CompletableFuture get(DataFetchingEnvironment environment) final int start = input.getStart() != null ? input.getStart() : DEFAULT_START; final int count = input.getCount() != null ? input.getCount() : DEFAULT_COUNT; + Set finalOutputPorts = outputPorts; return GraphQLConcurrencyUtils.supplyAsync( () -> { // if no assets in data product properties, exit early before search and return empty @@ -130,13 +144,17 @@ public CompletableFuture get(DataFetchingEnvironment environment) return results; } + List filters = input.getFilters(); + final List urnsToFilterOn = getUrnsToFilterOn(assetUrns, finalOutputPorts, filters); + // need to remove output ports filter so we don't send to elastic + if (filters != null) { + filters.removeIf(f -> f.getField().equals(OUTPUT_PORTS_FILTER_FIELD)); + } // add urns from the aspect to our filters - final Filter baseFilter = - ResolverUtils.buildFilter( - input.getFilters(), - input.getOrFilters(), - context.getOperationContext().getAspectRetriever()); - final Filter finalFilter = buildFilterWithUrns(new HashSet<>(assetUrns), baseFilter); + final Filter baseFilter = ResolverUtils.buildFilter(filters, input.getOrFilters()); + final Filter finalFilter = + buildFilterWithUrns( + context.getDataHubAppConfig(), new HashSet<>(urnsToFilterOn), baseFilter); final SearchFlags searchFlags; com.linkedin.datahub.graphql.generated.SearchFlags inputFlags = input.getSearchFlags(); @@ -155,18 +173,34 @@ public CompletableFuture get(DataFetchingEnvironment environment) start, count); - return UrnSearchResultsMapper.map( - context, - _entityClient.searchAcrossEntities( - context - .getOperationContext() - .withSearchFlags(flags -> searchFlags != null ? searchFlags : flags), - finalEntityNames, - sanitizedQuery, - finalFilter, - start, - count, - null)); + SearchResults results = + UrnSearchResultsMapper.map( + context, + _entityClient.searchAcrossEntities( + context + .getOperationContext() + .withSearchFlags(flags -> searchFlags != null ? searchFlags : flags), + finalEntityNames, + sanitizedQuery, + finalFilter, + start, + count, + null, + null)); + results + .getSearchResults() + .forEach( + searchResult -> { + if (finalOutputPorts.contains(searchResult.getEntity().getUrn())) { + if (searchResult.getExtraProperties() == null) { + searchResult.setExtraProperties(new ArrayList<>()); + } + searchResult + .getExtraProperties() + .add(new ExtraProperty("isOutputPort", "true")); + } + }); + return results; } catch (Exception e) { log.error( "Failed to execute search for data product assets: entity types {}, query {}, filters: {}, start: {}, count: {}", @@ -186,4 +220,37 @@ public CompletableFuture get(DataFetchingEnvironment environment) this.getClass().getSimpleName(), "get"); } + + /** + * Check to see if our filters list has a hardcoded filter for output ports. If so, let this + * filter determine which urns we filter search results on. Otherwise, if no output port filter is + * found, return all asset urns as per usual. + */ + @Nonnull + private List getUrnsToFilterOn( + @Nonnull final List assetUrns, + @Nonnull final Set outputPortUrns, + @Nullable final List filters) { + Optional isOutputPort = + filters != null + ? filters.stream() + .filter(f -> f.getField().equals(OUTPUT_PORTS_FILTER_FIELD)) + .findFirst() + : Optional.empty(); + + // optionally get entities that explicitly are or are not output ports + List urnsToFilterOn = assetUrns; + if (isOutputPort.isPresent()) { + if (isOutputPort.get().getValue().equals("true")) { + urnsToFilterOn = outputPortUrns.stream().map(UrnUtils::getUrn).collect(Collectors.toList()); + } else { + urnsToFilterOn = + assetUrns.stream() + .filter(u -> !outputPortUrns.contains(u.toString())) + .collect(Collectors.toList()); + } + } + + return urnsToFilterOn; + } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetStatsSummaryResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetStatsSummaryResolver.java index 7d3603ec050e94..bab7ecaf302f54 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetStatsSummaryResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetStatsSummaryResolver.java @@ -49,7 +49,7 @@ public CompletableFuture get(DataFetchingEnvironment enviro log.debug( "User {} is not authorized to view profile information for dataset {}", context.getActorUrn(), - resourceUrn.toString()); + resourceUrn); return null; } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/deprecation/UpdateDeprecationResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/deprecation/UpdateDeprecationResolver.java index c568ff6db3a27d..a2230cf6b6e886 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/deprecation/UpdateDeprecationResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/deprecation/UpdateDeprecationResolver.java @@ -100,11 +100,7 @@ private boolean isAuthorizedToUpdateDeprecationForEntity( ImmutableList.of(PoliciesConfig.EDIT_ENTITY_DEPRECATION_PRIVILEGE.getType())))); return AuthorizationUtils.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - entityUrn.getEntityType(), - entityUrn.toString(), - orPrivilegeGroups); + context, entityUrn.getEntityType(), entityUrn.toString(), orPrivilegeGroups); } public static Boolean validateUpdateDeprecationInput( diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/DomainEntitiesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/DomainEntitiesResolver.java index 6a880503802cb4..c6265380fb2fd0 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/DomainEntitiesResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/DomainEntitiesResolver.java @@ -79,9 +79,7 @@ public CompletableFuture get(final DataFetchingEnvironment enviro .getFilters() .forEach( filter -> { - criteria.add( - criterionFromFilter( - filter, true, context.getOperationContext().getAspectRetriever())); + criteria.add(criterionFromFilter(filter)); }); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/entity/EntityPrivilegesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/entity/EntityPrivilegesResolver.java index b25f5598b44bc0..67ab9bb2878141 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/entity/EntityPrivilegesResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/entity/EntityPrivilegesResolver.java @@ -106,8 +106,7 @@ private EntityPrivileges getGlossaryNodePrivileges(Urn nodeUrn, QueryContext con } private boolean canEditEntityLineage(Urn urn, QueryContext context) { - return AuthUtil.isAuthorizedUrns( - context.getAuthorizer(), context.getActorUrn(), LINEAGE, UPDATE, List.of(urn)); + return AuthUtil.isAuthorizedUrns(context.getOperationContext(), LINEAGE, UPDATE, List.of(urn)); } private EntityPrivileges getDatasetPrivileges(Urn urn, QueryContext context) { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/form/CreateDynamicFormAssignmentResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/form/CreateDynamicFormAssignmentResolver.java index 3cf4d9175d45bf..b9d74f8af660e8 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/form/CreateDynamicFormAssignmentResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/form/CreateDynamicFormAssignmentResolver.java @@ -33,9 +33,7 @@ public CompletableFuture get(final DataFetchingEnvironment environment) final CreateDynamicFormAssignmentInput input = bindArgument(environment.getArgument("input"), CreateDynamicFormAssignmentInput.class); final Urn formUrn = UrnUtils.getUrn(input.getFormUrn()); - final DynamicFormAssignment formAssignment = - FormUtils.mapDynamicFormAssignment( - input, context.getOperationContext().getAspectRetriever()); + final DynamicFormAssignment formAssignment = FormUtils.mapDynamicFormAssignment(input); return GraphQLConcurrencyUtils.supplyAsync( () -> { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/incident/RaiseIncidentResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/incident/RaiseIncidentResolver.java index 454ba693da95a7..68aef26bf4aa17 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/incident/RaiseIncidentResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/incident/RaiseIncidentResolver.java @@ -123,10 +123,6 @@ private boolean isAuthorizedToCreateIncidentForResource( ImmutableList.of(PoliciesConfig.EDIT_ENTITY_INCIDENTS_PRIVILEGE.getType())))); return AuthorizationUtils.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - resourceUrn.getEntityType(), - resourceUrn.toString(), - orPrivilegeGroups); + context, resourceUrn.getEntityType(), resourceUrn.toString(), orPrivilegeGroups); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/incident/UpdateIncidentStatusResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/incident/UpdateIncidentStatusResolver.java index d51ceab31e60ec..dee92247ba311a 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/incident/UpdateIncidentStatusResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/incident/UpdateIncidentStatusResolver.java @@ -103,10 +103,6 @@ private boolean isAuthorizedToUpdateIncident(final Urn resourceUrn, final QueryC new ConjunctivePrivilegeGroup( ImmutableList.of(PoliciesConfig.EDIT_ENTITY_INCIDENTS_PRIVILEGE.getType())))); return AuthorizationUtils.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - resourceUrn.getEntityType(), - resourceUrn.toString(), - orPrivilegeGroups); + context, resourceUrn.getEntityType(), resourceUrn.toString(), orPrivilegeGroups); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/IngestionAuthUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/IngestionAuthUtils.java index 24d0e946145054..be8d4fa7b8c68d 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/IngestionAuthUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/IngestionAuthUtils.java @@ -14,15 +14,12 @@ public class IngestionAuthUtils { public static boolean canManageIngestion(@Nonnull QueryContext context) { return AuthUtil.isAuthorizedEntityType( - context.getActorUrn(), - context.getAuthorizer(), - MANAGE, - List.of(INGESTION_SOURCE_ENTITY_NAME)); + context.getOperationContext(), MANAGE, List.of(INGESTION_SOURCE_ENTITY_NAME)); } public static boolean canManageSecrets(@Nonnull QueryContext context) { return isAuthorizedEntityType( - context.getActorUrn(), context.getAuthorizer(), MANAGE, List.of(SECRETS_ENTITY_NAME)); + context.getOperationContext(), MANAGE, List.of(SECRETS_ENTITY_NAME)); } private IngestionAuthUtils() {} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/source/ListIngestionSourcesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/source/ListIngestionSourcesResolver.java index 1a2806224e4a92..8ead47aa65ceb0 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/source/ListIngestionSourcesResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/source/ListIngestionSourcesResolver.java @@ -68,10 +68,7 @@ public CompletableFuture get( .withSearchFlags(flags -> flags.setFulltext(true)), Constants.INGESTION_SOURCE_ENTITY_NAME, query, - buildFilter( - filters, - Collections.emptyList(), - context.getOperationContext().getAspectRetriever()), + buildFilter(filters, Collections.emptyList()), null, start, count); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/jobs/DataJobRunsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/jobs/DataJobRunsResolver.java index 09039e530631d0..d7c76c0235dcc0 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/jobs/DataJobRunsResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/jobs/DataJobRunsResolver.java @@ -4,9 +4,7 @@ import com.linkedin.common.urn.Urn; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.concurrency.GraphQLConcurrencyUtils; -import com.linkedin.datahub.graphql.generated.DataProcessInstance; -import com.linkedin.datahub.graphql.generated.DataProcessInstanceResult; -import com.linkedin.datahub.graphql.generated.Entity; +import com.linkedin.datahub.graphql.generated.*; import com.linkedin.datahub.graphql.types.dataprocessinst.mappers.DataProcessInstanceMapper; import com.linkedin.entity.EntityResponse; import com.linkedin.entity.client.EntityClient; @@ -33,6 +31,8 @@ import java.util.Objects; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** GraphQL Resolver used for fetching a list of Task Runs associated with a Data Job */ public class DataJobRunsResolver @@ -40,6 +40,8 @@ public class DataJobRunsResolver private static final String PARENT_TEMPLATE_URN_SEARCH_INDEX_FIELD_NAME = "parentTemplate"; private static final String CREATED_TIME_SEARCH_INDEX_FIELD_NAME = "created"; + private static final String HAS_RUN_EVENTS_FIELD_NAME = "hasRunEvents"; + private static final Logger log = LoggerFactory.getLogger(DataJobRunsResolver.class); private final EntityClient _entityClient; @@ -117,7 +119,12 @@ private Filter buildTaskRunsEntityFilter(final String entityUrn) { new Criterion() .setField(PARENT_TEMPLATE_URN_SEARCH_INDEX_FIELD_NAME) .setCondition(Condition.EQUAL) - .setValue(entityUrn))); + .setValue(entityUrn), + new Criterion() + .setField(HAS_RUN_EVENTS_FIELD_NAME) + .setCondition(Condition.EQUAL) + .setValue(Boolean.TRUE.toString()))); + final Filter filter = new Filter(); filter.setOr( new ConjunctiveCriterionArray(ImmutableList.of(new ConjunctiveCriterion().setAnd(array)))); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/lineage/UpdateLineageResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/lineage/UpdateLineageResolver.java index d462fb0820aa03..928e33d44c84e7 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/lineage/UpdateLineageResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/lineage/UpdateLineageResolver.java @@ -211,11 +211,7 @@ private boolean isAuthorized( @Nonnull final Urn urn, @Nonnull final DisjunctivePrivilegeGroup orPrivilegesGroup) { return AuthorizationUtils.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - urn.getEntityType(), - urn.toString(), - orPrivilegesGroup); + context, urn.getEntityType(), urn.toString(), orPrivilegesGroup); } private void checkLineageEdgePrivileges( diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/TimeSeriesAspectResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/TimeSeriesAspectResolver.java index 8fc26e3cec9d06..0f4f0b2645e425 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/TimeSeriesAspectResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/TimeSeriesAspectResolver.java @@ -9,9 +9,9 @@ import com.linkedin.datahub.graphql.generated.Entity; import com.linkedin.datahub.graphql.generated.FilterInput; import com.linkedin.datahub.graphql.generated.TimeSeriesAspect; +import com.linkedin.datahub.graphql.resolvers.ResolverUtils; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.Constants; -import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.aspect.EnvelopedAspect; import com.linkedin.metadata.authorization.PoliciesConfig; import com.linkedin.metadata.query.filter.ConjunctiveCriterion; @@ -78,8 +78,7 @@ private boolean isAuthorized(QueryContext context, String urn) { if (_entityName.equals(Constants.DATASET_ENTITY_NAME) && _aspectName.equals(Constants.DATASET_PROFILE_ASPECT_NAME)) { return AuthUtil.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), + context.getOperationContext(), PoliciesConfig.VIEW_DATASET_PROFILE_PRIVILEGE, new EntitySpec(_entityName, urn)); } @@ -121,7 +120,7 @@ public CompletableFuture> get(DataFetchingEnvironment env maybeStartTimeMillis, maybeEndTimeMillis, maybeLimit, - buildFilters(maybeFilters, context.getOperationContext().getAspectRetriever()), + buildFilters(maybeFilters), maybeSort); // Step 2: Bind profiles into GraphQL strong types. @@ -136,8 +135,7 @@ public CompletableFuture> get(DataFetchingEnvironment env "get"); } - private Filter buildFilters( - @Nullable FilterInput maybeFilters, @Nullable AspectRetriever aspectRetriever) { + private Filter buildFilters(@Nullable FilterInput maybeFilters) { if (maybeFilters == null) { return null; } @@ -148,7 +146,7 @@ private Filter buildFilters( .setAnd( new CriterionArray( maybeFilters.getAnd().stream() - .map(filter -> criterionFromFilter(filter, true, aspectRetriever)) + .map(ResolverUtils::criterionFromFilter) .collect(Collectors.toList()))))); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/DescriptionUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/DescriptionUtils.java index 6e2fc77e703af3..917f1b1c1d574d 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/DescriptionUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/DescriptionUtils.java @@ -336,11 +336,7 @@ public static boolean isAuthorizedToUpdateFieldDescription( PoliciesConfig.EDIT_DATASET_COL_DESCRIPTION_PRIVILEGE.getType())))); return AuthorizationUtils.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - targetUrn.getEntityType(), - targetUrn.toString(), - orPrivilegeGroups); + context, targetUrn.getEntityType(), targetUrn.toString(), orPrivilegeGroups); } public static boolean isAuthorizedToUpdateDomainDescription( @@ -353,11 +349,7 @@ public static boolean isAuthorizedToUpdateDomainDescription( ImmutableList.of(PoliciesConfig.EDIT_ENTITY_DOCS_PRIVILEGE.getType())))); return AuthorizationUtils.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - targetUrn.getEntityType(), - targetUrn.toString(), - orPrivilegeGroups); + context, targetUrn.getEntityType(), targetUrn.toString(), orPrivilegeGroups); } public static boolean isAuthorizedToUpdateContainerDescription( @@ -370,11 +362,7 @@ public static boolean isAuthorizedToUpdateContainerDescription( ImmutableList.of(PoliciesConfig.EDIT_ENTITY_DOCS_PRIVILEGE.getType())))); return AuthorizationUtils.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - targetUrn.getEntityType(), - targetUrn.toString(), - orPrivilegeGroups); + context, targetUrn.getEntityType(), targetUrn.toString(), orPrivilegeGroups); } public static boolean isAuthorizedToUpdateDescription( @@ -387,11 +375,7 @@ public static boolean isAuthorizedToUpdateDescription( ImmutableList.of(PoliciesConfig.EDIT_ENTITY_DOCS_PRIVILEGE.getType())))); return AuthorizationUtils.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - targetUrn.getEntityType(), - targetUrn.toString(), - orPrivilegeGroups); + context, targetUrn.getEntityType(), targetUrn.toString(), orPrivilegeGroups); } public static void updateMlModelDescription( diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DeleteUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DeleteUtils.java index 73e7f9ec1cca7c..1d3a9c229e63e2 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DeleteUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DeleteUtils.java @@ -25,7 +25,7 @@ private DeleteUtils() {} public static boolean isAuthorizedToDeleteEntity(@Nonnull QueryContext context, Urn entityUrn) { return AuthUtil.isAuthorizedEntityUrns( - context.getAuthorizer(), context.getActorUrn(), DELETE, List.of(entityUrn)); + context.getOperationContext(), DELETE, List.of(entityUrn)); } public static void updateStatusForResources( diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DeprecationUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DeprecationUtils.java index 3912ffa6226bff..541224b02c1b52 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DeprecationUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DeprecationUtils.java @@ -42,11 +42,7 @@ public static boolean isAuthorizedToUpdateDeprecationForEntity( ImmutableList.of(PoliciesConfig.EDIT_ENTITY_DEPRECATION_PRIVILEGE.getType())))); return AuthorizationUtils.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - entityUrn.getEntityType(), - entityUrn.toString(), - orPrivilegeGroups); + context, entityUrn.getEntityType(), entityUrn.toString(), orPrivilegeGroups); } public static void updateDeprecationForResources( diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DomainUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DomainUtils.java index 1114cf7344e8f4..1dcdd988f5e7c1 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DomainUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DomainUtils.java @@ -67,11 +67,7 @@ public static boolean isAuthorizedToUpdateDomainsForEntity( ImmutableList.of(PoliciesConfig.EDIT_ENTITY_DOMAINS_PRIVILEGE.getType())))); return AuthorizationUtils.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - entityUrn.getEntityType(), - entityUrn.toString(), - orPrivilegeGroups); + context, entityUrn.getEntityType(), entityUrn.toString(), orPrivilegeGroups); } public static void setDomainForResources( diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/EmbedUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/EmbedUtils.java index 15c93904fc3bdd..5ebb434b21c9f4 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/EmbedUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/EmbedUtils.java @@ -28,10 +28,6 @@ public static boolean isAuthorizedToUpdateEmbedForEntity( ImmutableList.of(PoliciesConfig.EDIT_ENTITY_EMBED_PRIVILEGE.getType())))); return AuthorizationUtils.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - entityUrn.getEntityType(), - entityUrn.toString(), - orPrivilegeGroups); + context, entityUrn.getEntityType(), entityUrn.toString(), orPrivilegeGroups); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/FormUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/FormUtils.java index d118c04d19393d..cac0cca2682e84 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/FormUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/FormUtils.java @@ -18,7 +18,6 @@ import com.linkedin.form.FormPromptType; import com.linkedin.form.FormType; import com.linkedin.form.StructuredPropertyParams; -import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.query.filter.Condition; import com.linkedin.metadata.query.filter.ConjunctiveCriterion; import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; @@ -31,7 +30,6 @@ import java.util.UUID; import java.util.stream.Collectors; import javax.annotation.Nonnull; -import javax.annotation.Nullable; public class FormUtils { @@ -61,16 +59,13 @@ public static PrimitivePropertyValueArray getStructuredPropertyValuesFromInput( /** Map a GraphQL CreateDynamicFormAssignmentInput to the GMS DynamicFormAssignment aspect */ @Nonnull public static DynamicFormAssignment mapDynamicFormAssignment( - @Nonnull final CreateDynamicFormAssignmentInput input, - @Nullable AspectRetriever aspectRetriever) { + @Nonnull final CreateDynamicFormAssignmentInput input) { Objects.requireNonNull(input, "input must not be null"); final DynamicFormAssignment result = new DynamicFormAssignment(); final Filter filter = new Filter() - .setOr( - ResolverUtils.buildConjunctiveCriterionArrayWithOr( - input.getOrFilters(), aspectRetriever)); + .setOr(ResolverUtils.buildConjunctiveCriterionArrayWithOr(input.getOrFilters())); result.setFilter(filter); return result; } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/GlossaryUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/GlossaryUtils.java index 16df9911f3bec3..0d8e505a948e5a 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/GlossaryUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/GlossaryUtils.java @@ -33,7 +33,7 @@ private GlossaryUtils() {} */ public static boolean canManageGlossaries(@Nonnull QueryContext context) { return AuthUtil.isAuthorized( - context.getAuthorizer(), context.getActorUrn(), PoliciesConfig.MANAGE_GLOSSARIES_PRIVILEGE); + context.getOperationContext(), PoliciesConfig.MANAGE_GLOSSARIES_PRIVILEGE); } /** @@ -79,11 +79,7 @@ public static boolean hasManagePrivilege( ImmutableList.of(new ConjunctivePrivilegeGroup(ImmutableList.of(privilege.getType())))); return AuthorizationUtils.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - parentNodeUrn.getEntityType(), - parentNodeUrn.toString(), - orPrivilegeGroups); + context, parentNodeUrn.getEntityType(), parentNodeUrn.toString(), orPrivilegeGroups); } /** diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/LabelUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/LabelUtils.java index 3eac819a9cc48d..cffd019307f34a 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/LabelUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/LabelUtils.java @@ -251,11 +251,7 @@ public static boolean isAuthorizedToUpdateTags( : PoliciesConfig.EDIT_ENTITY_TAGS_PRIVILEGE.getType())))); return AuthorizationUtils.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - targetUrn.getEntityType(), - targetUrn.toString(), - orPrivilegeGroups); + context, targetUrn.getEntityType(), targetUrn.toString(), orPrivilegeGroups); } public static boolean isAuthorizedToUpdateTerms( @@ -277,11 +273,7 @@ public static boolean isAuthorizedToUpdateTerms( : PoliciesConfig.EDIT_ENTITY_GLOSSARY_TERMS_PRIVILEGE.getType())))); return AuthorizationUtils.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - targetUrn.getEntityType(), - targetUrn.toString(), - orPrivilegeGroups); + context, targetUrn.getEntityType(), targetUrn.toString(), orPrivilegeGroups); } public static void validateResourceAndLabel( diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/LinkUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/LinkUtils.java index a2d4692db5b7b7..e6f9d09412119a 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/LinkUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/LinkUtils.java @@ -115,11 +115,7 @@ public static boolean isAuthorizedToUpdateLinks(@Nonnull QueryContext context, U ImmutableList.of(PoliciesConfig.EDIT_ENTITY_DOC_LINKS_PRIVILEGE.getType())))); return AuthorizationUtils.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - resourceUrn.getEntityType(), - resourceUrn.toString(), - orPrivilegeGroups); + context, resourceUrn.getEntityType(), resourceUrn.toString(), orPrivilegeGroups); } public static Boolean validateAddRemoveInput( diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/OwnerUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/OwnerUtils.java index ddb795189c0e3d..2f2b52f7ab5864 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/OwnerUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/OwnerUtils.java @@ -205,11 +205,7 @@ public static void validateAuthorizedToUpdateOwners( boolean authorized = AuthorizationUtils.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - resourceUrn.getEntityType(), - resourceUrn.toString(), - orPrivilegeGroups); + context, resourceUrn.getEntityType(), resourceUrn.toString(), orPrivilegeGroups); if (!authorized) { throw new AuthorizationException( "Unauthorized to update owners. Please contact your DataHub administrator."); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/operation/ReportOperationResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/operation/ReportOperationResolver.java index 6ef3222bc068f2..48f231fee5093a 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/operation/ReportOperationResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/operation/ReportOperationResolver.java @@ -137,10 +137,6 @@ private boolean isAuthorizedToReportOperationForResource( ImmutableList.of(PoliciesConfig.EDIT_ENTITY_OPERATIONS_PRIVILEGE.getType())))); return AuthorizationUtils.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - resourceUrn.getEntityType(), - resourceUrn.toString(), - orPrivilegeGroups); + context, resourceUrn.getEntityType(), resourceUrn.toString(), orPrivilegeGroups); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ownership/ListOwnershipTypesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ownership/ListOwnershipTypesResolver.java index da0d5dd07a94f0..05fc540e39de4c 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ownership/ListOwnershipTypesResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ownership/ListOwnershipTypesResolver.java @@ -63,10 +63,7 @@ public CompletableFuture get(DataFetchingEnvironment e context.getOperationContext().withSearchFlags(flags -> flags.setFulltext(true)), Constants.OWNERSHIP_TYPE_ENTITY_NAME, query, - buildFilter( - filters, - Collections.emptyList(), - context.getOperationContext().getAspectRetriever()), + buildFilter(filters, Collections.emptyList()), Collections.singletonList(DEFAULT_SORT_CRITERION), start, count); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/ListPoliciesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/ListPoliciesResolver.java index ce11451aa1913f..4120401e0150f9 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/ListPoliciesResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/ListPoliciesResolver.java @@ -59,11 +59,7 @@ public CompletableFuture get(final DataFetchingEnvironment e log.debug( "User {} listing policies with filters {}", context.getActorUrn(), filters.toString()); - final Filter filter = - ResolverUtils.buildFilter( - facetFilters, - Collections.emptyList(), - context.getOperationContext().getAspectRetriever()); + final Filter filter = ResolverUtils.buildFilter(facetFilters, Collections.emptyList()); return _policyFetcher .fetchPolicies(context.getOperationContext(), start, query, count, filter) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/PolicyAuthUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/PolicyAuthUtils.java index 7babe63745f727..775a4aaf6e090b 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/PolicyAuthUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/PolicyAuthUtils.java @@ -12,7 +12,7 @@ public class PolicyAuthUtils { static boolean canManagePolicies(@Nonnull QueryContext context) { return AuthUtil.isAuthorizedEntityType( - context.getActorUrn(), context.getAuthorizer(), MANAGE, List.of(POLICY_ENTITY_NAME)); + context.getOperationContext(), MANAGE, List.of(POLICY_ENTITY_NAME)); } private PolicyAuthUtils() {} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/post/CreatePostResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/post/CreatePostResolver.java index 8b4253501dedc7..312950e44ffbda 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/post/CreatePostResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/post/CreatePostResolver.java @@ -5,15 +5,13 @@ import com.datahub.authentication.Authentication; import com.datahub.authentication.post.PostService; import com.linkedin.common.Media; +import com.linkedin.common.urn.UrnUtils; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; import com.linkedin.datahub.graphql.concurrency.GraphQLConcurrencyUtils; import com.linkedin.datahub.graphql.exception.AuthorizationException; -import com.linkedin.datahub.graphql.generated.CreatePostInput; -import com.linkedin.datahub.graphql.generated.PostContentType; -import com.linkedin.datahub.graphql.generated.PostType; -import com.linkedin.datahub.graphql.generated.UpdateMediaInput; -import com.linkedin.datahub.graphql.generated.UpdatePostContentInput; +import com.linkedin.datahub.graphql.generated.*; +import com.linkedin.metadata.utils.SchemaFieldUtils; import com.linkedin.post.PostContent; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; @@ -46,6 +44,18 @@ public CompletableFuture get(final DataFetchingEnvironment environment) final String description = content.getDescription(); final UpdateMediaInput updateMediaInput = content.getMedia(); final Authentication authentication = context.getAuthentication(); + final String targetResource = input.getResourceUrn(); + final String targetSubresource = input.getSubResource(); + + String targetUrn; + if (targetSubresource != null) { + targetUrn = + SchemaFieldUtils.generateSchemaFieldUrn( + UrnUtils.getUrn(targetResource), targetSubresource) + .toString(); + } else { + targetUrn = targetResource; + } Media media = updateMediaInput == null @@ -59,7 +69,7 @@ public CompletableFuture get(final DataFetchingEnvironment environment) () -> { try { return _postService.createPost( - context.getOperationContext(), type.toString(), postContent); + context.getOperationContext(), type.toString(), postContent, targetUrn); } catch (Exception e) { throw new RuntimeException("Failed to create a new post", e); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/query/ListQueriesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/query/ListQueriesResolver.java index aa411f019a4c08..3c84884bedbae8 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/query/ListQueriesResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/query/ListQueriesResolver.java @@ -15,7 +15,6 @@ import com.linkedin.datahub.graphql.generated.ListQueriesResult; import com.linkedin.datahub.graphql.generated.QueryEntity; import com.linkedin.entity.client.EntityClient; -import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.query.filter.SortCriterion; import com.linkedin.metadata.query.filter.SortOrder; @@ -74,7 +73,7 @@ public CompletableFuture get(final DataFetchingEnvironment en flags -> flags.setFulltext(true).setSkipHighlighting(true)), QUERY_ENTITY_NAME, query, - buildFilters(input, context.getOperationContext().getAspectRetriever()), + buildFilters(input), sortCriteria, start, count); @@ -111,8 +110,7 @@ private List mapUnresolvedQueries(final List queryUrns) { } @Nullable - private Filter buildFilters( - @Nonnull final ListQueriesInput input, @Nullable AspectRetriever aspectRetriever) { + private Filter buildFilters(@Nonnull final ListQueriesInput input) { final AndFilterInput criteria = new AndFilterInput(); List andConditions = new ArrayList<>(); @@ -139,6 +137,6 @@ private Filter buildFilters( } criteria.setAnd(andConditions); - return buildFilter(Collections.emptyList(), ImmutableList.of(criteria), aspectRetriever); + return buildFilter(Collections.emptyList(), ImmutableList.of(criteria)); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/recommendation/ListRecommendationsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/recommendation/ListRecommendationsResolver.java index 01818778643905..28334b2c0af9a4 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/recommendation/ListRecommendationsResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/recommendation/ListRecommendationsResolver.java @@ -17,6 +17,7 @@ import com.linkedin.datahub.graphql.generated.RecommendationRenderType; import com.linkedin.datahub.graphql.generated.RecommendationRequestContext; import com.linkedin.datahub.graphql.generated.SearchParams; +import com.linkedin.datahub.graphql.resolvers.ResolverUtils; import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper; import com.linkedin.datahub.graphql.types.entitytype.EntityTypeMapper; import com.linkedin.metadata.query.filter.CriterionArray; @@ -105,9 +106,7 @@ private com.linkedin.metadata.recommendation.RecommendationRequestContext mapReq searchRequestContext.setFilters( new CriterionArray( requestContext.getSearchRequestContext().getFilters().stream() - .map( - facetField -> - criterionFromFilter(facetField, opContext.getAspectRetriever())) + .map(ResolverUtils::criterionFromFilter) .collect(Collectors.toList()))); } mappedRequestContext.setSearchRequestContext(searchRequestContext); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolver.java index 19bccaf2650866..29b71d95ad9749 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolver.java @@ -64,9 +64,7 @@ public CompletableFuture get(DataFetchingEnvironment environme UrnUtils.getUrn(input.getViewUrn())) : null; - final Filter inputFilter = - ResolverUtils.buildFilter( - null, input.getOrFilters(), context.getOperationContext().getAspectRetriever()); + final Filter inputFilter = ResolverUtils.buildFilter(null, input.getOrFilters()); final SearchFlags searchFlags = mapInputFlags(context, input.getSearchFlags()); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AutoCompleteResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AutoCompleteResolver.java index 79792940ef27f7..5e4f56dbfff980 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AutoCompleteResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AutoCompleteResolver.java @@ -51,11 +51,7 @@ public CompletableFuture get(DataFetchingEnvironment enviro throw new ValidationException("'query' parameter can not be null or empty"); } - final Filter filter = - ResolverUtils.buildFilter( - input.getFilters(), - input.getOrFilters(), - context.getOperationContext().getRetrieverContext().orElseThrow().getAspectRetriever()); + final Filter filter = ResolverUtils.buildFilter(input.getFilters(), input.getOrFilters()); final int limit = input.getLimit() != null ? input.getLimit() : DEFAULT_LIMIT; return GraphQLConcurrencyUtils.supplyAsync( () -> { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AutocompleteUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AutocompleteUtils.java index 5b5888b89b241b..c28b18acf195b8 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AutocompleteUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AutocompleteUtils.java @@ -43,10 +43,7 @@ public static CompletableFuture batchGetAutocomplet GraphQLConcurrencyUtils.supplyAsync( () -> { final Filter filter = - ResolverUtils.buildFilter( - input.getFilters(), - input.getOrFilters(), - context.getOperationContext().getAspectRetriever()); + ResolverUtils.buildFilter(input.getFilters(), input.getOrFilters()); final Filter finalFilter = view != null ? SearchUtils.combineFilters( diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/ScrollAcrossEntitiesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/ScrollAcrossEntitiesResolver.java index 8b8b93353bc6e8..77eef1b9a25c69 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/ScrollAcrossEntitiesResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/ScrollAcrossEntitiesResolver.java @@ -72,9 +72,7 @@ public CompletableFuture get(DataFetchingEnvironment environment) UrnUtils.getUrn(input.getViewUrn())) : null; - final Filter baseFilter = - ResolverUtils.buildFilter( - null, input.getOrFilters(), context.getOperationContext().getAspectRetriever()); + final Filter baseFilter = ResolverUtils.buildFilter(null, input.getOrFilters()); final SearchFlags searchFlags; com.linkedin.datahub.graphql.generated.SearchFlags inputFlags = input.getSearchFlags(); if (inputFlags != null) { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/ScrollAcrossLineageResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/ScrollAcrossLineageResolver.java index 1b719b6f786205..2c058eb60a7ee3 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/ScrollAcrossLineageResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/ScrollAcrossLineageResolver.java @@ -130,10 +130,7 @@ public CompletableFuture get(DataFetchingEnvironment entityNames, sanitizedQuery, maxHops, - ResolverUtils.buildFilter( - facetFilters, - input.getOrFilters(), - context.getOperationContext().getAspectRetriever()), + ResolverUtils.buildFilter(facetFilters, input.getOrFilters()), null, scrollId, keepAlive, diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolver.java index 0dbed92b7d58e7..d103704146d399 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolver.java @@ -61,10 +61,7 @@ public CompletableFuture get(DataFetchingEnvironment environment) : null; final Filter baseFilter = - ResolverUtils.buildFilter( - input.getFilters(), - input.getOrFilters(), - context.getOperationContext().getAspectRetriever()); + ResolverUtils.buildFilter(input.getFilters(), input.getOrFilters()); SearchFlags searchFlags = mapInputFlags(context, input.getSearchFlags()); List sortCriteria; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossLineageResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossLineageResolver.java index dc3a1fc17e4ec9..c90be924ac69f5 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossLineageResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossLineageResolver.java @@ -139,10 +139,7 @@ public CompletableFuture get(DataFetchingEnvironment count); final Filter filter = - ResolverUtils.buildFilter( - input.getFilters(), - input.getOrFilters(), - context.getOperationContext().getAspectRetriever()); + ResolverUtils.buildFilter(input.getFilters(), input.getOrFilters()); final SearchFlags searchFlags; com.linkedin.datahub.graphql.generated.SearchFlags inputFlags = input.getSearchFlags(); if (inputFlags != null) { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchResolver.java index 7a48e305dbfe49..45751fc6eb8cb2 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchResolver.java @@ -86,10 +86,7 @@ public CompletableFuture get(DataFetchingEnvironment environment) context.getOperationContext().withSearchFlags(flags -> searchFlags), entityName, sanitizedQuery, - ResolverUtils.buildFilter( - input.getFilters(), - input.getOrFilters(), - context.getOperationContext().getAspectRetriever()), + ResolverUtils.buildFilter(input.getFilters(), input.getOrFilters()), Collections.emptyList(), start, count)); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/CreateStructuredPropertyResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/CreateStructuredPropertyResolver.java index 3be7ea505abbf3..328f63b893d06f 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/CreateStructuredPropertyResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/CreateStructuredPropertyResolver.java @@ -83,6 +83,8 @@ public CompletableFuture get(final DataFetchingEnviron builder.setCardinality( PropertyCardinality.valueOf(input.getCardinality().toString())); } + builder.setCreated(context.getOperationContext().getAuditStamp()); + builder.setLastModified(context.getOperationContext().getAuditStamp()); MetadataChangeProposal mcp = builder.build(); _entityClient.ingestProposal(context.getOperationContext(), mcp, false); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/DeleteStructuredPropertyResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/DeleteStructuredPropertyResolver.java new file mode 100644 index 00000000000000..e7d59494654fdd --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/DeleteStructuredPropertyResolver.java @@ -0,0 +1,52 @@ +package com.linkedin.datahub.graphql.resolvers.structuredproperties; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument; + +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.DeleteStructuredPropertyInput; +import com.linkedin.entity.client.EntityClient; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import javax.annotation.Nonnull; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class DeleteStructuredPropertyResolver implements DataFetcher> { + + private final EntityClient _entityClient; + + public DeleteStructuredPropertyResolver(@Nonnull final EntityClient entityClient) { + _entityClient = Objects.requireNonNull(entityClient, "entityClient must not be null"); + } + + @Override + public CompletableFuture get(final DataFetchingEnvironment environment) + throws Exception { + final QueryContext context = environment.getContext(); + + final DeleteStructuredPropertyInput input = + bindArgument(environment.getArgument("input"), DeleteStructuredPropertyInput.class); + final Urn propertyUrn = UrnUtils.getUrn(input.getUrn()); + + return CompletableFuture.supplyAsync( + () -> { + try { + if (!AuthorizationUtils.canManageStructuredProperties(context)) { + throw new AuthorizationException( + "Unable to delete structured property. Please contact your admin."); + } + _entityClient.deleteEntity(context.getOperationContext(), propertyUrn); + return true; + } catch (Exception e) { + throw new RuntimeException( + String.format("Failed to perform update against input %s", input), e); + } + }); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpdateStructuredPropertyResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpdateStructuredPropertyResolver.java index 2549f303bacd95..f5c101ba2bf64c 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpdateStructuredPropertyResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpdateStructuredPropertyResolver.java @@ -1,6 +1,7 @@ package com.linkedin.datahub.graphql.resolvers.structuredproperties; import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument; +import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME; import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_ENTITY_NAME; import com.linkedin.common.urn.Urn; @@ -21,17 +22,21 @@ import com.linkedin.structured.PrimitivePropertyValue; import com.linkedin.structured.PropertyCardinality; import com.linkedin.structured.PropertyValue; +import com.linkedin.structured.StructuredPropertyDefinition; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; import java.util.Objects; import java.util.concurrent.CompletableFuture; import javax.annotation.Nonnull; +import javax.annotation.Nullable; public class UpdateStructuredPropertyResolver implements DataFetcher> { private final EntityClient _entityClient; + private static final String ALLOWED_TYPES = "allowedTypes"; + public UpdateStructuredPropertyResolver(@Nonnull final EntityClient entityClient) { _entityClient = Objects.requireNonNull(entityClient, "entityClient must not be null"); } @@ -52,6 +57,8 @@ public CompletableFuture get(final DataFetchingEnviron "Unable to update structured property. Please contact your admin."); } final Urn propertyUrn = UrnUtils.getUrn(input.getUrn()); + StructuredPropertyDefinition existingDefinition = + getExistingStructuredProperty(context, propertyUrn); StructuredPropertyDefinitionPatchBuilder builder = new StructuredPropertyDefinitionPatchBuilder().urn(propertyUrn); @@ -65,7 +72,7 @@ public CompletableFuture get(final DataFetchingEnviron builder.setImmutable(input.getImmutable()); } if (input.getTypeQualifier() != null) { - buildTypeQualifier(input, builder); + buildTypeQualifier(input, builder, existingDefinition); } if (input.getNewAllowedValues() != null) { buildAllowedValues(input, builder); @@ -76,6 +83,7 @@ public CompletableFuture get(final DataFetchingEnviron if (input.getNewEntityTypes() != null) { input.getNewEntityTypes().forEach(builder::addEntityType); } + builder.setLastModified(context.getOperationContext().getAuditStamp()); MetadataChangeProposal mcp = builder.build(); _entityClient.ingestProposal(context.getOperationContext(), mcp, false); @@ -96,10 +104,16 @@ public CompletableFuture get(final DataFetchingEnviron private void buildTypeQualifier( @Nonnull final UpdateStructuredPropertyInput input, - @Nonnull final StructuredPropertyDefinitionPatchBuilder builder) { + @Nonnull final StructuredPropertyDefinitionPatchBuilder builder, + @Nullable final StructuredPropertyDefinition existingDefinition) { if (input.getTypeQualifier().getNewAllowedTypes() != null) { final StringArrayMap typeQualifier = new StringArrayMap(); StringArray allowedTypes = new StringArray(); + if (existingDefinition != null + && existingDefinition.getTypeQualifier() != null + && existingDefinition.getTypeQualifier().get(ALLOWED_TYPES) != null) { + allowedTypes.addAll(existingDefinition.getTypeQualifier().get(ALLOWED_TYPES)); + } allowedTypes.addAll(input.getTypeQualifier().getNewAllowedTypes()); typeQualifier.put("allowedTypes", allowedTypes); builder.setTypeQualifier(typeQualifier); @@ -126,4 +140,18 @@ private void buildAllowedValues( builder.addAllowedValue(value); }); } + + private StructuredPropertyDefinition getExistingStructuredProperty( + @Nonnull final QueryContext context, @Nonnull final Urn propertyUrn) throws Exception { + EntityResponse response = + _entityClient.getV2( + context.getOperationContext(), STRUCTURED_PROPERTY_ENTITY_NAME, propertyUrn, null); + + if (response != null + && response.getAspects().containsKey(STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME)) { + return new StructuredPropertyDefinition( + response.getAspects().get(STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME).getValue().data()); + } + return null; + } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/tag/SetTagColorResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/tag/SetTagColorResolver.java index 7a059ed9a1aeda..9a8e0fc3647274 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/tag/SetTagColorResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/tag/SetTagColorResolver.java @@ -101,10 +101,6 @@ public static boolean isAuthorizedToSetTagColor(@Nonnull QueryContext context, U ImmutableList.of(PoliciesConfig.EDIT_TAG_COLOR_PRIVILEGE.getType())))); return AuthorizationUtils.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - entityUrn.getEntityType(), - entityUrn.toString(), - orPrivilegeGroups); + context, entityUrn.getEntityType(), entityUrn.toString(), orPrivilegeGroups); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/test/TestUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/test/TestUtils.java index 020064ed643c88..80e5abd245b281 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/test/TestUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/test/TestUtils.java @@ -22,13 +22,13 @@ public class TestUtils { /** Returns true if the authenticated user is able to view tests. */ public static boolean canViewTests(@Nonnull QueryContext context) { return AuthUtil.isAuthorized( - context.getAuthorizer(), context.getActorUrn(), PoliciesConfig.VIEW_TESTS_PRIVILEGE); + context.getOperationContext(), PoliciesConfig.VIEW_TESTS_PRIVILEGE); } /** Returns true if the authenticated user is able to manage tests. */ public static boolean canManageTests(@Nonnull QueryContext context) { return AuthUtil.isAuthorized( - context.getAuthorizer(), context.getActorUrn(), PoliciesConfig.MANAGE_TESTS_PRIVILEGE); + context.getOperationContext(), PoliciesConfig.MANAGE_TESTS_PRIVILEGE); } public static TestDefinition mapDefinition(final TestDefinitionInput testDefInput) { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ListGlobalViewsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ListGlobalViewsResolver.java index 265f4d5f5d56e2..bda950ef3eb581 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ListGlobalViewsResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ListGlobalViewsResolver.java @@ -16,7 +16,6 @@ import com.linkedin.datahub.graphql.generated.ListViewsResult; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.Constants; -import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.query.filter.SortCriterion; import com.linkedin.metadata.query.filter.SortOrder; @@ -31,7 +30,6 @@ import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; import javax.annotation.Nonnull; -import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; /** Resolver used for listing global DataHub Views. */ @@ -73,7 +71,7 @@ public CompletableFuture get(final DataFetchingEnvironment envi context.getOperationContext().withSearchFlags(flags -> flags.setFulltext(true)), Constants.DATAHUB_VIEW_ENTITY_NAME, query, - buildFilters(context.getOperationContext().getAspectRetriever()), + buildFilters(), Collections.singletonList(DEFAULT_SORT_CRITERION), start, count); @@ -109,7 +107,7 @@ private List mapUnresolvedViews(final List entityUrns) { return results; } - private Filter buildFilters(@Nullable AspectRetriever aspectRetriever) { + private Filter buildFilters() { final AndFilterInput globalCriteria = new AndFilterInput(); List andConditions = new ArrayList<>(); andConditions.add( @@ -120,6 +118,6 @@ private Filter buildFilters(@Nullable AspectRetriever aspectRetriever) { false, FilterOperator.EQUAL)); globalCriteria.setAnd(andConditions); - return buildFilter(Collections.emptyList(), ImmutableList.of(globalCriteria), aspectRetriever); + return buildFilter(Collections.emptyList(), ImmutableList.of(globalCriteria)); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ListMyViewsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ListMyViewsResolver.java index abfdeb2d608693..53a7c92bbe8ec3 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ListMyViewsResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ListMyViewsResolver.java @@ -132,6 +132,6 @@ private Filter buildFilters( filterCriteria.setAnd(andConditions); // Currently, there is no way to fetch the views belonging to another user. - return buildFilter(Collections.emptyList(), ImmutableList.of(filterCriteria), aspectRetriever); + return buildFilter(Collections.emptyList(), ImmutableList.of(filterCriteria)); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ViewUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ViewUtils.java index 70a5ced4bfbf10..aa41a1e4876517 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ViewUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ViewUtils.java @@ -141,7 +141,7 @@ private static Filter buildAndFilter( .setAnd( new CriterionArray( input.stream() - .map(f -> ResolverUtils.criterionFromFilter(f, aspectRetriever)) + .map(ResolverUtils::criterionFromFilter) .collect(Collectors.toList())))))); return result; } @@ -157,9 +157,7 @@ private static Filter buildOrFilter( new ConjunctiveCriterion() .setAnd( new CriterionArray( - ImmutableList.of( - ResolverUtils.criterionFromFilter( - filter, aspectRetriever))))) + ImmutableList.of(ResolverUtils.criterionFromFilter(filter))))) .collect(Collectors.toList()))); return result; } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/ChartType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/ChartType.java index fe9b511f4a7dde..054dcec15af32d 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/ChartType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/ChartType.java @@ -242,11 +242,7 @@ private boolean isAuthorized( // Decide whether the current principal should be allowed to update the Dataset. final DisjunctivePrivilegeGroup orPrivilegeGroups = getAuthorizedPrivileges(update); return AuthorizationUtils.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - PoliciesConfig.CHART_PRIVILEGES.getResourceType(), - urn, - orPrivilegeGroups); + context, PoliciesConfig.CHART_PRIVILEGES.getResourceType(), urn, orPrivilegeGroups); } private DisjunctivePrivilegeGroup getAuthorizedPrivileges(final ChartUpdateInput updateInput) { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/SearchFlagsInputMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/SearchFlagsInputMapper.java index e6b75f9482f59f..9f5025ccf303a2 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/SearchFlagsInputMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/SearchFlagsInputMapper.java @@ -1,5 +1,6 @@ package com.linkedin.datahub.graphql.types.common.mappers; +import com.linkedin.data.template.StringArray; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.SearchFlags; import com.linkedin.datahub.graphql.types.mappers.ModelMapper; @@ -64,6 +65,10 @@ public com.linkedin.metadata.query.SearchFlags apply( .map(c -> GroupingCriterionInputMapper.map(context, c)) .collect(Collectors.toList())))); } + if (searchFlags.getCustomHighlightingFields() != null) { + result.setCustomHighlightingFields( + new StringArray(searchFlags.getCustomHighlightingFields())); + } return result; } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpgroup/CorpGroupType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpgroup/CorpGroupType.java index 16d2940a392447..27b97bfb2124f7 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpgroup/CorpGroupType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpgroup/CorpGroupType.java @@ -188,11 +188,7 @@ private boolean isAuthorizedToUpdate( // Decide whether the current principal should be allowed to update the Dataset. final DisjunctivePrivilegeGroup orPrivilegeGroups = getAuthorizedPrivileges(input); return AuthorizationUtils.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - PoliciesConfig.CORP_GROUP_PRIVILEGES.getResourceType(), - urn, - orPrivilegeGroups); + context, PoliciesConfig.CORP_GROUP_PRIVILEGES.getResourceType(), urn, orPrivilegeGroups); } private DisjunctivePrivilegeGroup getAuthorizedPrivileges( diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpuser/CorpUserType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpuser/CorpUserType.java index 3c2bfd7225edf5..5a812daa264bbf 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpuser/CorpUserType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpuser/CorpUserType.java @@ -183,8 +183,7 @@ private boolean isAuthorizedToUpdate( // information. return context.getActorUrn().equals(urn) || AuthorizationUtils.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), + context, PoliciesConfig.CORP_GROUP_PRIVILEGES.getResourceType(), urn, orPrivilegeGroups); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/DashboardType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/DashboardType.java index 89a41732109964..6ad362e5905904 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/DashboardType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/DashboardType.java @@ -241,11 +241,7 @@ private boolean isAuthorized( // Decide whether the current principal should be allowed to update the Dataset. final DisjunctivePrivilegeGroup orPrivilegeGroups = getAuthorizedPrivileges(update); return AuthorizationUtils.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - PoliciesConfig.DASHBOARD_PRIVILEGES.getResourceType(), - urn, - orPrivilegeGroups); + context, PoliciesConfig.DASHBOARD_PRIVILEGES.getResourceType(), urn, orPrivilegeGroups); } private DisjunctivePrivilegeGroup getAuthorizedPrivileges( diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/DataFlowType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/DataFlowType.java index f8248aedf289c0..3a697517bdecee 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/DataFlowType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/DataFlowType.java @@ -228,11 +228,7 @@ private boolean isAuthorized( // Decide whether the current principal should be allowed to update the Dataset. final DisjunctivePrivilegeGroup orPrivilegeGroups = getAuthorizedPrivileges(update); return AuthorizationUtils.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - PoliciesConfig.DATA_FLOW_PRIVILEGES.getResourceType(), - urn, - orPrivilegeGroups); + context, PoliciesConfig.DATA_FLOW_PRIVILEGES.getResourceType(), urn, orPrivilegeGroups); } private DisjunctivePrivilegeGroup getAuthorizedPrivileges(final DataFlowUpdateInput updateInput) { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/DataJobType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/DataJobType.java index 1e1de615b5911b..b32832a28d5d57 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/DataJobType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/DataJobType.java @@ -229,11 +229,7 @@ private boolean isAuthorized( // Decide whether the current principal should be allowed to update the Dataset. final DisjunctivePrivilegeGroup orPrivilegeGroups = getAuthorizedPrivileges(update); return AuthorizationUtils.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - PoliciesConfig.DATA_JOB_PRIVILEGES.getResourceType(), - urn, - orPrivilegeGroups); + context, PoliciesConfig.DATA_JOB_PRIVILEGES.getResourceType(), urn, orPrivilegeGroups); } private DisjunctivePrivilegeGroup getAuthorizedPrivileges(final DataJobUpdateInput updateInput) { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java index 46c810ac00d621..65b5d39e315692 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java @@ -283,11 +283,7 @@ private boolean isAuthorized( // Decide whether the current principal should be allowed to update the Dataset. final DisjunctivePrivilegeGroup orPrivilegeGroups = getAuthorizedPrivileges(update); return AuthorizationUtils.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - PoliciesConfig.DATASET_PRIVILEGES.getResourceType(), - urn, - orPrivilegeGroups); + context, PoliciesConfig.DATASET_PRIVILEGES.getResourceType(), urn, orPrivilegeGroups); } private DisjunctivePrivilegeGroup getAuthorizedPrivileges(final DatasetUpdateInput updateInput) { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaFieldMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaFieldMapper.java index 3674186ac23fe6..23d94ff85bd3d4 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaFieldMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaFieldMapper.java @@ -91,8 +91,7 @@ private SchemaFieldEntity createSchemaFieldEntity( @Nonnull final com.linkedin.schema.SchemaField input, @Nonnull Urn entityUrn) { SchemaFieldEntity schemaFieldEntity = new SchemaFieldEntity(); schemaFieldEntity.setUrn( - SchemaFieldUtils.generateSchemaFieldUrn(entityUrn.toString(), input.getFieldPath()) - .toString()); + SchemaFieldUtils.generateSchemaFieldUrn(entityUrn, input.getFieldPath()).toString()); schemaFieldEntity.setType(EntityType.SCHEMA_FIELD); return schemaFieldEntity; } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/ermodelrelationship/ERModelRelationshipType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/ermodelrelationship/ERModelRelationshipType.java index fd340aca119b59..ed52cc5486e92f 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/ermodelrelationship/ERModelRelationshipType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/ermodelrelationship/ERModelRelationshipType.java @@ -211,11 +211,7 @@ public static boolean canUpdateERModelRelation( new DisjunctivePrivilegeGroup( ImmutableList.of(editPrivilegesGroup, specificPrivilegeGroup)); return AuthorizationUtils.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - resourceUrn.getEntityType(), - resourceUrn.toString(), - orPrivilegeGroups); + context, resourceUrn.getEntityType(), resourceUrn.toString(), orPrivilegeGroups); } public static boolean canCreateERModelRelation( @@ -232,18 +228,10 @@ public static boolean canCreateERModelRelation( new DisjunctivePrivilegeGroup(ImmutableList.of(editPrivilegesGroup, createPrivilegesGroup)); boolean sourcePrivilege = AuthorizationUtils.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - sourceUrn.getEntityType(), - sourceUrn.toString(), - orPrivilegeGroups); + context, sourceUrn.getEntityType(), sourceUrn.toString(), orPrivilegeGroups); boolean destinationPrivilege = AuthorizationUtils.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - destinationUrn.getEntityType(), - destinationUrn.toString(), - orPrivilegeGroups); + context, destinationUrn.getEntityType(), destinationUrn.toString(), orPrivilegeGroups); return sourcePrivilege && destinationPrivilege; } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/MapperUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/MapperUtils.java index 7dd12d62765c60..0d69e62c621a60 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/MapperUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/MapperUtils.java @@ -3,13 +3,18 @@ import static com.linkedin.datahub.graphql.util.SearchInsightsUtil.*; import static com.linkedin.metadata.utils.SearchUtil.*; +import com.linkedin.common.AuditStamp; import com.linkedin.common.UrnArray; import com.linkedin.common.urn.Urn; +import com.linkedin.data.template.StringMap; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.AggregationMetadata; +import com.linkedin.datahub.graphql.generated.CorpUser; import com.linkedin.datahub.graphql.generated.EntityPath; +import com.linkedin.datahub.graphql.generated.ExtraProperty; import com.linkedin.datahub.graphql.generated.FacetMetadata; import com.linkedin.datahub.graphql.generated.MatchedField; +import com.linkedin.datahub.graphql.generated.ResolvedAuditStamp; import com.linkedin.datahub.graphql.generated.SearchResult; import com.linkedin.datahub.graphql.generated.SearchSuggestion; import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper; @@ -35,7 +40,24 @@ public static SearchResult mapResult( return new SearchResult( UrnToEntityMapper.map(context, searchEntity.getEntity()), getInsightsFromFeatures(searchEntity.getFeatures()), - getMatchedFieldEntry(context, searchEntity.getMatchedFields())); + getMatchedFieldEntry(context, searchEntity.getMatchedFields()), + getExtraProperties(searchEntity.getExtraFields())); + } + + private static List getExtraProperties(@Nullable StringMap extraFields) { + if (extraFields == null) { + return List.of(); + } else { + return extraFields.entrySet().stream() + .map( + entry -> { + ExtraProperty extraProperty = new ExtraProperty(); + extraProperty.setName(entry.getKey()); + extraProperty.setValue(entry.getValue()); + return extraProperty; + }) + .collect(Collectors.toList()); + } } public static FacetMetadata mapFacet( @@ -113,4 +135,13 @@ public static EntityPath mapPath(@Nullable final QueryContext context, UrnArray path.stream().map(p -> UrnToEntityMapper.map(context, p)).collect(Collectors.toList())); return entityPath; } + + public static ResolvedAuditStamp createResolvedAuditStamp(AuditStamp auditStamp) { + final ResolvedAuditStamp resolvedAuditStamp = new ResolvedAuditStamp(); + final CorpUser emptyCreatedUser = new CorpUser(); + emptyCreatedUser.setUrn(auditStamp.getActor().toString()); + resolvedAuditStamp.setActor(emptyCreatedUser); + resolvedAuditStamp.setTime(auditStamp.getTime()); + return resolvedAuditStamp; + } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/notebook/NotebookType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/notebook/NotebookType.java index 8eeda9331ad8ff..a6f29c1917397f 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/notebook/NotebookType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/notebook/NotebookType.java @@ -226,11 +226,7 @@ private boolean isAuthorized( // Decide whether the current principal should be allowed to update the Dataset. final DisjunctivePrivilegeGroup orPrivilegeGroups = getAuthorizedPrivileges(update); return AuthorizationUtils.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - PoliciesConfig.NOTEBOOK_PRIVILEGES.getResourceType(), - urn, - orPrivilegeGroups); + context, PoliciesConfig.NOTEBOOK_PRIVILEGES.getResourceType(), urn, orPrivilegeGroups); } private DisjunctivePrivilegeGroup getAuthorizedPrivileges(final NotebookUpdateInput updateInput) { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertyMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertyMapper.java index ff54131506a7cc..c539c65118ac6d 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertyMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertyMapper.java @@ -2,7 +2,9 @@ import static com.linkedin.metadata.Constants.*; +import com.google.common.collect.ImmutableList; import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; import com.linkedin.data.DataMap; import com.linkedin.data.template.StringArrayMap; import com.linkedin.datahub.graphql.QueryContext; @@ -17,6 +19,7 @@ import com.linkedin.datahub.graphql.generated.StructuredPropertyEntity; import com.linkedin.datahub.graphql.generated.TypeQualifier; import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper; +import com.linkedin.datahub.graphql.types.mappers.MapperUtils; import com.linkedin.datahub.graphql.types.mappers.ModelMapper; import com.linkedin.entity.EntityResponse; import com.linkedin.entity.EnvelopedAspectMap; @@ -45,6 +48,9 @@ public StructuredPropertyEntity apply( final StructuredPropertyEntity result = new StructuredPropertyEntity(); result.setUrn(entityResponse.getUrn().toString()); result.setType(EntityType.STRUCTURED_PROPERTY); + // set the default required values for a structured property in case references are still being + // cleaned up + setDefaultProperty(result); EnvelopedAspectMap aspectMap = entityResponse.getAspects(); MappingHelper mappingHelper = new MappingHelper<>(aspectMap, result); mappingHelper.mapToResult( @@ -74,6 +80,13 @@ private void mapStructuredPropertyDefinition( if (gmsDefinition.hasTypeQualifier()) { definition.setTypeQualifier(mapTypeQualifier(gmsDefinition.getTypeQualifier())); } + if (gmsDefinition.getCreated() != null) { + definition.setCreated(MapperUtils.createResolvedAuditStamp(gmsDefinition.getCreated())); + } + if (gmsDefinition.getLastModified() != null) { + definition.setLastModified( + MapperUtils.createResolvedAuditStamp(gmsDefinition.getLastModified())); + } definition.setEntityTypes( gmsDefinition.getEntityTypes().stream() .map(this::createEntityTypeEntity) @@ -126,4 +139,22 @@ private EntityTypeEntity createEntityTypeEntity(final String entityTypeUrnStr) { entityType.setType(EntityType.ENTITY_TYPE); return entityType; } + + /* + * In the case that a property is deleted and the references haven't been cleaned up yet (this process is async) + * set a default property to prevent APIs breaking. The UI queries for whether the entity exists and it will + * be filtered out. + */ + private void setDefaultProperty(final StructuredPropertyEntity result) { + StructuredPropertyDefinition definition = new StructuredPropertyDefinition(); + definition.setQualifiedName(""); + definition.setCardinality(PropertyCardinality.SINGLE); + definition.setImmutable(true); + definition.setValueType( + createDataTypeEntity(UrnUtils.getUrn("urn:li:dataType:datahub.string"))); + definition.setEntityTypes( + ImmutableList.of( + createEntityTypeEntity(UrnUtils.getUrn("urn:li:entityType:datahub.dataset")))); + result.setDefinition(definition); + } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/tag/TagType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/tag/TagType.java index 3c07b242e9d813..9a5d0f1f41b8cc 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/tag/TagType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/tag/TagType.java @@ -164,8 +164,7 @@ private boolean isAuthorized(@Nonnull TagUpdateInput update, @Nonnull QueryConte // Decide whether the current principal should be allowed to update the Dataset. final DisjunctivePrivilegeGroup orPrivilegeGroups = getAuthorizedPrivileges(update); return AuthorizationUtils.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), + context, PoliciesConfig.TAG_PRIVILEGES.getResourceType(), update.getUrn(), orPrivilegeGroups); diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql index 609597beee51bd..a2e2fe9163f536 100644 --- a/datahub-graphql-core/src/main/resources/entity.graphql +++ b/datahub-graphql-core/src/main/resources/entity.graphql @@ -84,6 +84,11 @@ type Query { """ role(urn: String!): Role + """ + Fetch a Structured Property by primary key (urn) + """ + structuredProperty(urn: String!): StructuredPropertyEntity + """ Fetch a ERModelRelationship by primary key (urn) """ @@ -11274,6 +11279,21 @@ input CreatePostInput { The content of the post """ content: UpdatePostContentInput! + + """ + Optional target URN for the post + """ + resourceUrn: String + + """ + An optional type of a sub resource to attach the Tag to + """ + subResourceType: SubResourceType + + """ + Optional target subresource for the post + """ + subResource: String } """ diff --git a/datahub-graphql-core/src/main/resources/properties.graphql b/datahub-graphql-core/src/main/resources/properties.graphql index dfe84686456814..292381d064f362 100644 --- a/datahub-graphql-core/src/main/resources/properties.graphql +++ b/datahub-graphql-core/src/main/resources/properties.graphql @@ -18,6 +18,11 @@ extend type Mutation { Update an existing structured property """ updateStructuredProperty(input: UpdateStructuredPropertyInput!): StructuredPropertyEntity! + + """ + Delete an existing structured property + """ + deleteStructuredProperty(input: DeleteStructuredPropertyInput!): Boolean! } """ @@ -34,6 +39,11 @@ type StructuredPropertyEntity implements Entity { """ type: EntityType! + """ + Whether or not this entity exists on DataHub + """ + exists: Boolean + """ Definition of this structured property including its name """ @@ -95,6 +105,16 @@ type StructuredPropertyDefinition { Whether or not this structured property is immutable """ immutable: Boolean! + + """ + Audit stamp for when this structured property was created + """ + created: ResolvedAuditStamp + + """ + Audit stamp for when this structured property was last modified + """ + lastModified: ResolvedAuditStamp } """ @@ -447,3 +467,13 @@ input UpdateTypeQualifierInput { """ newAllowedTypes: [String!] } + +""" +Input for deleting a form +""" +input DeleteStructuredPropertyInput { + """ + The urn of the structured properties that is being deleted + """ + urn: String! +} diff --git a/datahub-graphql-core/src/main/resources/search.graphql b/datahub-graphql-core/src/main/resources/search.graphql index 09a7217073527b..32d73845e1ae40 100644 --- a/datahub-graphql-core/src/main/resources/search.graphql +++ b/datahub-graphql-core/src/main/resources/search.graphql @@ -162,6 +162,11 @@ input SearchFlags { Whether to include restricted entities """ includeRestricted: Boolean + + """ + fields to include for custom Highlighting + """ + customHighlightingFields: [String!] } """ @@ -545,6 +550,31 @@ enum FilterOperator { Represent the relation less than or equal to, e.g. ownerCount <= 3 """ LESS_THAN_OR_EQUAL_TO + + """ + Represent the relation: String field starts with value, e.g. name starts with PageView + """ + START_WITH + + """ + Represent the relation: String field ends with value, e.g. name ends with Event + """ + END_WITH + + """ + Represent the relation: URN field any nested children in addition to the given URN + """ + DESCENDANTS_INCL + + """ + Represent the relation: URN field matches any nested parent in addition to the given URN + """ + ANCESTORS_INCL + + """ + Represent the relation: URN field matches any nested child or parent in addition to the given URN + """ + RELATED_INCL } """ @@ -582,6 +612,18 @@ type SearchResults { suggestions: [SearchSuggestion!] } +type ExtraProperty { + """ + Name of the extra property + """ + name: String! + + """ + Value of the extra property + """ + value: String! +} + """ An individual search result hit """ @@ -600,6 +642,11 @@ type SearchResult { Matched field hint """ matchedFields: [MatchedField!]! + + """ + Additional properties about the search result. Used for rendering in the UI + """ + extraProperties: [ExtraProperty!] } """ diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ResolverUtilsTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ResolverUtilsTest.java index f98284e92ede58..2950b50bec8f18 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ResolverUtilsTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ResolverUtilsTest.java @@ -1,6 +1,7 @@ package com.linkedin.datahub.graphql.resolvers; import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; +import static com.linkedin.metadata.search.utils.QueryUtils.buildFilterWithUrns; import static org.mockito.Mockito.mock; import static org.testng.AssertJUnit.assertEquals; @@ -12,7 +13,8 @@ import com.linkedin.datahub.graphql.TestUtils; import com.linkedin.datahub.graphql.generated.FacetFilterInput; import com.linkedin.datahub.graphql.generated.FilterOperator; -import com.linkedin.metadata.aspect.AspectRetriever; +import com.linkedin.metadata.config.DataHubAppConfiguration; +import com.linkedin.metadata.config.MetadataChangeProposalConfig; import com.linkedin.metadata.query.filter.Condition; import com.linkedin.metadata.query.filter.ConjunctiveCriterion; import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; @@ -42,8 +44,7 @@ public void testCriterionFromFilter() throws Exception { null, ImmutableList.of("urn:li:tag:abc", "urn:li:tag:def"), false, - FilterOperator.EQUAL), - mock(AspectRetriever.class)); + FilterOperator.EQUAL)); assertEquals( valuesCriterion, new Criterion() @@ -51,13 +52,12 @@ public void testCriterionFromFilter() throws Exception { .setValues(new StringArray(ImmutableList.of("urn:li:tag:abc", "urn:li:tag:def"))) .setNegated(false) .setCondition(Condition.EQUAL) - .setField("tags.keyword")); + .setField("tags")); // this is the legacy pathway Criterion valueCriterion = criterionFromFilter( - new FacetFilterInput("tags", "urn:li:tag:abc", null, true, FilterOperator.EQUAL), - mock(AspectRetriever.class)); + new FacetFilterInput("tags", "urn:li:tag:abc", null, true, FilterOperator.EQUAL)); assertEquals( valueCriterion, new Criterion() @@ -65,14 +65,12 @@ public void testCriterionFromFilter() throws Exception { .setValues(new StringArray(ImmutableList.of("urn:li:tag:abc"))) .setNegated(true) .setCondition(Condition.EQUAL) - .setField("tags.keyword")); + .setField("tags")); // check that both being null doesn't cause a NPE. this should never happen except via API // interaction Criterion doubleNullCriterion = - criterionFromFilter( - new FacetFilterInput("tags", null, null, true, FilterOperator.EQUAL), - mock(AspectRetriever.class)); + criterionFromFilter(new FacetFilterInput("tags", null, null, true, FilterOperator.EQUAL)); assertEquals( doubleNullCriterion, new Criterion() @@ -80,7 +78,7 @@ public void testCriterionFromFilter() throws Exception { .setValues(new StringArray(ImmutableList.of())) .setNegated(true) .setCondition(Condition.EQUAL) - .setField("tags.keyword")); + .setField("tags")); } @Test @@ -102,7 +100,18 @@ public void testBuildFilterWithUrns() throws Exception { new ConjunctiveCriterionArray( ImmutableList.of(new ConjunctiveCriterion().setAnd(andCriterionArray)))); - Filter finalFilter = buildFilterWithUrns(urns, filter); + DataHubAppConfiguration appConfig = new DataHubAppConfiguration(); + appConfig.setMetadataChangeProposal(new MetadataChangeProposalConfig()); + appConfig + .getMetadataChangeProposal() + .setSideEffects(new MetadataChangeProposalConfig.SideEffectsConfig()); + appConfig + .getMetadataChangeProposal() + .getSideEffects() + .setSchemaField(new MetadataChangeProposalConfig.SideEffectConfig()); + appConfig.getMetadataChangeProposal().getSideEffects().getSchemaField().setEnabled(true); + + Filter finalFilter = buildFilterWithUrns(appConfig, urns, filter); Criterion urnsCriterion = new Criterion() diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/auth/ListAccessTokensResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/auth/ListAccessTokensResolverTest.java index 020f74475ea607..30962aaef54125 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/auth/ListAccessTokensResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/auth/ListAccessTokensResolverTest.java @@ -46,7 +46,7 @@ public void testGetSuccess() throws Exception { any(), Mockito.eq(Constants.ACCESS_TOKEN_ENTITY_NAME), Mockito.eq(""), - Mockito.eq(buildFilter(filters, Collections.emptyList(), null)), + Mockito.eq(buildFilter(filters, Collections.emptyList())), Mockito.any(List.class), Mockito.eq(input.getStart()), Mockito.eq(input.getCount()))) diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/browse/BrowseV2ResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/browse/BrowseV2ResolverTest.java index 9cf7e62e65e253..4897d0819b59fb 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/browse/BrowseV2ResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/browse/BrowseV2ResolverTest.java @@ -2,7 +2,6 @@ import static com.linkedin.datahub.graphql.TestUtils.getMockAllowContext; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; import com.google.common.collect.ImmutableList; import com.linkedin.common.AuditStamp; @@ -18,7 +17,6 @@ import com.linkedin.datahub.graphql.resolvers.ResolverUtils; import com.linkedin.datahub.graphql.resolvers.chart.BrowseV2Resolver; import com.linkedin.entity.client.EntityClient; -import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.browse.BrowseResultGroupV2; import com.linkedin.metadata.browse.BrowseResultGroupV2Array; import com.linkedin.metadata.browse.BrowseResultMetadata; @@ -102,7 +100,7 @@ public static void testBrowseV2SuccessWithQueryAndFilter() throws Exception { facetFilterInput.setValues(ImmutableList.of("urn:li:corpuser:test")); andFilterInput.setAnd(ImmutableList.of(facetFilterInput)); orFilters.add(andFilterInput); - Filter filter = ResolverUtils.buildFilter(null, orFilters, mock(AspectRetriever.class)); + Filter filter = ResolverUtils.buildFilter(null, orFilters); EntityClient mockClient = initMockEntityClient( diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/dashboard/DashboardStatsSummaryTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/dashboard/DashboardStatsSummaryTest.java index 837dec2f528ed3..76879addc5e6f3 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/dashboard/DashboardStatsSummaryTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/dashboard/DashboardStatsSummaryTest.java @@ -7,7 +7,6 @@ import com.datahub.authentication.Authentication; import com.datahub.authorization.AuthorizationResult; -import com.datahub.plugins.auth.authorization.Authorizer; import com.google.common.collect.ImmutableList; import com.linkedin.common.urn.UrnUtils; import com.linkedin.dashboard.DashboardUsageStatistics; @@ -55,13 +54,10 @@ public void testGetSuccess() throws Exception { // Execute resolver DashboardStatsSummaryResolver resolver = new DashboardStatsSummaryResolver(mockClient); QueryContext mockContext = Mockito.mock(QueryContext.class); - Authorizer mockAuthorizor = mock(Authorizer.class); - when(mockAuthorizor.authorize(any())) - .thenAnswer( - args -> - new AuthorizationResult(args.getArgument(0), AuthorizationResult.Type.ALLOW, "")); - when(mockContext.getAuthorizer()).thenReturn(mockAuthorizor); - Mockito.when(mockContext.getAuthentication()).thenReturn(Mockito.mock(Authentication.class)); + when(mockContext.getOperationContext()).thenReturn(mock(OperationContext.class)); + when(mockContext.getOperationContext().authorize(any(), any())) + .thenReturn(new AuthorizationResult(null, AuthorizationResult.Type.ALLOW, "")); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); Mockito.when(mockEnv.getSource()).thenReturn(TEST_SOURCE); Mockito.when(mockEnv.getContext()).thenReturn(mockContext); diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/datacontract/DataContractUtilsTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/datacontract/DataContractUtilsTest.java index 18ede7c306e424..646c72422aa618 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/datacontract/DataContractUtilsTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/datacontract/DataContractUtilsTest.java @@ -12,6 +12,7 @@ import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.metadata.config.DataHubAppConfiguration; import graphql.Assert; import io.datahubproject.metadata.context.OperationContext; import io.datahubproject.test.metadata.context.TestOperationContexts; @@ -56,6 +57,11 @@ public OperationContext getOperationContext() { return TestOperationContexts.userContextNoSearchAuthorization( getAuthorizer(), getAuthentication()); } + + @Override + public DataHubAppConfiguration getDataHubAppConfig() { + return new DataHubAppConfiguration(); + } }, testUrn); Assert.assertTrue(result); diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetStatsSummaryResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetStatsSummaryResolverTest.java index f8a7e4fc6a13c8..57dd5ebc86e86a 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetStatsSummaryResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetStatsSummaryResolverTest.java @@ -4,7 +4,6 @@ import com.datahub.authentication.Authentication; import com.datahub.authorization.AuthorizationResult; -import com.datahub.plugins.auth.authorization.Authorizer; import com.google.common.collect.ImmutableList; import com.linkedin.common.urn.UrnUtils; import com.linkedin.datahub.graphql.QueryContext; @@ -66,14 +65,15 @@ public void testGetSuccess() throws Exception { DatasetStatsSummaryResolver resolver = new DatasetStatsSummaryResolver(mockClient); QueryContext mockContext = Mockito.mock(QueryContext.class); Mockito.when(mockContext.getActorUrn()).thenReturn("urn:li:corpuser:test"); - Authorizer mockAuthorizer = Mockito.mock(Authorizer.class); + AuthorizationResult mockAuthorizerResult = Mockito.mock(AuthorizationResult.class); Mockito.when(mockAuthorizerResult.getType()).thenReturn(AuthorizationResult.Type.ALLOW); - Mockito.when(mockAuthorizer.authorize(any())).thenReturn(mockAuthorizerResult); - Mockito.when(mockContext.getAuthorizer()).thenReturn(mockAuthorizer); - Mockito.when(mockContext.getAuthentication()).thenReturn(Mockito.mock(Authentication.class)); + Mockito.when(mockContext.getOperationContext()) .thenReturn(Mockito.mock(OperationContext.class)); + Mockito.when(mockContext.getOperationContext().authorize(any(), any())) + .thenReturn(mockAuthorizerResult); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); Mockito.when(mockEnv.getSource()).thenReturn(TEST_SOURCE); Mockito.when(mockEnv.getContext()).thenReturn(mockContext); diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/GlossaryUtilsTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/GlossaryUtilsTest.java index 448c3420625929..25d48ddec74069 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/GlossaryUtilsTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/GlossaryUtilsTest.java @@ -2,13 +2,13 @@ import static com.linkedin.metadata.Constants.GLOSSARY_NODE_INFO_ASPECT_NAME; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import static org.testng.Assert.*; -import com.datahub.authentication.Authentication; -import com.datahub.authorization.AuthorizationRequest; import com.datahub.authorization.AuthorizationResult; import com.datahub.authorization.EntitySpec; -import com.datahub.plugins.auth.authorization.Authorizer; import com.google.common.collect.ImmutableSet; import com.linkedin.common.urn.GlossaryNodeUrn; import com.linkedin.common.urn.Urn; @@ -22,27 +22,27 @@ import com.linkedin.entity.client.EntityClient; import com.linkedin.glossary.GlossaryNodeInfo; import com.linkedin.metadata.Constants; +import io.datahubproject.metadata.context.OperationContext; import java.util.HashMap; import java.util.Map; -import java.util.Optional; import org.mockito.Mockito; +import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; public class GlossaryUtilsTest { private final String userUrn = "urn:li:corpuser:authorized"; private final QueryContext mockContext = Mockito.mock(QueryContext.class); - private final Authorizer mockAuthorizer = Mockito.mock(Authorizer.class); private final EntityClient mockClient = Mockito.mock(EntityClient.class); private final Urn parentNodeUrn = UrnUtils.getUrn("urn:li:glossaryNode:parent_node"); private final Urn parentNodeUrn1 = UrnUtils.getUrn("urn:li:glossaryNode:parent_node1"); private final Urn parentNodeUrn2 = UrnUtils.getUrn("urn:li:glossaryNode:parent_node2"); private final Urn parentNodeUrn3 = UrnUtils.getUrn("urn:li:glossaryNode:parent_node3"); + @BeforeMethod private void setUpTests() throws Exception { - Mockito.when(mockContext.getAuthorizer()).thenReturn(mockAuthorizer); Mockito.when(mockContext.getActorUrn()).thenReturn(userUrn); - Mockito.when(mockContext.getAuthentication()).thenReturn(Mockito.mock(Authentication.class)); + when(mockContext.getOperationContext()).thenReturn(mock(OperationContext.class)); GlossaryNodeInfo parentNode1 = new GlossaryNodeInfo() @@ -84,25 +84,25 @@ private void setUpTests() throws Exception { Mockito.when( mockClient.getV2( any(), - Mockito.eq(Constants.GLOSSARY_NODE_ENTITY_NAME), - Mockito.eq(parentNodeUrn1), - Mockito.eq(ImmutableSet.of(GLOSSARY_NODE_INFO_ASPECT_NAME)))) + eq(Constants.GLOSSARY_NODE_ENTITY_NAME), + eq(parentNodeUrn1), + eq(ImmutableSet.of(GLOSSARY_NODE_INFO_ASPECT_NAME)))) .thenReturn(new EntityResponse().setAspects(new EnvelopedAspectMap(parentNode1Aspects))); Mockito.when( mockClient.getV2( any(), - Mockito.eq(Constants.GLOSSARY_NODE_ENTITY_NAME), - Mockito.eq(parentNodeUrn2), - Mockito.eq(ImmutableSet.of(GLOSSARY_NODE_INFO_ASPECT_NAME)))) + eq(Constants.GLOSSARY_NODE_ENTITY_NAME), + eq(parentNodeUrn2), + eq(ImmutableSet.of(GLOSSARY_NODE_INFO_ASPECT_NAME)))) .thenReturn(new EntityResponse().setAspects(new EnvelopedAspectMap(parentNode2Aspects))); Mockito.when( mockClient.getV2( any(), - Mockito.eq(Constants.GLOSSARY_NODE_ENTITY_NAME), - Mockito.eq(parentNodeUrn3), - Mockito.eq(ImmutableSet.of(GLOSSARY_NODE_INFO_ASPECT_NAME)))) + eq(Constants.GLOSSARY_NODE_ENTITY_NAME), + eq(parentNodeUrn3), + eq(ImmutableSet.of(GLOSSARY_NODE_INFO_ASPECT_NAME)))) .thenReturn(new EntityResponse().setAspects(new EnvelopedAspectMap(parentNode3Aspects))); final EntitySpec resourceSpec3 = @@ -120,19 +120,14 @@ private void setUpTests() throws Exception { private void mockAuthRequest( String privilege, AuthorizationResult.Type allowOrDeny, EntitySpec resourceSpec) { - final AuthorizationRequest authorizationRequest = - new AuthorizationRequest( - userUrn, - privilege, - resourceSpec != null ? Optional.of(resourceSpec) : Optional.empty()); AuthorizationResult result = Mockito.mock(AuthorizationResult.class); Mockito.when(result.getType()).thenReturn(allowOrDeny); - Mockito.when(mockAuthorizer.authorize(Mockito.eq(authorizationRequest))).thenReturn(result); + when(mockContext.getOperationContext().authorize(eq(privilege), eq(resourceSpec))) + .thenReturn(result); } @Test public void testCanManageGlossariesAuthorized() throws Exception { - setUpTests(); mockAuthRequest("MANAGE_GLOSSARIES", AuthorizationResult.Type.ALLOW, null); assertTrue(GlossaryUtils.canManageGlossaries(mockContext)); @@ -140,7 +135,6 @@ public void testCanManageGlossariesAuthorized() throws Exception { @Test public void testCanManageGlossariesUnauthorized() throws Exception { - setUpTests(); mockAuthRequest("MANAGE_GLOSSARIES", AuthorizationResult.Type.DENY, null); assertFalse(GlossaryUtils.canManageGlossaries(mockContext)); @@ -148,7 +142,6 @@ public void testCanManageGlossariesUnauthorized() throws Exception { @Test public void testCanManageChildrenEntitiesWithManageGlossaries() throws Exception { - setUpTests(); // they have the MANAGE_GLOSSARIES platform privilege mockAuthRequest("MANAGE_GLOSSARIES", AuthorizationResult.Type.ALLOW, null); @@ -157,7 +150,6 @@ public void testCanManageChildrenEntitiesWithManageGlossaries() throws Exception @Test public void testCanManageChildrenEntitiesNoParentNode() throws Exception { - setUpTests(); // they do NOT have the MANAGE_GLOSSARIES platform privilege mockAuthRequest("MANAGE_GLOSSARIES", AuthorizationResult.Type.DENY, null); @@ -166,7 +158,6 @@ public void testCanManageChildrenEntitiesNoParentNode() throws Exception { @Test public void testCanManageChildrenEntitiesAuthorized() throws Exception { - setUpTests(); // they do NOT have the MANAGE_GLOSSARIES platform privilege mockAuthRequest("MANAGE_GLOSSARIES", AuthorizationResult.Type.DENY, null); @@ -179,7 +170,6 @@ public void testCanManageChildrenEntitiesAuthorized() throws Exception { @Test public void testCanManageChildrenEntitiesUnauthorized() throws Exception { - setUpTests(); // they do NOT have the MANAGE_GLOSSARIES platform privilege mockAuthRequest("MANAGE_GLOSSARIES", AuthorizationResult.Type.DENY, null); @@ -193,7 +183,6 @@ public void testCanManageChildrenEntitiesUnauthorized() throws Exception { @Test public void testCanManageChildrenRecursivelyEntitiesAuthorized() throws Exception { - setUpTests(); // they do NOT have the MANAGE_GLOSSARIES platform privilege mockAuthRequest("MANAGE_GLOSSARIES", AuthorizationResult.Type.DENY, null); @@ -214,7 +203,6 @@ public void testCanManageChildrenRecursivelyEntitiesAuthorized() throws Exceptio @Test public void testCanManageChildrenRecursivelyEntitiesUnauthorized() throws Exception { - setUpTests(); // they do NOT have the MANAGE_GLOSSARIES platform privilege mockAuthRequest("MANAGE_GLOSSARIES", AuthorizationResult.Type.DENY, null); @@ -235,7 +223,6 @@ public void testCanManageChildrenRecursivelyEntitiesUnauthorized() throws Except @Test public void testCanManageChildrenRecursivelyEntitiesAuthorizedLevel2() throws Exception { - setUpTests(); // they do NOT have the MANAGE_GLOSSARIES platform privilege mockAuthRequest("MANAGE_GLOSSARIES", AuthorizationResult.Type.DENY, null); @@ -252,7 +239,6 @@ public void testCanManageChildrenRecursivelyEntitiesAuthorizedLevel2() throws Ex @Test public void testCanManageChildrenRecursivelyEntitiesUnauthorizedLevel2() throws Exception { - setUpTests(); // they do NOT have the MANAGE_GLOSSARIES platform privilege mockAuthRequest("MANAGE_GLOSSARIES", AuthorizationResult.Type.DENY, null); @@ -269,7 +255,6 @@ public void testCanManageChildrenRecursivelyEntitiesUnauthorizedLevel2() throws @Test public void testCanManageChildrenRecursivelyEntitiesNoLevel2() throws Exception { - setUpTests(); // they do NOT have the MANAGE_GLOSSARIES platform privilege mockAuthRequest("MANAGE_GLOSSARIES", AuthorizationResult.Type.DENY, null); diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/IngestTestUtils.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/IngestTestUtils.java index e0555f5886b8bb..963bdf93bc9f1f 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/IngestTestUtils.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/IngestTestUtils.java @@ -1,11 +1,13 @@ package com.linkedin.datahub.graphql.resolvers.ingest; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.nullable; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import static org.testng.Assert.*; -import com.datahub.authentication.Authentication; import com.datahub.authorization.AuthorizationResult; -import com.datahub.plugins.auth.authorization.Authorizer; +import com.datahub.authorization.EntitySpec; import com.google.common.collect.ImmutableMap; import com.linkedin.common.urn.Urn; import com.linkedin.data.template.StringMap; @@ -38,14 +40,9 @@ public static QueryContext getMockAllowContext() { QueryContext mockContext = Mockito.mock(QueryContext.class); Mockito.when(mockContext.getActorUrn()).thenReturn("urn:li:corpuser:test"); - Authorizer mockAuthorizer = Mockito.mock(Authorizer.class); - AuthorizationResult result = Mockito.mock(AuthorizationResult.class); - Mockito.when(result.getType()).thenReturn(AuthorizationResult.Type.ALLOW); - Mockito.when(mockAuthorizer.authorize(Mockito.any())).thenReturn(result); - - Mockito.when(mockContext.getAuthorizer()).thenReturn(mockAuthorizer); - Mockito.when(mockContext.getAuthentication()).thenReturn(Mockito.mock(Authentication.class)); - Mockito.when(mockContext.getOperationContext()).thenReturn(mock(OperationContext.class)); + when(mockContext.getOperationContext()).thenReturn(mock(OperationContext.class)); + when(mockContext.getOperationContext().authorize(any(), nullable(EntitySpec.class))) + .thenReturn(new AuthorizationResult(null, AuthorizationResult.Type.ALLOW, "")); return mockContext; } @@ -53,13 +50,9 @@ public static QueryContext getMockDenyContext() { QueryContext mockContext = Mockito.mock(QueryContext.class); Mockito.when(mockContext.getActorUrn()).thenReturn("urn:li:corpuser:test"); - Authorizer mockAuthorizer = Mockito.mock(Authorizer.class); - AuthorizationResult result = Mockito.mock(AuthorizationResult.class); - Mockito.when(result.getType()).thenReturn(AuthorizationResult.Type.DENY); - Mockito.when(mockAuthorizer.authorize(Mockito.any())).thenReturn(result); - - Mockito.when(mockContext.getAuthorizer()).thenReturn(mockAuthorizer); - Mockito.when(mockContext.getAuthentication()).thenReturn(Mockito.mock(Authentication.class)); + when(mockContext.getOperationContext()).thenReturn(mock(OperationContext.class)); + when(mockContext.getOperationContext().authorize(any(), nullable(EntitySpec.class))) + .thenReturn(new AuthorizationResult(null, AuthorizationResult.Type.DENY, "")); return mockContext; } diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/IngestionAuthUtilsTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/IngestionAuthUtilsTest.java index f3e27d91f39df0..ba7f80deec4b77 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/IngestionAuthUtilsTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/IngestionAuthUtilsTest.java @@ -1,13 +1,16 @@ package com.linkedin.datahub.graphql.resolvers.ingest; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import static org.testng.Assert.*; import com.datahub.authorization.AuthorizationRequest; import com.datahub.authorization.AuthorizationResult; import com.datahub.authorization.EntitySpec; -import com.datahub.plugins.auth.authorization.Authorizer; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.metadata.Constants; +import io.datahubproject.metadata.context.OperationContext; import java.util.Optional; import org.mockito.Mockito; import org.testng.annotations.Test; @@ -17,19 +20,18 @@ public class IngestionAuthUtilsTest { @Test public void testCanManageIngestionAuthorized() throws Exception { QueryContext mockContext = Mockito.mock(QueryContext.class); - Authorizer mockAuthorizer = Mockito.mock(Authorizer.class); - - AuthorizationRequest request = - new AuthorizationRequest( - "urn:li:corpuser:authorized", - "MANAGE_INGESTION", - Optional.of(new EntitySpec(Constants.INGESTION_SOURCE_ENTITY_NAME, ""))); + when(mockContext.getOperationContext()).thenReturn(mock(OperationContext.class)); AuthorizationResult result = Mockito.mock(AuthorizationResult.class); Mockito.when(result.getType()).thenReturn(AuthorizationResult.Type.ALLOW); - Mockito.when(mockAuthorizer.authorize(Mockito.eq(request))).thenReturn(result); + Mockito.when( + mockContext + .getOperationContext() + .authorize( + eq("MANAGE_INGESTION"), + eq(new EntitySpec(Constants.INGESTION_SOURCE_ENTITY_NAME, "")))) + .thenReturn(result); - Mockito.when(mockContext.getAuthorizer()).thenReturn(mockAuthorizer); Mockito.when(mockContext.getActorUrn()).thenReturn("urn:li:corpuser:authorized"); assertTrue(IngestionAuthUtils.canManageIngestion(mockContext)); @@ -38,19 +40,18 @@ public void testCanManageIngestionAuthorized() throws Exception { @Test public void testCanManageIngestionUnauthorized() throws Exception { QueryContext mockContext = Mockito.mock(QueryContext.class); - Authorizer mockAuthorizer = Mockito.mock(Authorizer.class); - - AuthorizationRequest request = - new AuthorizationRequest( - "urn:li:corpuser:unauthorized", - "MANAGE_INGESTION", - Optional.of(new EntitySpec(Constants.INGESTION_SOURCE_ENTITY_NAME, ""))); + when(mockContext.getOperationContext()).thenReturn(mock(OperationContext.class)); AuthorizationResult result = Mockito.mock(AuthorizationResult.class); Mockito.when(result.getType()).thenReturn(AuthorizationResult.Type.DENY); - Mockito.when(mockAuthorizer.authorize(Mockito.eq(request))).thenReturn(result); + Mockito.when( + mockContext + .getOperationContext() + .authorize( + eq("MANAGE_INGESTION"), + eq(new EntitySpec(Constants.INGESTION_SOURCE_ENTITY_NAME, "")))) + .thenReturn(result); - Mockito.when(mockContext.getAuthorizer()).thenReturn(mockAuthorizer); Mockito.when(mockContext.getActorUrn()).thenReturn("urn:li:corpuser:unauthorized"); assertFalse(IngestionAuthUtils.canManageIngestion(mockContext)); @@ -59,19 +60,17 @@ public void testCanManageIngestionUnauthorized() throws Exception { @Test public void testCanManageSecretsAuthorized() throws Exception { QueryContext mockContext = Mockito.mock(QueryContext.class); - Authorizer mockAuthorizer = Mockito.mock(Authorizer.class); - - AuthorizationRequest request = - new AuthorizationRequest( - "urn:li:corpuser:authorized", - "MANAGE_SECRETS", - Optional.of(new EntitySpec(Constants.SECRETS_ENTITY_NAME, ""))); + when(mockContext.getOperationContext()).thenReturn(mock(OperationContext.class)); AuthorizationResult result = Mockito.mock(AuthorizationResult.class); Mockito.when(result.getType()).thenReturn(AuthorizationResult.Type.ALLOW); - Mockito.when(mockAuthorizer.authorize(Mockito.eq(request))).thenReturn(result); + Mockito.when( + mockContext + .getOperationContext() + .authorize( + eq("MANAGE_SECRETS"), eq(new EntitySpec(Constants.SECRETS_ENTITY_NAME, "")))) + .thenReturn(result); - Mockito.when(mockContext.getAuthorizer()).thenReturn(mockAuthorizer); Mockito.when(mockContext.getActorUrn()).thenReturn("urn:li:corpuser:authorized"); assertTrue(IngestionAuthUtils.canManageSecrets(mockContext)); @@ -80,7 +79,7 @@ public void testCanManageSecretsAuthorized() throws Exception { @Test public void testCanManageSecretsUnauthorized() throws Exception { QueryContext mockContext = Mockito.mock(QueryContext.class); - Authorizer mockAuthorizer = Mockito.mock(Authorizer.class); + when(mockContext.getOperationContext()).thenReturn(mock(OperationContext.class)); AuthorizationRequest request = new AuthorizationRequest( @@ -90,9 +89,13 @@ public void testCanManageSecretsUnauthorized() throws Exception { AuthorizationResult result = Mockito.mock(AuthorizationResult.class); Mockito.when(result.getType()).thenReturn(AuthorizationResult.Type.DENY); - Mockito.when(mockAuthorizer.authorize(Mockito.eq(request))).thenReturn(result); + Mockito.when( + mockContext + .getOperationContext() + .authorize( + eq("MANAGE_SECRETS"), eq(new EntitySpec(Constants.SECRETS_ENTITY_NAME, "")))) + .thenReturn(result); - Mockito.when(mockContext.getAuthorizer()).thenReturn(mockAuthorizer); Mockito.when(mockContext.getActorUrn()).thenReturn("urn:li:corpuser:unauthorized"); assertFalse(IngestionAuthUtils.canManageSecrets(mockContext)); diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/source/ListIngestionSourceResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/source/ListIngestionSourceResolverTest.java index cd3b5c9dce47e8..05428788dc3c92 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/source/ListIngestionSourceResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/source/ListIngestionSourceResolverTest.java @@ -2,7 +2,6 @@ import static com.linkedin.datahub.graphql.resolvers.ingest.IngestTestUtils.*; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; import static org.testng.Assert.*; import com.google.common.collect.ImmutableMap; @@ -22,7 +21,6 @@ import com.linkedin.metadata.search.SearchResult; import com.linkedin.r2.RemoteInvocationException; import graphql.schema.DataFetchingEnvironment; -import io.datahubproject.metadata.context.OperationContext; import java.util.HashSet; import org.mockito.Mockito; import org.testng.annotations.Test; @@ -85,7 +83,7 @@ public void testGetSuccess() throws Exception { // Execute resolver QueryContext mockContext = getMockAllowContext(); - Mockito.when(mockContext.getOperationContext()).thenReturn(mock(OperationContext.class)); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); Mockito.when(mockEnv.getContext()).thenReturn(mockContext); diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/post/CreatePostResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/post/CreatePostResolverTest.java index 7f14193737e00e..8e160237d095bf 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/post/CreatePostResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/post/CreatePostResolverTest.java @@ -94,7 +94,7 @@ public void testCreatePost() throws Exception { input.setContent(content); when(_dataFetchingEnvironment.getArgument(eq("input"))).thenReturn(input); when(_postService.createPost( - any(), eq(PostType.HOME_PAGE_ANNOUNCEMENT.toString()), eq(postContentObj))) + any(), eq(PostType.HOME_PAGE_ANNOUNCEMENT.toString()), eq(postContentObj), any())) .thenReturn(true); assertTrue(_resolver.get(_dataFetchingEnvironment).join()); diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/query/CreateQueryResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/query/CreateQueryResolverTest.java index 034a8215c4a8ca..5617321c98e84f 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/query/CreateQueryResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/query/CreateQueryResolverTest.java @@ -2,15 +2,15 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import static org.testng.Assert.*; import com.datahub.authentication.Actor; import com.datahub.authentication.ActorType; import com.datahub.authentication.Authentication; -import com.datahub.authorization.AuthorizationRequest; import com.datahub.authorization.AuthorizationResult; import com.datahub.authorization.EntitySpec; -import com.datahub.plugins.auth.authorization.Authorizer; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.linkedin.common.AuditStamp; @@ -38,8 +38,10 @@ import com.linkedin.query.QuerySubject; import com.linkedin.query.QuerySubjectArray; import com.linkedin.query.QuerySubjects; +import com.linkedin.util.Pair; import graphql.schema.DataFetchingEnvironment; -import java.util.Optional; +import io.datahubproject.metadata.context.OperationContext; +import java.util.Map; import java.util.concurrent.CompletionException; import org.mockito.Mockito; import org.testng.annotations.Test; @@ -214,22 +216,7 @@ private QueryService initMockService() { private QueryContext getMockQueryContext(boolean allowEditEntityQueries) { QueryContext mockContext = Mockito.mock(QueryContext.class); Mockito.when(mockContext.getActorUrn()).thenReturn(TEST_ACTOR_URN.toString()); - - Authorizer mockAuthorizer = Mockito.mock(Authorizer.class); - - AuthorizationRequest editQueriesRequest = - new AuthorizationRequest( - TEST_ACTOR_URN.toString(), - PoliciesConfig.EDIT_QUERIES_PRIVILEGE.getType(), - Optional.of( - new EntitySpec(TEST_DATASET_URN.getEntityType(), TEST_DATASET_URN.toString()))); - - AuthorizationRequest editAllRequest = - new AuthorizationRequest( - TEST_ACTOR_URN.toString(), - PoliciesConfig.EDIT_ENTITY_PRIVILEGE.getType(), - Optional.of( - new EntitySpec(TEST_DATASET_URN.getEntityType(), TEST_DATASET_URN.toString()))); + when(mockContext.getOperationContext()).thenReturn(mock(OperationContext.class)); AuthorizationResult editQueriesResult = Mockito.mock(AuthorizationResult.class); Mockito.when(editQueriesResult.getType()) @@ -237,8 +224,6 @@ private QueryContext getMockQueryContext(boolean allowEditEntityQueries) { allowEditEntityQueries ? AuthorizationResult.Type.ALLOW : AuthorizationResult.Type.DENY); - Mockito.when(mockAuthorizer.authorize(Mockito.eq(editQueriesRequest))) - .thenReturn(editQueriesResult); AuthorizationResult editAllResult = Mockito.mock(AuthorizationResult.class); Mockito.when(editAllResult.getType()) @@ -246,9 +231,25 @@ private QueryContext getMockQueryContext(boolean allowEditEntityQueries) { allowEditEntityQueries ? AuthorizationResult.Type.ALLOW : AuthorizationResult.Type.DENY); - Mockito.when(mockAuthorizer.authorize(Mockito.eq(editAllRequest))).thenReturn(editAllResult); - Mockito.when(mockContext.getAuthorizer()).thenReturn(mockAuthorizer); + Map, AuthorizationResult> responses = + Map.of( + Pair.of( + PoliciesConfig.EDIT_QUERIES_PRIVILEGE.getType(), + new EntitySpec(TEST_DATASET_URN.getEntityType(), TEST_DATASET_URN.toString())), + editQueriesResult, + Pair.of( + PoliciesConfig.EDIT_ENTITY_PRIVILEGE.getType(), + new EntitySpec(TEST_DATASET_URN.getEntityType(), TEST_DATASET_URN.toString())), + editAllResult); + + when(mockContext.getOperationContext().authorize(any(), any())) + .thenAnswer( + args -> + responses.getOrDefault( + Pair.of(args.getArgument(0), args.getArgument(1)), + new AuthorizationResult(null, AuthorizationResult.Type.DENY, ""))); + Mockito.when(mockContext.getAuthentication()) .thenReturn(new Authentication(new Actor(ActorType.USER, TEST_ACTOR_URN.getId()), "creds")); return mockContext; diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/query/DeleteQueryResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/query/DeleteQueryResolverTest.java index 491f06e800d709..2045e0ee52d683 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/query/DeleteQueryResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/query/DeleteQueryResolverTest.java @@ -2,15 +2,15 @@ import static com.linkedin.datahub.graphql.TestUtils.*; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import static org.testng.Assert.*; import com.datahub.authentication.Actor; import com.datahub.authentication.ActorType; import com.datahub.authentication.Authentication; -import com.datahub.authorization.AuthorizationRequest; import com.datahub.authorization.AuthorizationResult; import com.datahub.authorization.EntitySpec; -import com.datahub.plugins.auth.authorization.Authorizer; import com.google.common.collect.ImmutableList; import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; @@ -20,8 +20,10 @@ import com.linkedin.query.QuerySubject; import com.linkedin.query.QuerySubjectArray; import com.linkedin.query.QuerySubjects; +import com.linkedin.util.Pair; import graphql.schema.DataFetchingEnvironment; -import java.util.Optional; +import io.datahubproject.metadata.context.OperationContext; +import java.util.Map; import java.util.concurrent.CompletionException; import org.mockito.Mockito; import org.testng.annotations.Test; @@ -115,24 +117,7 @@ private QueryContext getMockQueryContext(boolean allowEditEntityQueries) { QueryContext mockContext = Mockito.mock(QueryContext.class); Mockito.when(mockContext.getActorUrn()) .thenReturn(DeleteQueryResolverTest.TEST_ACTOR_URN.toString()); - - Authorizer mockAuthorizer = Mockito.mock(Authorizer.class); - - AuthorizationRequest editQueriesRequest = - new AuthorizationRequest( - DeleteQueryResolverTest.TEST_ACTOR_URN.toString(), - PoliciesConfig.EDIT_QUERIES_PRIVILEGE.getType(), - Optional.of( - new EntitySpec( - DeleteQueryResolverTest.TEST_DATASET_URN.getEntityType(), - DeleteQueryResolverTest.TEST_DATASET_URN.toString()))); - - AuthorizationRequest editAllRequest = - new AuthorizationRequest( - TEST_ACTOR_URN.toString(), - PoliciesConfig.EDIT_ENTITY_PRIVILEGE.getType(), - Optional.of( - new EntitySpec(TEST_DATASET_URN.getEntityType(), TEST_DATASET_URN.toString()))); + when(mockContext.getOperationContext()).thenReturn(mock(OperationContext.class)); AuthorizationResult editQueriesResult = Mockito.mock(AuthorizationResult.class); Mockito.when(editQueriesResult.getType()) @@ -140,8 +125,6 @@ private QueryContext getMockQueryContext(boolean allowEditEntityQueries) { allowEditEntityQueries ? AuthorizationResult.Type.ALLOW : AuthorizationResult.Type.DENY); - Mockito.when(mockAuthorizer.authorize(Mockito.eq(editQueriesRequest))) - .thenReturn(editQueriesResult); AuthorizationResult editAllResult = Mockito.mock(AuthorizationResult.class); Mockito.when(editAllResult.getType()) @@ -149,9 +132,25 @@ private QueryContext getMockQueryContext(boolean allowEditEntityQueries) { allowEditEntityQueries ? AuthorizationResult.Type.ALLOW : AuthorizationResult.Type.DENY); - Mockito.when(mockAuthorizer.authorize(Mockito.eq(editAllRequest))).thenReturn(editAllResult); - Mockito.when(mockContext.getAuthorizer()).thenReturn(mockAuthorizer); + Map, AuthorizationResult> responses = + Map.of( + Pair.of( + PoliciesConfig.EDIT_QUERIES_PRIVILEGE.getType(), + new EntitySpec(TEST_DATASET_URN.getEntityType(), TEST_DATASET_URN.toString())), + editQueriesResult, + Pair.of( + PoliciesConfig.EDIT_ENTITY_PRIVILEGE.getType(), + new EntitySpec(TEST_DATASET_URN.getEntityType(), TEST_DATASET_URN.toString())), + editAllResult); + + when(mockContext.getOperationContext().authorize(any(), any())) + .thenAnswer( + args -> + responses.getOrDefault( + Pair.of(args.getArgument(0), args.getArgument(1)), + new AuthorizationResult(null, AuthorizationResult.Type.DENY, ""))); + Mockito.when(mockContext.getAuthentication()) .thenReturn(new Authentication(new Actor(ActorType.USER, TEST_ACTOR_URN.getId()), "creds")); return mockContext; diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/query/ListQueriesResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/query/ListQueriesResolverTest.java index ee728b17e8c621..414c62b693b698 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/query/ListQueriesResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/query/ListQueriesResolverTest.java @@ -170,6 +170,6 @@ private Filter buildFilter(@Nullable QuerySource source, @Nullable String entity FilterOperator.EQUAL)); } criteria.setAnd(andConditions); - return ResolverUtils.buildFilter(Collections.emptyList(), ImmutableList.of(criteria), null); + return ResolverUtils.buildFilter(Collections.emptyList(), ImmutableList.of(criteria)); } } diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/query/UpdateQueryResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/query/UpdateQueryResolverTest.java index ce21ed99595660..8b81523b58d105 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/query/UpdateQueryResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/query/UpdateQueryResolverTest.java @@ -2,15 +2,15 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import static org.testng.Assert.*; import com.datahub.authentication.Actor; import com.datahub.authentication.ActorType; import com.datahub.authentication.Authentication; -import com.datahub.authorization.AuthorizationRequest; import com.datahub.authorization.AuthorizationResult; import com.datahub.authorization.EntitySpec; -import com.datahub.plugins.auth.authorization.Authorizer; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.linkedin.common.AuditStamp; @@ -38,8 +38,10 @@ import com.linkedin.query.QuerySubject; import com.linkedin.query.QuerySubjectArray; import com.linkedin.query.QuerySubjects; +import com.linkedin.util.Pair; import graphql.schema.DataFetchingEnvironment; -import java.util.Optional; +import io.datahubproject.metadata.context.OperationContext; +import java.util.Map; import java.util.concurrent.CompletionException; import org.mockito.Mockito; import org.testng.annotations.Test; @@ -212,36 +214,7 @@ private QueryService initMockService() { private QueryContext getMockQueryContext(boolean allowEditEntityQueries) { QueryContext mockContext = Mockito.mock(QueryContext.class); Mockito.when(mockContext.getActorUrn()).thenReturn(TEST_ACTOR_URN.toString()); - - Authorizer mockAuthorizer = Mockito.mock(Authorizer.class); - - AuthorizationRequest editQueriesRequest1 = - new AuthorizationRequest( - TEST_ACTOR_URN.toString(), - PoliciesConfig.EDIT_QUERIES_PRIVILEGE.getType(), - Optional.of( - new EntitySpec(TEST_DATASET_URN.getEntityType(), TEST_DATASET_URN.toString()))); - - AuthorizationRequest editAllRequest1 = - new AuthorizationRequest( - TEST_ACTOR_URN.toString(), - PoliciesConfig.EDIT_ENTITY_PRIVILEGE.getType(), - Optional.of( - new EntitySpec(TEST_DATASET_URN.getEntityType(), TEST_DATASET_URN.toString()))); - - AuthorizationRequest editQueriesRequest2 = - new AuthorizationRequest( - TEST_ACTOR_URN.toString(), - PoliciesConfig.EDIT_QUERIES_PRIVILEGE.getType(), - Optional.of( - new EntitySpec(TEST_DATASET_URN_2.getEntityType(), TEST_DATASET_URN_2.toString()))); - - AuthorizationRequest editAllRequest2 = - new AuthorizationRequest( - TEST_ACTOR_URN.toString(), - PoliciesConfig.EDIT_ENTITY_PRIVILEGE.getType(), - Optional.of( - new EntitySpec(TEST_DATASET_URN_2.getEntityType(), TEST_DATASET_URN_2.toString()))); + when(mockContext.getOperationContext()).thenReturn(mock(OperationContext.class)); AuthorizationResult editQueriesResult1 = Mockito.mock(AuthorizationResult.class); Mockito.when(editQueriesResult1.getType()) @@ -249,8 +222,6 @@ private QueryContext getMockQueryContext(boolean allowEditEntityQueries) { allowEditEntityQueries ? AuthorizationResult.Type.ALLOW : AuthorizationResult.Type.DENY); - Mockito.when(mockAuthorizer.authorize(Mockito.eq(editQueriesRequest1))) - .thenReturn(editQueriesResult1); AuthorizationResult editAllResult1 = Mockito.mock(AuthorizationResult.class); Mockito.when(editAllResult1.getType()) @@ -258,7 +229,6 @@ private QueryContext getMockQueryContext(boolean allowEditEntityQueries) { allowEditEntityQueries ? AuthorizationResult.Type.ALLOW : AuthorizationResult.Type.DENY); - Mockito.when(mockAuthorizer.authorize(Mockito.eq(editAllRequest1))).thenReturn(editAllResult1); AuthorizationResult editQueriesResult2 = Mockito.mock(AuthorizationResult.class); Mockito.when(editQueriesResult2.getType()) @@ -266,8 +236,6 @@ private QueryContext getMockQueryContext(boolean allowEditEntityQueries) { allowEditEntityQueries ? AuthorizationResult.Type.ALLOW : AuthorizationResult.Type.DENY); - Mockito.when(mockAuthorizer.authorize(Mockito.eq(editQueriesRequest2))) - .thenReturn(editQueriesResult2); AuthorizationResult editAllResult2 = Mockito.mock(AuthorizationResult.class); Mockito.when(editAllResult2.getType()) @@ -275,9 +243,35 @@ private QueryContext getMockQueryContext(boolean allowEditEntityQueries) { allowEditEntityQueries ? AuthorizationResult.Type.ALLOW : AuthorizationResult.Type.DENY); - Mockito.when(mockAuthorizer.authorize(Mockito.eq(editAllRequest2))).thenReturn(editAllResult2); - Mockito.when(mockContext.getAuthorizer()).thenReturn(mockAuthorizer); + Map, AuthorizationResult> responses = + Map.of( + Pair.of( + PoliciesConfig.EDIT_QUERIES_PRIVILEGE.getType(), + new EntitySpec(TEST_DATASET_URN.getEntityType(), TEST_DATASET_URN.toString())), + editQueriesResult1, + Pair.of( + PoliciesConfig.EDIT_ENTITY_PRIVILEGE.getType(), + new EntitySpec(TEST_DATASET_URN.getEntityType(), TEST_DATASET_URN.toString())), + editAllResult1, + Pair.of( + PoliciesConfig.EDIT_QUERIES_PRIVILEGE.getType(), + new EntitySpec( + TEST_DATASET_URN_2.getEntityType(), TEST_DATASET_URN_2.toString())), + editQueriesResult2, + Pair.of( + PoliciesConfig.EDIT_ENTITY_PRIVILEGE.getType(), + new EntitySpec( + TEST_DATASET_URN_2.getEntityType(), TEST_DATASET_URN_2.toString())), + editAllResult2); + + when(mockContext.getOperationContext().authorize(any(), any())) + .thenAnswer( + args -> + responses.getOrDefault( + Pair.of(args.getArgument(0), args.getArgument(1)), + new AuthorizationResult(null, AuthorizationResult.Type.DENY, ""))); + Mockito.when(mockContext.getAuthentication()) .thenReturn(new Authentication(new Actor(ActorType.USER, TEST_ACTOR_URN.getId()), "creds")); return mockContext; diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolverTest.java index d32eb9fcf120ca..129866bb0fa07a 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolverTest.java @@ -107,7 +107,7 @@ public static void testApplyViewBaseFilter() throws Exception { FormService mockFormService = Mockito.mock(FormService.class); ViewService mockService = initMockViewService(TEST_VIEW_URN, info); - Filter baseFilter = createFilter("baseField.keyword", "baseTest"); + Filter baseFilter = createFilter("baseField", "baseTest"); EntityClient mockClient = initMockEntityClient( diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolverTest.java index 30d6f2dc6f2836..86508f1fd2742b 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolverTest.java @@ -164,7 +164,7 @@ public static void testApplyViewBaseFilter() throws Exception { new CriterionArray( ImmutableList.of( new Criterion() - .setField("baseField.keyword") + .setField("baseField") .setValue("baseTest") .setCondition(Condition.EQUAL) .setNegated(false) diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpdateStructuredPropertyResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpdateStructuredPropertyResolverTest.java index 971a53de9473b5..b818bcfb7d7f4f 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpdateStructuredPropertyResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpdateStructuredPropertyResolverTest.java @@ -89,8 +89,8 @@ public void testGetFailure() throws Exception { assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); - // Validate that ingest was called, but that caused a failure - Mockito.verify(mockEntityClient, Mockito.times(1)) + // Validate that ingest was not called since there was a get failure before ingesting + Mockito.verify(mockEntityClient, Mockito.times(0)) .ingestProposal(any(), any(MetadataChangeProposal.class), Mockito.eq(false)); } diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/CreateViewResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/CreateViewResolverTest.java index 502188d4977a53..c009cf37c53971 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/CreateViewResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/CreateViewResolverTest.java @@ -110,7 +110,7 @@ public void testGetSuccess() throws Exception { ImmutableList.of( new Criterion() .setCondition(Condition.EQUAL) - .setField("test1.keyword") + .setField("test1") .setValue( "value1") // Unfortunate --- For // backwards compat. @@ -121,7 +121,7 @@ public void testGetSuccess() throws Exception { .setNegated(false), new Criterion() .setCondition(Condition.IN) - .setField("test2.keyword") + .setField("test2") .setValue( "value1") // Unfortunate --- For // backwards compat. diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/ListGlobalViewsResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/ListGlobalViewsResolverTest.java index a3b9e25e992259..b5c0531db792ba 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/ListGlobalViewsResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/ListGlobalViewsResolverTest.java @@ -56,7 +56,7 @@ public void testGetSuccessInput() throws Exception { new CriterionArray( ImmutableList.of( new Criterion() - .setField("type.keyword") + .setField("type") .setValue(DataHubViewType.GLOBAL.toString()) .setValues( new StringArray( diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/ListMyViewsResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/ListMyViewsResolverTest.java index 99b0e76976748e..85d24f9251eaa3 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/ListMyViewsResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/ListMyViewsResolverTest.java @@ -58,7 +58,7 @@ public void testGetSuccessInput1() throws Exception { new CriterionArray( ImmutableList.of( new Criterion() - .setField("createdBy.keyword") + .setField("createdBy") .setValue(TEST_USER.toString()) .setValues( new StringArray( @@ -67,7 +67,7 @@ public void testGetSuccessInput1() throws Exception { .setCondition(Condition.EQUAL) .setNegated(false), new Criterion() - .setField("type.keyword") + .setField("type") .setValue(DataHubViewType.GLOBAL.toString()) .setValues( new StringArray( @@ -124,7 +124,7 @@ public void testGetSuccessInput2() throws Exception { new CriterionArray( ImmutableList.of( new Criterion() - .setField("createdBy.keyword") + .setField("createdBy") .setValue(TEST_USER.toString()) .setValues( new StringArray( diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/UpdateViewResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/UpdateViewResolverTest.java index fe32bec01fd7ed..86a502b40b9364 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/UpdateViewResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/UpdateViewResolverTest.java @@ -111,7 +111,7 @@ public void testGetSuccessGlobalViewIsCreator() throws Exception { ImmutableList.of( new Criterion() .setCondition(Condition.EQUAL) - .setField("test1.keyword") + .setField("test1") .setValue( "value1") // Unfortunate --- For // backwards compat. @@ -122,7 +122,7 @@ public void testGetSuccessGlobalViewIsCreator() throws Exception { .setNegated(false), new Criterion() .setCondition(Condition.IN) - .setField("test2.keyword") + .setField("test2") .setValue( "value1") // Unfortunate --- For // backwards compat. @@ -175,7 +175,7 @@ public void testGetSuccessGlobalViewManageGlobalViews() throws Exception { ImmutableList.of( new Criterion() .setCondition(Condition.EQUAL) - .setField("test1.keyword") + .setField("test1") .setValue( "value1") // Unfortunate --- For // backwards compat. @@ -186,7 +186,7 @@ public void testGetSuccessGlobalViewManageGlobalViews() throws Exception { .setNegated(false), new Criterion() .setCondition(Condition.IN) - .setField("test2.keyword") + .setField("test2") .setValue( "value1") // Unfortunate --- For // backwards compat. diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/ViewUtilsTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/ViewUtilsTest.java index 701ddd84c173e7..d142be1321a5c3 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/ViewUtilsTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/ViewUtilsTest.java @@ -154,8 +154,7 @@ public void testMapDefinition() throws Exception { new StringArray( ImmutableList.of("value1", "value2"))) .setValue("value1") // Disgraceful - .setField( - "test1.keyword") // Consider whether we + .setField("test1") // Consider whether we // should NOT go through // the keyword mapping. .setCondition(Condition.IN), @@ -165,8 +164,7 @@ public void testMapDefinition() throws Exception { new StringArray( ImmutableList.of("value3", "value4"))) .setValue("value3") // Disgraceful - .setField( - "test2.keyword") // Consider whether we + .setField("test2") // Consider whether we // should NOT go through // the keyword mapping. .setCondition(Condition.CONTAIN)))))))); diff --git a/datahub-upgrade/build.gradle b/datahub-upgrade/build.gradle index b66371db73386f..5d814dd876679e 100644 --- a/datahub-upgrade/build.gradle +++ b/datahub-upgrade/build.gradle @@ -115,7 +115,10 @@ task run(type: Exec) { environment "ENTITY_REGISTRY_CONFIG_PATH", "../metadata-models/src/main/resources/entity-registry.yml" environment "ENABLE_STRUCTURED_PROPERTIES_SYSTEM_UPDATE", "true" environment "ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX", "true" - commandLine "java", "-jar", "-Dserver.port=8083", bootJar.getArchiveFile().get(), "-u", "SystemUpdate" + commandLine "java", + "-agentlib:jdwp=transport=dt_socket,address=5003,server=y,suspend=n", + "-jar", + "-Dserver.port=8083", bootJar.getArchiveFile().get(), "-u", "SystemUpdate" } docker { diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/Upgrade.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/Upgrade.java index d3aea2a3dac12a..d159622eb292b6 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/Upgrade.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/Upgrade.java @@ -1,7 +1,17 @@ package com.linkedin.datahub.upgrade; import com.google.common.collect.ImmutableList; +import com.linkedin.common.urn.Urn; +import com.linkedin.metadata.boot.BootstrapStep; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.upgrade.DataHubUpgradeResult; +import com.linkedin.upgrade.DataHubUpgradeState; +import io.datahubproject.metadata.context.OperationContext; import java.util.List; +import java.util.Map; +import java.util.Optional; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; /** Specification of an upgrade to be performed to the DataHub platform. */ public interface Upgrade { @@ -16,4 +26,18 @@ public interface Upgrade { default List cleanupSteps() { return ImmutableList.of(); } + + default Optional getUpgradeResult( + @Nonnull OperationContext opContext, Urn upgradeId, EntityService entityService) { + return BootstrapStep.getUpgradeResult(opContext, upgradeId, entityService); + } + + default void setUpgradeResult( + @Nonnull OperationContext opContext, + @Nonnull Urn upgradeId, + EntityService entityService, + @Nullable DataHubUpgradeState state, + @Nullable Map result) { + BootstrapStep.setUpgradeResult(opContext, upgradeId, entityService, state, result); + } } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/UpgradeCli.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/UpgradeCli.java index 6d10a7ed6b3b48..bff65fd03dd572 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/UpgradeCli.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/UpgradeCli.java @@ -9,6 +9,7 @@ import com.linkedin.datahub.upgrade.system.SystemUpdate; import com.linkedin.datahub.upgrade.system.SystemUpdateBlocking; import com.linkedin.datahub.upgrade.system.SystemUpdateNonBlocking; +import com.linkedin.upgrade.DataHubUpgradeState; import io.datahubproject.metadata.context.OperationContext; import java.util.List; import javax.inject.Inject; @@ -91,7 +92,7 @@ public void run(String... cmdLineArgs) { UpgradeResult result = _upgradeManager.execute(systemOperationContext, args.upgradeId.trim(), args.args); - if (UpgradeResult.Result.FAILED.equals(result.result())) { + if (DataHubUpgradeState.FAILED.equals(result.result())) { System.exit(1); } else { System.exit(0); diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/UpgradeContext.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/UpgradeContext.java index 73643175ab9c67..2b5598184ee6b9 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/UpgradeContext.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/UpgradeContext.java @@ -23,6 +23,6 @@ public interface UpgradeContext { /** Returns a map of argument to <>optional value, as delimited by an '=' character. */ Map> parsedArgs(); - /** Returns the operation context ffor the upgrade */ + /** Returns the operation context for the upgrade */ OperationContext opContext(); } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/UpgradeResult.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/UpgradeResult.java index 25dc758575fd16..57a01b4de413fa 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/UpgradeResult.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/UpgradeResult.java @@ -1,20 +1,12 @@ package com.linkedin.datahub.upgrade; +import com.linkedin.upgrade.DataHubUpgradeState; + /** Represents the result of executing an {@link Upgrade} */ public interface UpgradeResult { - /** The execution result. */ - enum Result { - /** Upgrade succeeded. */ - SUCCEEDED, - /** Upgrade failed. */ - FAILED, - /** Upgrade was aborted. */ - ABORTED - } - - /** Returns the {@link Result} of executing an {@link Upgrade} */ - Result result(); + /** Returns the {@link DataHubUpgradeState} of executing an {@link Upgrade} */ + DataHubUpgradeState result(); /** Returns the {@link UpgradeReport} associated with the completed {@link Upgrade}. */ UpgradeReport report(); diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/UpgradeStepResult.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/UpgradeStepResult.java index 04b3d4b8559e67..e88e3c75f86954 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/UpgradeStepResult.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/UpgradeStepResult.java @@ -1,18 +1,12 @@ package com.linkedin.datahub.upgrade; +import com.linkedin.upgrade.DataHubUpgradeState; + public interface UpgradeStepResult { /** Returns a string identifier associated with the step. */ String stepId(); - /** The outcome of the step execution. */ - enum Result { - /** The step succeeded. */ - SUCCEEDED, - /** The step failed. */ - FAILED - } - /** A control-flow action to perform as a result of the step execution. */ enum Action { /** Continue attempting the upgrade. */ @@ -24,7 +18,7 @@ enum Action { } /** Returns the result of executing the step, either success or failure. */ - Result result(); + DataHubUpgradeState result(); /** Returns the action to perform after executing the step, either continue or abort. */ default Action action() { diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/ClearGraphServiceStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/ClearGraphServiceStep.java index 393b5411599adc..357f1ffd663a9a 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/ClearGraphServiceStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/ClearGraphServiceStep.java @@ -6,6 +6,7 @@ import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult; import com.linkedin.datahub.upgrade.nocode.NoCodeUpgrade; import com.linkedin.metadata.graph.GraphService; +import com.linkedin.upgrade.DataHubUpgradeState; import java.util.function.Function; public class ClearGraphServiceStep implements UpgradeStep { @@ -49,9 +50,9 @@ public Function executable() { _graphService.clear(); } catch (Exception e) { context.report().addLine("Failed to clear graph indices", e); - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.FAILED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.FAILED); } - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.SUCCEEDED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.SUCCEEDED); }; } } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/ClearSearchServiceStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/ClearSearchServiceStep.java index 44592ecf92dbda..0ccd2029c93ac3 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/ClearSearchServiceStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/ClearSearchServiceStep.java @@ -6,6 +6,7 @@ import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult; import com.linkedin.datahub.upgrade.nocode.NoCodeUpgrade; import com.linkedin.metadata.search.EntitySearchService; +import com.linkedin.upgrade.DataHubUpgradeState; import java.util.function.Function; public class ClearSearchServiceStep implements UpgradeStep { @@ -48,9 +49,9 @@ public Function executable() { _entitySearchService.clear(context.opContext()); } catch (Exception e) { context.report().addLine("Failed to clear search service", e); - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.FAILED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.FAILED); } - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.SUCCEEDED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.SUCCEEDED); }; } } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/ClearSystemMetadataServiceStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/ClearSystemMetadataServiceStep.java index 889d2f0a79edf6..f0f2958e917330 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/ClearSystemMetadataServiceStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/ClearSystemMetadataServiceStep.java @@ -6,6 +6,7 @@ import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult; import com.linkedin.datahub.upgrade.nocode.NoCodeUpgrade; import com.linkedin.metadata.systemmetadata.SystemMetadataService; +import com.linkedin.upgrade.DataHubUpgradeState; import java.util.function.Function; public class ClearSystemMetadataServiceStep implements UpgradeStep { @@ -48,9 +49,9 @@ public Function executable() { _systemMetadataService.clear(); } catch (Exception e) { context.report().addLine("Failed to clear system metadata service", e); - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.FAILED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.FAILED); } - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.SUCCEEDED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.SUCCEEDED); }; } } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/GMSDisableWriteModeStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/GMSDisableWriteModeStep.java index a80adabc60e743..2e31d1eb62bd0c 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/GMSDisableWriteModeStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/GMSDisableWriteModeStep.java @@ -5,6 +5,7 @@ import com.linkedin.datahub.upgrade.UpgradeStepResult; import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult; import com.linkedin.entity.client.SystemEntityClient; +import com.linkedin.upgrade.DataHubUpgradeState; import java.util.function.Function; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -33,9 +34,9 @@ public Function executable() { } catch (Exception e) { log.error("Failed to turn write mode off in GMS", e); context.report().addLine("Failed to turn write mode off in GMS"); - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.FAILED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.FAILED); } - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.SUCCEEDED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.SUCCEEDED); }; } } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/GMSEnableWriteModeStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/GMSEnableWriteModeStep.java index a6b2942fabffd7..12bfccf1c6749a 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/GMSEnableWriteModeStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/GMSEnableWriteModeStep.java @@ -5,6 +5,7 @@ import com.linkedin.datahub.upgrade.UpgradeStepResult; import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult; import com.linkedin.entity.client.SystemEntityClient; +import com.linkedin.upgrade.DataHubUpgradeState; import java.util.function.Function; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -32,9 +33,9 @@ public Function executable() { } catch (Exception e) { log.error("Failed to turn write mode back on in GMS", e); context.report().addLine("Failed to turn write mode back on in GMS"); - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.FAILED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.FAILED); } - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.SUCCEEDED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.SUCCEEDED); }; } } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/GMSQualificationStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/GMSQualificationStep.java index 4e7447cb1e2cb6..9ee8244ee370d0 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/GMSQualificationStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/GMSQualificationStep.java @@ -10,6 +10,7 @@ import com.linkedin.datahub.upgrade.UpgradeStep; import com.linkedin.datahub.upgrade.UpgradeStepResult; import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult; +import com.linkedin.upgrade.DataHubUpgradeState; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; @@ -97,7 +98,7 @@ public Function executable() { StreamReadConstraints.builder().maxStringLength(maxSize).build()); JsonNode configJson = mapper.readTree(responseString); if (isEligible((ObjectNode) configJson)) { - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.SUCCEEDED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.SUCCEEDED); } else { context .report() @@ -105,7 +106,7 @@ public Function executable() { String.format( "Failed to qualify GMS. It is not running on the latest version." + "Re-run GMS on the latest datahub release")); - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.FAILED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.FAILED); } } catch (Exception e) { e.printStackTrace(); @@ -117,7 +118,7 @@ public Function executable() { + "at %s://host %s port %s. Make sure GMS is on the latest version " + "and is running at that host before starting the migration.", gmsProtocol, gmsHost, gmsPort)); - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.FAILED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.FAILED); } }; } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/BackfillDataProcessInstancesConfig.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/BackfillDataProcessInstancesConfig.java new file mode 100644 index 00000000000000..bc55ad38765ed5 --- /dev/null +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/BackfillDataProcessInstancesConfig.java @@ -0,0 +1,43 @@ +package com.linkedin.datahub.upgrade.config; + +import com.linkedin.datahub.upgrade.system.NonBlockingSystemUpgrade; +import com.linkedin.datahub.upgrade.system.dataprocessinstances.BackfillDataProcessInstances; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.search.elasticsearch.ElasticSearchService; +import io.datahubproject.metadata.context.OperationContext; +import org.opensearch.client.RestHighLevelClient; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Conditional; +import org.springframework.context.annotation.Configuration; + +@Configuration +@Conditional(SystemUpdateCondition.NonBlockingSystemUpdateCondition.class) +public class BackfillDataProcessInstancesConfig { + + @Bean + public NonBlockingSystemUpgrade backfillProcessInstancesHasRunEvents( + final OperationContext opContext, + EntityService entityService, + ElasticSearchService elasticSearchService, + RestHighLevelClient restHighLevelClient, + @Value("${systemUpdate.processInstanceHasRunEvents.enabled}") final boolean enabled, + @Value("${systemUpdate.processInstanceHasRunEvents.reprocess.enabled}") + boolean reprocessEnabled, + @Value("${systemUpdate.processInstanceHasRunEvents.batchSize}") final Integer batchSize, + @Value("${systemUpdate.processInstanceHasRunEvents.delayMs}") final Integer delayMs, + @Value("${systemUpdate.processInstanceHasRunEvents.totalDays}") Integer totalDays, + @Value("${systemUpdate.processInstanceHasRunEvents.windowDays}") Integer windowDays) { + return new BackfillDataProcessInstances( + opContext, + entityService, + elasticSearchService, + restHighLevelClient, + enabled, + reprocessEnabled, + batchSize, + delayMs, + totalDays, + windowDays); + } +} diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/SchemaFieldsConfig.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/SchemaFieldsConfig.java new file mode 100644 index 00000000000000..5630379c566183 --- /dev/null +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/SchemaFieldsConfig.java @@ -0,0 +1,56 @@ +package com.linkedin.datahub.upgrade.config; + +import com.linkedin.datahub.upgrade.system.NonBlockingSystemUpgrade; +import com.linkedin.datahub.upgrade.system.schemafield.GenerateSchemaFieldsFromSchemaMetadata; +import com.linkedin.datahub.upgrade.system.schemafield.MigrateSchemaFieldDocIds; +import com.linkedin.gms.factory.search.BaseElasticSearchComponentsFactory; +import com.linkedin.metadata.entity.AspectDao; +import com.linkedin.metadata.entity.EntityService; +import io.datahubproject.metadata.context.OperationContext; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Conditional; +import org.springframework.context.annotation.Configuration; + +@Configuration +@Conditional(SystemUpdateCondition.NonBlockingSystemUpdateCondition.class) +public class SchemaFieldsConfig { + + @Bean + public NonBlockingSystemUpgrade schemaFieldsFromSchemaMetadata( + @Qualifier("systemOperationContext") final OperationContext opContext, + final EntityService entityService, + final AspectDao aspectDao, + // SYSTEM_UPDATE_SCHEMA_FIELDS_FROM_SCHEMA_METADATA_ENABLED + @Value("${systemUpdate.schemaFieldsFromSchemaMetadata.enabled}") final boolean enabled, + // SYSTEM_UPDATE_SCHEMA_FIELDS_FROM_SCHEMA_METADATA_BATCH_SIZE + @Value("${systemUpdate.schemaFieldsFromSchemaMetadata.batchSize}") final Integer batchSize, + // SYSTEM_UPDATE_SCHEMA_FIELDS_FROM_SCHEMA_METADATA_DELAY_MS + @Value("${systemUpdate.schemaFieldsFromSchemaMetadata.delayMs}") final Integer delayMs, + // SYSTEM_UPDATE_SCHEMA_FIELDS_FROM_SCHEMA_METADATA_LIMIT + @Value("${systemUpdate.schemaFieldsFromSchemaMetadata.limit}") final Integer limit) { + return new GenerateSchemaFieldsFromSchemaMetadata( + opContext, entityService, aspectDao, enabled, batchSize, delayMs, limit); + } + + @Bean + public NonBlockingSystemUpgrade schemaFieldsDocIds( + @Qualifier("systemOperationContext") final OperationContext opContext, + @Qualifier("baseElasticSearchComponents") + final BaseElasticSearchComponentsFactory.BaseElasticSearchComponents components, + final EntityService entityService, + // ELASTICSEARCH_INDEX_DOC_IDS_SCHEMA_FIELD_HASH_ID_ENABLED + @Value("${elasticsearch.index.docIds.schemaField.hashIdEnabled}") final boolean hashEnabled, + // SYSTEM_UPDATE_SCHEMA_FIELDS_DOC_IDS_ENABLED + @Value("${systemUpdate.schemaFieldsDocIds.enabled}") final boolean enabled, + // SYSTEM_UPDATE_SCHEMA_FIELDS_DOC_IDS_BATCH_SIZE + @Value("${systemUpdate.schemaFieldsDocIds.batchSize}") final Integer batchSize, + // SYSTEM_UPDATE_SCHEMA_FIELDS_DOC_IDS_DELAY_MS + @Value("${systemUpdate.schemaFieldsDocIds.delayMs}") final Integer delayMs, + // SYSTEM_UPDATE_SCHEMA_FIELDS_DOC_IDS_LIMIT + @Value("${systemUpdate.schemaFieldsDocIds.limit}") final Integer limit) { + return new MigrateSchemaFieldDocIds( + opContext, components, entityService, enabled && hashEnabled, batchSize, delayMs, limit); + } +} diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/SystemUpdateConfig.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/SystemUpdateConfig.java index cac9b5f9483d41..dea98c5cbcb132 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/SystemUpdateConfig.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/SystemUpdateConfig.java @@ -1,5 +1,6 @@ package com.linkedin.datahub.upgrade.config; +import com.datahub.authentication.Authentication; import com.linkedin.datahub.upgrade.system.BlockingSystemUpgrade; import com.linkedin.datahub.upgrade.system.NonBlockingSystemUpgrade; import com.linkedin.datahub.upgrade.system.SystemUpdate; @@ -10,13 +11,25 @@ import com.linkedin.gms.factory.kafka.DataHubKafkaProducerFactory; import com.linkedin.gms.factory.kafka.common.TopicConventionFactory; import com.linkedin.gms.factory.kafka.schemaregistry.InternalSchemaRegistryFactory; -import com.linkedin.gms.factory.kafka.schemaregistry.SchemaRegistryConfig; +import com.linkedin.gms.factory.search.BaseElasticSearchComponentsFactory; +import com.linkedin.metadata.aspect.GraphRetriever; import com.linkedin.metadata.config.kafka.KafkaConfiguration; import com.linkedin.metadata.dao.producer.KafkaEventProducer; import com.linkedin.metadata.dao.producer.KafkaHealthChecker; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.entity.EntityServiceAspectRetriever; +import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.metadata.search.SearchService; +import com.linkedin.metadata.search.SearchServiceSearchRetriever; import com.linkedin.metadata.version.GitVersion; import com.linkedin.mxe.TopicConvention; +import io.datahubproject.metadata.context.OperationContext; +import io.datahubproject.metadata.context.OperationContextConfig; +import io.datahubproject.metadata.context.RetrieverContext; +import io.datahubproject.metadata.context.ServicesRegistryContext; +import io.datahubproject.metadata.services.RestrictedService; import java.util.List; +import javax.annotation.Nonnull; import lombok.extern.slf4j.Slf4j; import org.apache.avro.generic.IndexedRecord; import org.apache.kafka.clients.producer.KafkaProducer; @@ -84,7 +97,8 @@ public DataHubStartupStep dataHubStartupStep( protected KafkaEventProducer duheKafkaEventProducer( @Qualifier("configurationProvider") ConfigurationProvider provider, KafkaProperties properties, - @Qualifier("duheSchemaRegistryConfig") SchemaRegistryConfig duheSchemaRegistryConfig) { + @Qualifier("duheSchemaRegistryConfig") + KafkaConfiguration.SerDeKeyValueConfig duheSchemaRegistryConfig) { KafkaConfiguration kafkaConfiguration = provider.getKafka(); Producer producer = new KafkaProducer<>( @@ -116,8 +130,51 @@ protected KafkaEventProducer kafkaEventProducer( @ConditionalOnProperty( name = "kafka.schemaRegistry.type", havingValue = InternalSchemaRegistryFactory.TYPE) - protected SchemaRegistryConfig schemaRegistryConfig( - @Qualifier("duheSchemaRegistryConfig") SchemaRegistryConfig duheSchemaRegistryConfig) { + protected KafkaConfiguration.SerDeKeyValueConfig schemaRegistryConfig( + @Qualifier("duheSchemaRegistryConfig") + KafkaConfiguration.SerDeKeyValueConfig duheSchemaRegistryConfig) { return duheSchemaRegistryConfig; } + + @Primary + @Nonnull + @Bean(name = "systemOperationContext") + protected OperationContext javaSystemOperationContext( + @Nonnull @Qualifier("systemAuthentication") final Authentication systemAuthentication, + @Nonnull final OperationContextConfig operationContextConfig, + @Nonnull final EntityRegistry entityRegistry, + @Nonnull final EntityService entityService, + @Nonnull final RestrictedService restrictedService, + @Nonnull final GraphRetriever graphRetriever, + @Nonnull final SearchService searchService, + @Qualifier("baseElasticSearchComponents") + BaseElasticSearchComponentsFactory.BaseElasticSearchComponents components) { + + EntityServiceAspectRetriever entityServiceAspectRetriever = + EntityServiceAspectRetriever.builder() + .entityRegistry(entityRegistry) + .entityService(entityService) + .build(); + + SearchServiceSearchRetriever searchServiceSearchRetriever = + SearchServiceSearchRetriever.builder().searchService(searchService).build(); + + OperationContext systemOperationContext = + OperationContext.asSystem( + operationContextConfig, + systemAuthentication, + entityServiceAspectRetriever.getEntityRegistry(), + ServicesRegistryContext.builder().restrictedService(restrictedService).build(), + components.getIndexConvention(), + RetrieverContext.builder() + .aspectRetriever(entityServiceAspectRetriever) + .graphRetriever(graphRetriever) + .searchRetriever(searchServiceSearchRetriever) + .build()); + + entityServiceAspectRetriever.setSystemOperationContext(systemOperationContext); + searchServiceSearchRetriever.setSystemOperationContext(systemOperationContext); + + return systemOperationContext; + } } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/impl/DefaultUpgradeManager.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/impl/DefaultUpgradeManager.java index 240ec9f7bb2fed..27ba6abbc5ba93 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/impl/DefaultUpgradeManager.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/impl/DefaultUpgradeManager.java @@ -11,6 +11,7 @@ import com.linkedin.datahub.upgrade.UpgradeStep; import com.linkedin.datahub.upgrade.UpgradeStepResult; import com.linkedin.metadata.utils.metrics.MetricUtils; +import com.linkedin.upgrade.DataHubUpgradeState; import io.datahubproject.metadata.context.OperationContext; import java.util.ArrayList; import java.util.HashMap; @@ -85,11 +86,11 @@ private UpgradeResult executeInternal(UpgradeContext context) { String.format( "Step with id %s requested an abort of the in-progress update. Aborting the upgrade...", step.id())); - return new DefaultUpgradeResult(UpgradeResult.Result.ABORTED, upgradeReport); + return new DefaultUpgradeResult(DataHubUpgradeState.ABORTED, upgradeReport); } // Handle Results - if (UpgradeStepResult.Result.FAILED.equals(stepResult.result())) { + if (DataHubUpgradeState.FAILED.equals(stepResult.result())) { if (step.isOptional()) { upgradeReport.addLine( String.format( @@ -104,7 +105,7 @@ private UpgradeResult executeInternal(UpgradeContext context) { "Failed Step %s/%s: %s. Failed after %s retries.", i + 1, steps.size(), step.id(), step.retryCount())); upgradeReport.addLine(String.format("Exiting upgrade %s with failure.", upgrade.id())); - return new DefaultUpgradeResult(UpgradeResult.Result.FAILED, upgradeReport); + return new DefaultUpgradeResult(DataHubUpgradeState.FAILED, upgradeReport); } upgradeReport.addLine( @@ -113,7 +114,7 @@ private UpgradeResult executeInternal(UpgradeContext context) { upgradeReport.addLine( String.format("Success! Completed upgrade with id %s successfully.", upgrade.id())); - return new DefaultUpgradeResult(UpgradeResult.Result.SUCCEEDED, upgradeReport); + return new DefaultUpgradeResult(DataHubUpgradeState.SUCCEEDED, upgradeReport); } private UpgradeStepResult executeStepInternal(UpgradeContext context, UpgradeStep step) { @@ -130,14 +131,14 @@ private UpgradeStepResult executeStepInternal(UpgradeContext context, UpgradeSte if (result == null) { // Failed to even retrieve a result. Create a default failure result. - result = new DefaultUpgradeStepResult(step.id(), UpgradeStepResult.Result.FAILED); + result = new DefaultUpgradeStepResult(step.id(), DataHubUpgradeState.FAILED); context .report() .addLine(String.format("Retrying %s more times...", maxAttempts - (i + 1))); MetricUtils.counter(MetricRegistry.name(step.id(), "retry")).inc(); } - if (UpgradeStepResult.Result.SUCCEEDED.equals(result.result())) { + if (DataHubUpgradeState.SUCCEEDED.equals(result.result())) { MetricUtils.counter(MetricRegistry.name(step.id(), "succeeded")).inc(); break; } @@ -149,7 +150,7 @@ private UpgradeStepResult executeStepInternal(UpgradeContext context, UpgradeSte String.format( "Caught exception during attempt %s of Step with id %s: %s", i, step.id(), e)); MetricUtils.counter(MetricRegistry.name(step.id(), "failed")).inc(); - result = new DefaultUpgradeStepResult(step.id(), UpgradeStepResult.Result.FAILED); + result = new DefaultUpgradeStepResult(step.id(), DataHubUpgradeState.FAILED); context.report().addLine(String.format("Retrying %s more times...", maxAttempts - (i + 1))); } } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/impl/DefaultUpgradeResult.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/impl/DefaultUpgradeResult.java index cf0e7221b406b0..5131cadf5d3fa7 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/impl/DefaultUpgradeResult.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/impl/DefaultUpgradeResult.java @@ -2,19 +2,20 @@ import com.linkedin.datahub.upgrade.UpgradeReport; import com.linkedin.datahub.upgrade.UpgradeResult; +import com.linkedin.upgrade.DataHubUpgradeState; public class DefaultUpgradeResult implements UpgradeResult { - private final Result _result; + private final DataHubUpgradeState _result; private final UpgradeReport _report; - DefaultUpgradeResult(Result result, UpgradeReport report) { + DefaultUpgradeResult(DataHubUpgradeState result, UpgradeReport report) { _result = result; _report = report; } @Override - public Result result() { + public DataHubUpgradeState result() { return _result; } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/impl/DefaultUpgradeStepResult.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/impl/DefaultUpgradeStepResult.java index e11eaf89bfc8d2..cab0708fa52249 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/impl/DefaultUpgradeStepResult.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/impl/DefaultUpgradeStepResult.java @@ -1,18 +1,19 @@ package com.linkedin.datahub.upgrade.impl; import com.linkedin.datahub.upgrade.UpgradeStepResult; +import com.linkedin.upgrade.DataHubUpgradeState; public class DefaultUpgradeStepResult implements UpgradeStepResult { private final String _stepId; - private final Result _result; + private final DataHubUpgradeState _result; private final Action _action; - public DefaultUpgradeStepResult(String stepId, Result result) { + public DefaultUpgradeStepResult(String stepId, DataHubUpgradeState result) { this(stepId, result, Action.CONTINUE); } - public DefaultUpgradeStepResult(String stepId, Result result, Action action) { + public DefaultUpgradeStepResult(String stepId, DataHubUpgradeState result, Action action) { _stepId = stepId; _result = result; _action = action; @@ -24,7 +25,7 @@ public String stepId() { } @Override - public Result result() { + public DataHubUpgradeState result() { return _result; } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/CreateAspectTableStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/CreateAspectTableStep.java index 3b3098f43c4734..63b319d943a803 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/CreateAspectTableStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/CreateAspectTableStep.java @@ -4,6 +4,7 @@ import com.linkedin.datahub.upgrade.UpgradeStep; import com.linkedin.datahub.upgrade.UpgradeStepResult; import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult; +import com.linkedin.upgrade.DataHubUpgradeState; import io.ebean.Database; import java.util.function.Function; @@ -76,12 +77,12 @@ public Function executable() { } try { - _server.execute(_server.createSqlUpdate(sqlUpdateStr)); + _server.execute(_server.sqlUpdate(sqlUpdateStr)); } catch (Exception e) { context.report().addLine("Failed to create table metadata_aspect_v2", e); - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.FAILED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.FAILED); } - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.SUCCEEDED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.SUCCEEDED); }; } } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/DataMigrationStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/DataMigrationStep.java index 94bf97f3c9c9e7..ecff2e3ee6e18b 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/DataMigrationStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/DataMigrationStep.java @@ -18,6 +18,7 @@ import com.linkedin.metadata.models.EntitySpec; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.utils.PegasusUtils; +import com.linkedin.upgrade.DataHubUpgradeState; import com.linkedin.util.Pair; import io.ebean.Database; import io.ebean.PagedList; @@ -100,7 +101,7 @@ public Function executable() { "Failed to convert aspect with name %s into a RecordTemplate class", oldAspectName), e); - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.FAILED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.FAILED); } // 2. Extract an Entity type from the entity Urn @@ -127,7 +128,7 @@ public Function executable() { String.format( "Failed to find Entity with name %s in Entity Registry", entityName), e); - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.FAILED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.FAILED); } // 4. Extract new aspect name from Aspect schema @@ -142,7 +143,7 @@ public Function executable() { "Failed to retrieve @Aspect name from schema %s, urn %s", aspectRecord.schema().getFullName(), entityName), e); - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.FAILED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.FAILED); } // 5. Verify that the aspect is a valid aspect associated with the entity @@ -157,7 +158,7 @@ public Function executable() { "Failed to find aspect spec with name %s associated with entity named %s", newAspectName, entityName), e); - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.FAILED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.FAILED); } // 6. Write the row back using the EntityService @@ -214,9 +215,9 @@ public Function executable() { String.format( "Number of rows migrated %s does not equal the number of input rows %s...", totalRowsMigrated, rowCount)); - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.FAILED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.FAILED); } - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.SUCCEEDED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.SUCCEEDED); }; } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/RemoveAspectV2TableStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/RemoveAspectV2TableStep.java index 6180573d902d22..39346fa09d2319 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/RemoveAspectV2TableStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/RemoveAspectV2TableStep.java @@ -4,6 +4,7 @@ import com.linkedin.datahub.upgrade.UpgradeStep; import com.linkedin.datahub.upgrade.UpgradeStepResult; import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult; +import com.linkedin.upgrade.DataHubUpgradeState; import io.ebean.Database; import java.util.function.Function; @@ -26,7 +27,7 @@ public Function executable() { return (context) -> { context.report().addLine("Cleanup requested. Dropping metadata_aspect_v2"); _server.execute(_server.sqlUpdate("DROP TABLE IF EXISTS metadata_aspect_v2")); - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.SUCCEEDED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.SUCCEEDED); }; } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/UpgradeQualificationStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/UpgradeQualificationStep.java index d22af9d2924003..33de9f842a3aed 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/UpgradeQualificationStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/UpgradeQualificationStep.java @@ -5,6 +5,7 @@ import com.linkedin.datahub.upgrade.UpgradeStepResult; import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult; import com.linkedin.metadata.entity.ebean.AspectStorageValidationUtil; +import com.linkedin.upgrade.DataHubUpgradeState; import io.ebean.Database; import java.util.function.Function; @@ -31,22 +32,22 @@ public Function executable() { return (context) -> { if (context.parsedArgs().containsKey(NoCodeUpgrade.FORCE_UPGRADE_ARG_NAME)) { context.report().addLine("Forced upgrade detected. Proceeding with upgrade..."); - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.SUCCEEDED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.SUCCEEDED); } try { if (isQualified(_server, context)) { // Qualified. context.report().addLine("Found qualified upgrade candidate. Proceeding with upgrade..."); - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.SUCCEEDED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.SUCCEEDED); } // Unqualified (Table already exists) context.report().addLine("Failed to qualify upgrade candidate. Aborting the upgrade..."); return new DefaultUpgradeStepResult( - id(), UpgradeStepResult.Result.SUCCEEDED, UpgradeStepResult.Action.ABORT); + id(), DataHubUpgradeState.SUCCEEDED, UpgradeStepResult.Action.ABORT); } catch (Exception e) { context.report().addLine("Failed to check if metadata_aspect_v2 table exists", e); - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.FAILED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.FAILED); } }; } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocodecleanup/DeleteAspectTableStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocodecleanup/DeleteAspectTableStep.java index ba0a0124545e9d..f310c65375fb3b 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocodecleanup/DeleteAspectTableStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocodecleanup/DeleteAspectTableStep.java @@ -4,6 +4,7 @@ import com.linkedin.datahub.upgrade.UpgradeStep; import com.linkedin.datahub.upgrade.UpgradeStepResult; import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult; +import com.linkedin.upgrade.DataHubUpgradeState; import io.ebean.Database; import java.util.function.Function; @@ -33,9 +34,9 @@ public Function executable() { _server.execute(_server.sqlUpdate("DROP TABLE IF EXISTS metadata_aspect;")); } catch (Exception e) { context.report().addLine("Failed to delete data from legacy table metadata_aspect", e); - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.FAILED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.FAILED); } - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.SUCCEEDED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.SUCCEEDED); }; } } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocodecleanup/DeleteLegacyGraphRelationshipsStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocodecleanup/DeleteLegacyGraphRelationshipsStep.java index 5066e05f8bf5a1..47da1da8396430 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocodecleanup/DeleteLegacyGraphRelationshipsStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocodecleanup/DeleteLegacyGraphRelationshipsStep.java @@ -6,6 +6,7 @@ import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult; import com.linkedin.metadata.graph.GraphService; import com.linkedin.metadata.graph.neo4j.Neo4jGraphService; +import com.linkedin.upgrade.DataHubUpgradeState; import java.util.function.Function; import lombok.extern.slf4j.Slf4j; @@ -42,9 +43,9 @@ public Function executable() { } } catch (Exception e) { context.report().addLine("Failed to delete legacy data from graph", e); - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.FAILED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.FAILED); } - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.SUCCEEDED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.SUCCEEDED); }; } } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocodecleanup/DeleteLegacySearchIndicesStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocodecleanup/DeleteLegacySearchIndicesStep.java index 05656373377b93..33ace43fee8b2a 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocodecleanup/DeleteLegacySearchIndicesStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocodecleanup/DeleteLegacySearchIndicesStep.java @@ -5,6 +5,7 @@ import com.linkedin.datahub.upgrade.UpgradeStepResult; import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult; import com.linkedin.metadata.utils.elasticsearch.IndexConvention; +import com.linkedin.upgrade.DataHubUpgradeState; import java.util.function.Function; import lombok.RequiredArgsConstructor; import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; @@ -43,9 +44,9 @@ public Function executable() { _searchClient.indices().delete(request, RequestOptions.DEFAULT); } catch (Exception e) { context.report().addLine("Failed to delete legacy search index: %s", e); - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.FAILED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.FAILED); } - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.SUCCEEDED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.SUCCEEDED); }; } } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocodecleanup/NoCodeUpgradeQualificationStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocodecleanup/NoCodeUpgradeQualificationStep.java index 15c7584532e2ca..d8284fe47f9390 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocodecleanup/NoCodeUpgradeQualificationStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocodecleanup/NoCodeUpgradeQualificationStep.java @@ -5,6 +5,7 @@ import com.linkedin.datahub.upgrade.UpgradeStepResult; import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult; import com.linkedin.metadata.entity.ebean.AspectStorageValidationUtil; +import com.linkedin.upgrade.DataHubUpgradeState; import io.ebean.Database; import java.util.function.Function; @@ -36,15 +37,15 @@ public Function executable() { .report() .addLine("You have not successfully migrated yet. Aborting the cleanup..."); return new DefaultUpgradeStepResult( - id(), UpgradeStepResult.Result.SUCCEEDED, UpgradeStepResult.Action.ABORT); + id(), DataHubUpgradeState.SUCCEEDED, UpgradeStepResult.Action.ABORT); } else { // Qualified. context.report().addLine("Found qualified upgrade candidate. Proceeding with upgrade..."); - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.SUCCEEDED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.SUCCEEDED); } } catch (Exception e) { context.report().addLine("Failed to check if metadata_aspect_v2 table exists: %s", e); - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.FAILED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.FAILED); } }; } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/removeunknownaspects/RemoveClientIdAspectStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/removeunknownaspects/RemoveClientIdAspectStep.java index 6054599aa843c1..16b78a8dc8c6b3 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/removeunknownaspects/RemoveClientIdAspectStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/removeunknownaspects/RemoveClientIdAspectStep.java @@ -6,6 +6,7 @@ import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult; import com.linkedin.gms.factory.telemetry.TelemetryUtils; import com.linkedin.metadata.entity.EntityService; +import com.linkedin.upgrade.DataHubUpgradeState; import java.util.HashMap; import java.util.function.Function; import lombok.RequiredArgsConstructor; @@ -38,8 +39,7 @@ public Function executable() { INVALID_CLIENT_ID_ASPECT, new HashMap<>(), true); - return (UpgradeStepResult) - new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.SUCCEEDED); + return (UpgradeStepResult) new DefaultUpgradeStepResult(id(), DataHubUpgradeState.SUCCEEDED); }; } } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restorebackup/ClearAspectV2TableStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restorebackup/ClearAspectV2TableStep.java index addf6dcb89c1ae..febc1bec6fe266 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restorebackup/ClearAspectV2TableStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restorebackup/ClearAspectV2TableStep.java @@ -5,6 +5,7 @@ import com.linkedin.datahub.upgrade.UpgradeStepResult; import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult; import com.linkedin.metadata.entity.ebean.EbeanAspectV2; +import com.linkedin.upgrade.DataHubUpgradeState; import io.ebean.Database; import java.util.function.Function; @@ -26,7 +27,7 @@ public String id() { public Function executable() { return (context) -> { _server.find(EbeanAspectV2.class).delete(); - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.SUCCEEDED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.SUCCEEDED); }; } } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restorebackup/RestoreStorageStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restorebackup/RestoreStorageStep.java index eb0b24acc1ac37..4d53b603c1eaff 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restorebackup/RestoreStorageStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restorebackup/RestoreStorageStep.java @@ -20,6 +20,7 @@ import com.linkedin.metadata.models.AspectSpec; import com.linkedin.metadata.models.EntitySpec; import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.upgrade.DataHubUpgradeState; import com.linkedin.util.Pair; import java.lang.reflect.InvocationTargetException; import java.net.URISyntaxException; @@ -89,7 +90,7 @@ public Function executable() { context.report().addLine("BACKUP_READER: " + backupReaderName.toString()); if (!backupReaderName.isPresent() || !_backupReaders.containsKey(backupReaderName.get())) { context.report().addLine("BACKUP_READER is not set or is not valid"); - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.FAILED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.FAILED); } Class> clazz = @@ -134,7 +135,7 @@ public Function executable() { } context.report().addLine(String.format("Added %d rows to the aspect v2 table", numRows)); - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.SUCCEEDED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.SUCCEEDED); }; } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restoreindices/SendMAEStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restoreindices/SendMAEStep.java index 77d988f3176f29..8e62db444a5655 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restoreindices/SendMAEStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restoreindices/SendMAEStep.java @@ -10,6 +10,7 @@ import com.linkedin.metadata.entity.ebean.EbeanAspectV2; import com.linkedin.metadata.entity.restoreindices.RestoreIndicesArgs; import com.linkedin.metadata.entity.restoreindices.RestoreIndicesResult; +import com.linkedin.upgrade.DataHubUpgradeState; import io.ebean.Database; import io.ebean.ExpressionList; import java.util.ArrayList; @@ -188,7 +189,7 @@ public Function executable() { context.report().addLine(String.format("Rows processed this loop %d", rowsProcessed)); start += args.batchSize; } catch (InterruptedException | ExecutionException e) { - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.FAILED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.FAILED); } } } else { @@ -219,7 +220,7 @@ public Function executable() { "Failed to send MAEs for %d rows (%.2f%% of total).", rowCount - finalJobResult.rowsMigrated, percentFailed)); } - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.SUCCEEDED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.SUCCEEDED); }; } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/AbstractMCLStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/AbstractMCLStep.java index 27e98259c8beb5..6c70aee88675c5 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/AbstractMCLStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/AbstractMCLStep.java @@ -16,6 +16,7 @@ import com.linkedin.metadata.entity.ebean.PartitionedStream; import com.linkedin.metadata.entity.restoreindices.RestoreIndicesArgs; import com.linkedin.metadata.utils.AuditStampUtils; +import com.linkedin.upgrade.DataHubUpgradeState; import com.linkedin.util.Pair; import io.datahubproject.metadata.context.OperationContext; import java.util.List; @@ -134,7 +135,7 @@ public Function executable() { BootstrapStep.setUpgradeResult(opContext, getUpgradeIdUrn(), entityService); context.report().addLine("State updated: " + getUpgradeIdUrn()); - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.SUCCEEDED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.SUCCEEDED); }; } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/browsepaths/BackfillBrowsePathsV2Step.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/browsepaths/BackfillBrowsePathsV2Step.java index a1d559d05ad2fb..e6213a164febf7 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/browsepaths/BackfillBrowsePathsV2Step.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/browsepaths/BackfillBrowsePathsV2Step.java @@ -30,6 +30,7 @@ import com.linkedin.metadata.search.SearchService; import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.upgrade.DataHubUpgradeState; import io.datahubproject.metadata.context.OperationContext; import java.util.Set; import java.util.function.Function; @@ -98,7 +99,7 @@ public Function executable() { BootstrapStep.setUpgradeResult(context.opContext(), UPGRADE_ID_URN, entityService); - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.SUCCEEDED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.SUCCEEDED); }; } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/dataprocessinstances/BackfillDataProcessInstances.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/dataprocessinstances/BackfillDataProcessInstances.java new file mode 100644 index 00000000000000..643a0ff5a4ce25 --- /dev/null +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/dataprocessinstances/BackfillDataProcessInstances.java @@ -0,0 +1,54 @@ +package com.linkedin.datahub.upgrade.system.dataprocessinstances; + +import com.google.common.collect.ImmutableList; +import com.linkedin.datahub.upgrade.UpgradeStep; +import com.linkedin.datahub.upgrade.system.NonBlockingSystemUpgrade; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.search.elasticsearch.ElasticSearchService; +import io.datahubproject.metadata.context.OperationContext; +import java.util.List; +import org.opensearch.client.RestHighLevelClient; + +public class BackfillDataProcessInstances implements NonBlockingSystemUpgrade { + + private final List _steps; + + public BackfillDataProcessInstances( + OperationContext opContext, + EntityService entityService, + ElasticSearchService elasticSearchService, + RestHighLevelClient restHighLevelClient, + boolean enabled, + boolean reprocessEnabled, + Integer batchSize, + Integer batchDelayMs, + Integer totalDays, + Integer windowDays) { + if (enabled) { + _steps = + ImmutableList.of( + new BackfillDataProcessInstancesHasRunEventsStep( + opContext, + entityService, + elasticSearchService, + restHighLevelClient, + reprocessEnabled, + batchSize, + batchDelayMs, + totalDays, + windowDays)); + } else { + _steps = ImmutableList.of(); + } + } + + @Override + public String id() { + return "BackfillDataProcessInstances"; + } + + @Override + public List steps() { + return _steps; + } +} diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/dataprocessinstances/BackfillDataProcessInstancesHasRunEventsStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/dataprocessinstances/BackfillDataProcessInstancesHasRunEventsStep.java new file mode 100644 index 00000000000000..55cdcae931ab5b --- /dev/null +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/dataprocessinstances/BackfillDataProcessInstancesHasRunEventsStep.java @@ -0,0 +1,213 @@ +package com.linkedin.datahub.upgrade.system.dataprocessinstances; + +import static com.linkedin.metadata.Constants.*; + +import com.google.common.base.Throwables; +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.upgrade.UpgradeContext; +import com.linkedin.datahub.upgrade.UpgradeStep; +import com.linkedin.datahub.upgrade.UpgradeStepResult; +import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult; +import com.linkedin.metadata.boot.BootstrapStep; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.search.elasticsearch.ElasticSearchService; +import com.linkedin.metadata.utils.elasticsearch.IndexConvention; +import com.linkedin.upgrade.DataHubUpgradeState; +import io.datahubproject.metadata.context.OperationContext; +import java.io.IOException; +import java.net.URISyntaxException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.function.Function; +import lombok.extern.slf4j.Slf4j; +import org.codehaus.jackson.node.JsonNodeFactory; +import org.codehaus.jackson.node.ObjectNode; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.RestHighLevelClient; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.search.aggregations.Aggregation; +import org.opensearch.search.aggregations.AggregationBuilders; +import org.opensearch.search.aggregations.bucket.composite.CompositeAggregation; +import org.opensearch.search.aggregations.bucket.composite.CompositeAggregationBuilder; +import org.opensearch.search.aggregations.bucket.composite.TermsValuesSourceBuilder; +import org.opensearch.search.builder.SearchSourceBuilder; + +@Slf4j +public class BackfillDataProcessInstancesHasRunEventsStep implements UpgradeStep { + + private static final String UPGRADE_ID = "BackfillDataProcessInstancesHasRunEvents"; + private static final Urn UPGRADE_ID_URN = BootstrapStep.getUpgradeUrn(UPGRADE_ID); + + private final OperationContext opContext; + private final EntityService entityService; + private final ElasticSearchService elasticSearchService; + private final RestHighLevelClient restHighLevelClient; + + private final boolean reprocessEnabled; + private final Integer batchSize; + private final Integer batchDelayMs; + + private final Integer totalDays; + private final Integer windowDays; + + public BackfillDataProcessInstancesHasRunEventsStep( + OperationContext opContext, + EntityService entityService, + ElasticSearchService elasticSearchService, + RestHighLevelClient restHighLevelClient, + boolean reprocessEnabled, + Integer batchSize, + Integer batchDelayMs, + Integer totalDays, + Integer windowDays) { + this.opContext = opContext; + this.entityService = entityService; + this.elasticSearchService = elasticSearchService; + this.restHighLevelClient = restHighLevelClient; + this.reprocessEnabled = reprocessEnabled; + this.batchSize = batchSize; + this.batchDelayMs = batchDelayMs; + this.totalDays = totalDays; + this.windowDays = windowDays; + } + + @SuppressWarnings("BusyWait") + @Override + public Function executable() { + return (context) -> { + TermsValuesSourceBuilder termsValuesSourceBuilder = + new TermsValuesSourceBuilder("urn").field("urn"); + + ObjectNode json = JsonNodeFactory.instance.objectNode(); + json.put("hasRunEvents", true); + + IndexConvention indexConvention = opContext.getSearchContext().getIndexConvention(); + + String runEventsIndexName = + indexConvention.getTimeseriesAspectIndexName( + DATA_PROCESS_INSTANCE_ENTITY_NAME, DATA_PROCESS_INSTANCE_RUN_EVENT_ASPECT_NAME); + + DataHubUpgradeState upgradeState = DataHubUpgradeState.SUCCEEDED; + + Instant now = Instant.now(); + Instant overallStart = now.minus(totalDays, ChronoUnit.DAYS); + for (int i = 0; ; i++) { + Instant windowEnd = now.minus(i * windowDays, ChronoUnit.DAYS); + if (!windowEnd.isAfter(overallStart)) { + break; + } + Instant windowStart = windowEnd.minus(windowDays, ChronoUnit.DAYS); + if (windowStart.isBefore(overallStart)) { + // last iteration, cap at overallStart + windowStart = overallStart; + } + + QueryBuilder queryBuilder = + QueryBuilders.boolQuery() + .must( + QueryBuilders.rangeQuery("@timestamp") + .gte(windowStart.toString()) + .lt(windowEnd.toString())); + + CompositeAggregationBuilder aggregationBuilder = + AggregationBuilders.composite("aggs", List.of(termsValuesSourceBuilder)) + .size(batchSize); + + while (true) { + SearchRequest searchRequest = new SearchRequest(runEventsIndexName); + searchRequest.source( + new SearchSourceBuilder() + .size(0) + .aggregation(aggregationBuilder) + .query(queryBuilder)); + + SearchResponse response; + + try { + response = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); + } catch (IOException e) { + log.error(Throwables.getStackTraceAsString(e)); + log.error("Error querying index {}", runEventsIndexName); + upgradeState = DataHubUpgradeState.FAILED; + break; + } + List aggregations = response.getAggregations().asList(); + if (aggregations.isEmpty()) { + break; + } + CompositeAggregation aggregation = (CompositeAggregation) aggregations.get(0); + Set urns = new HashSet<>(); + for (CompositeAggregation.Bucket bucket : aggregation.getBuckets()) { + for (Object value : bucket.getKey().values()) { + try { + urns.add(Urn.createFromString(String.valueOf(value))); + } catch (URISyntaxException e) { + log.warn("Ignoring invalid urn {}", value); + } + } + } + if (!urns.isEmpty()) { + urns = entityService.exists(opContext, urns); + urns.forEach( + urn -> + elasticSearchService.upsertDocument( + opContext, + DATA_PROCESS_INSTANCE_ENTITY_NAME, + json.toString(), + indexConvention.getEntityDocumentId(urn))); + } + if (aggregation.afterKey() == null) { + break; + } + aggregationBuilder.aggregateAfter(aggregation.afterKey()); + if (batchDelayMs > 0) { + log.info("Sleeping for {} ms", batchDelayMs); + try { + Thread.sleep(batchDelayMs); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + } + } + BootstrapStep.setUpgradeResult(context.opContext(), UPGRADE_ID_URN, entityService); + return new DefaultUpgradeStepResult(id(), upgradeState); + }; + } + + @Override + public String id() { + return UPGRADE_ID; + } + + /** + * Returns whether the upgrade should proceed if the step fails after exceeding the maximum + * retries. + */ + @Override + public boolean isOptional() { + return true; + } + + /** Returns whether the upgrade should be skipped. */ + @Override + public boolean skip(UpgradeContext context) { + if (reprocessEnabled) { + return false; + } + + boolean previouslyRun = + entityService.exists( + context.opContext(), UPGRADE_ID_URN, DATA_HUB_UPGRADE_RESULT_ASPECT_NAME, true); + if (previouslyRun) { + log.info("{} was already run. Skipping.", id()); + } + return previouslyRun; + } +} diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/BuildIndicesPostStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/BuildIndicesPostStep.java index 09f65c84480279..c7e59998860c17 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/BuildIndicesPostStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/BuildIndicesPostStep.java @@ -14,6 +14,7 @@ import com.linkedin.metadata.search.elasticsearch.indexbuilder.ReindexConfig; import com.linkedin.metadata.shared.ElasticSearchIndexed; import com.linkedin.structured.StructuredPropertyDefinition; +import com.linkedin.upgrade.DataHubUpgradeState; import com.linkedin.util.Pair; import java.util.List; import java.util.Map; @@ -86,14 +87,14 @@ public Function executable() { log.error( "Partial index settings update, some indices may still be blocking writes." + " Please fix the error and rerun the BuildIndices upgrade job."); - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.FAILED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.FAILED); } } } catch (Exception e) { log.error("BuildIndicesPostStep failed.", e); - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.FAILED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.FAILED); } - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.SUCCEEDED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.SUCCEEDED); }; } } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/BuildIndicesPreStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/BuildIndicesPreStep.java index 983e7f0c97f38b..bd56042a1ff16d 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/BuildIndicesPreStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/BuildIndicesPreStep.java @@ -15,6 +15,7 @@ import com.linkedin.metadata.search.elasticsearch.indexbuilder.ReindexConfig; import com.linkedin.metadata.shared.ElasticSearchIndexed; import com.linkedin.structured.StructuredPropertyDefinition; +import com.linkedin.upgrade.DataHubUpgradeState; import com.linkedin.util.Pair; import java.io.IOException; import java.util.List; @@ -69,7 +70,7 @@ public Function executable() { log.error( "Partial index settings update, some indices may still be blocking writes." + " Please fix the error and re-run the BuildIndices upgrade job."); - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.FAILED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.FAILED); } // Clone indices @@ -87,15 +88,15 @@ public Function executable() { log.error( "Partial index settings update, cloned indices may need to be cleaned up: {}", clonedName); - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.FAILED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.FAILED); } } } } catch (Exception e) { log.error("BuildIndicesPreStep failed.", e); - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.FAILED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.FAILED); } - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.SUCCEEDED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.SUCCEEDED); }; } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/BuildIndicesStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/BuildIndicesStep.java index 5cf370162a3125..0aefc078ffdcba 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/BuildIndicesStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/BuildIndicesStep.java @@ -7,6 +7,7 @@ import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult; import com.linkedin.metadata.shared.ElasticSearchIndexed; import com.linkedin.structured.StructuredPropertyDefinition; +import com.linkedin.upgrade.DataHubUpgradeState; import com.linkedin.util.Pair; import java.util.List; import java.util.Set; @@ -40,9 +41,9 @@ public Function executable() { } } catch (Exception e) { log.error("BuildIndicesStep failed.", e); - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.FAILED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.FAILED); } - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.SUCCEEDED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.SUCCEEDED); }; } } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/CleanIndicesStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/CleanIndicesStep.java index fd5592c4ead25e..63b23fefceaf98 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/CleanIndicesStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/CleanIndicesStep.java @@ -10,6 +10,7 @@ import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; import com.linkedin.metadata.shared.ElasticSearchIndexed; import com.linkedin.structured.StructuredPropertyDefinition; +import com.linkedin.upgrade.DataHubUpgradeState; import com.linkedin.util.Pair; import java.util.List; import java.util.Set; @@ -54,9 +55,9 @@ public Function executable() { reindexConfig -> ESIndexBuilder.cleanIndex(searchClient, esConfig, reindexConfig)); } catch (Exception e) { log.error("CleanUpIndicesStep failed.", e); - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.FAILED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.FAILED); } - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.SUCCEEDED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.SUCCEEDED); }; } } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/DataHubStartupStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/DataHubStartupStep.java index d2b5965a3109ce..1d83810580cf55 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/DataHubStartupStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/DataHubStartupStep.java @@ -6,6 +6,7 @@ import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult; import com.linkedin.metadata.dao.producer.KafkaEventProducer; import com.linkedin.mxe.DataHubUpgradeHistoryEvent; +import com.linkedin.upgrade.DataHubUpgradeState; import java.util.function.Function; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -36,9 +37,9 @@ public Function executable() { log.info("System Update finished for version: {}", _version); } catch (Exception e) { log.error("DataHubStartupStep failed.", e); - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.FAILED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.FAILED); } - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.SUCCEEDED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.SUCCEEDED); }; } } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/policyfields/BackfillPolicyFieldsStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/policyfields/BackfillPolicyFieldsStep.java index c65a45aefc357f..ad28e6b6382d45 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/policyfields/BackfillPolicyFieldsStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/policyfields/BackfillPolicyFieldsStep.java @@ -27,6 +27,7 @@ import com.linkedin.metadata.search.SearchEntity; import com.linkedin.metadata.search.SearchService; import com.linkedin.policy.DataHubPolicyInfo; +import com.linkedin.upgrade.DataHubUpgradeState; import io.datahubproject.metadata.context.OperationContext; import java.net.URISyntaxException; import java.util.Collections; @@ -90,7 +91,7 @@ public Function executable() { BootstrapStep.setUpgradeResult(context.opContext(), UPGRADE_ID_URN, entityService); - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.SUCCEEDED); + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.SUCCEEDED); }; } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/schemafield/GenerateSchemaFieldsFromSchemaMetadata.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/schemafield/GenerateSchemaFieldsFromSchemaMetadata.java new file mode 100644 index 00000000000000..20bc65bf15daee --- /dev/null +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/schemafield/GenerateSchemaFieldsFromSchemaMetadata.java @@ -0,0 +1,48 @@ +package com.linkedin.datahub.upgrade.system.schemafield; + +import com.google.common.collect.ImmutableList; +import com.linkedin.datahub.upgrade.UpgradeStep; +import com.linkedin.datahub.upgrade.system.NonBlockingSystemUpgrade; +import com.linkedin.metadata.entity.AspectDao; +import com.linkedin.metadata.entity.EntityService; +import io.datahubproject.metadata.context.OperationContext; +import java.util.List; +import javax.annotation.Nonnull; +import lombok.extern.slf4j.Slf4j; + +/** + * A {@link NonBlockingSystemUpgrade} upgrade job that generates schema fields from schema metadata. + */ +@Slf4j +public class GenerateSchemaFieldsFromSchemaMetadata implements NonBlockingSystemUpgrade { + + private final List _steps; + + public GenerateSchemaFieldsFromSchemaMetadata( + @Nonnull OperationContext opContext, + EntityService entityService, + AspectDao aspectDao, + boolean enabled, + Integer batchSize, + Integer batchDelayMs, + Integer limit) { + if (enabled) { + _steps = + ImmutableList.of( + new GenerateSchemaFieldsFromSchemaMetadataStep( + opContext, entityService, aspectDao, batchSize, batchDelayMs, limit)); + } else { + _steps = ImmutableList.of(); + } + } + + @Override + public String id() { + return this.getClass().getName(); + } + + @Override + public List steps() { + return _steps; + } +} diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/schemafield/GenerateSchemaFieldsFromSchemaMetadataStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/schemafield/GenerateSchemaFieldsFromSchemaMetadataStep.java new file mode 100644 index 00000000000000..eece83f4ab713e --- /dev/null +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/schemafield/GenerateSchemaFieldsFromSchemaMetadataStep.java @@ -0,0 +1,256 @@ +package com.linkedin.datahub.upgrade.system.schemafield; + +import static com.linkedin.metadata.Constants.APP_SOURCE; +import static com.linkedin.metadata.Constants.DATASET_ENTITY_NAME; +import static com.linkedin.metadata.Constants.SCHEMA_METADATA_ASPECT_NAME; +import static com.linkedin.metadata.Constants.STATUS_ASPECT_NAME; +import static com.linkedin.metadata.Constants.SYSTEM_UPDATE_SOURCE; + +import com.google.common.annotations.VisibleForTesting; +import com.linkedin.common.urn.Urn; +import com.linkedin.data.template.StringMap; +import com.linkedin.datahub.upgrade.UpgradeContext; +import com.linkedin.datahub.upgrade.UpgradeStep; +import com.linkedin.datahub.upgrade.UpgradeStepResult; +import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.aspect.ReadItem; +import com.linkedin.metadata.aspect.batch.AspectsBatch; +import com.linkedin.metadata.boot.BootstrapStep; +import com.linkedin.metadata.entity.AspectDao; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.entity.EntityUtils; +import com.linkedin.metadata.entity.ebean.EbeanAspectV2; +import com.linkedin.metadata.entity.ebean.PartitionedStream; +import com.linkedin.metadata.entity.ebean.batch.AspectsBatchImpl; +import com.linkedin.metadata.entity.ebean.batch.ChangeItemImpl; +import com.linkedin.metadata.entity.restoreindices.RestoreIndicesArgs; +import com.linkedin.mxe.SystemMetadata; +import com.linkedin.upgrade.DataHubUpgradeResult; +import com.linkedin.upgrade.DataHubUpgradeState; +import io.datahubproject.metadata.context.OperationContext; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.Nullable; + +/** + * The `GenerateSchemaFieldsFromSchemaMetadataStep` class is an implementation of the `UpgradeStep` + * interface. This class is responsible for generating schema fields from schema metadata during an + * upgrade process. + * + *

The step performs the following actions: 1. Initializes with provided operation context, + * entity service, and aspect DAO. 2. Provides a unique identifier for the upgrade step. 3. + * Determines if the upgrade should be skipped based on the environment variable. 4. Executes the + * upgrade step which involves streaming aspects in batches, processing them, and updating schema + * fields. + * + *

This class utilizes various metadata and entity services to perform its operations, and + * includes configuration parameters such as batch size, delay between batches, and limits. + * + *

Environment Variables: - `SKIP_GENERATE_SCHEMA_FIELDS_FROM_SCHEMA_METADATA`: If set to `true`, + * the upgrade step is skipped. + * + *

Note: Schema Fields are generated with a status aspect to indicate presence of the field. No + * tags, documentation or other aspects are generated. We will write an upgrade to this job to + * generate the other aspects in the future (v2). + */ +@Slf4j +public class GenerateSchemaFieldsFromSchemaMetadataStep implements UpgradeStep { + private static final String LAST_URN_KEY = "lastUrn"; + private static final List REQUIRED_ASPECTS = + List.of(SCHEMA_METADATA_ASPECT_NAME, STATUS_ASPECT_NAME); + + private final OperationContext opContext; + private final EntityService entityService; + private final AspectDao aspectDao; + + private final int batchSize; + private final int batchDelayMs; + private final int limit; + + public GenerateSchemaFieldsFromSchemaMetadataStep( + OperationContext opContext, + EntityService entityService, + AspectDao aspectDao, + Integer batchSize, + Integer batchDelayMs, + Integer limit) { + this.opContext = opContext; + this.entityService = entityService; + this.aspectDao = aspectDao; + this.batchSize = batchSize; + this.batchDelayMs = batchDelayMs; + this.limit = limit; + log.info("GenerateSchemaFieldsFromSchemaMetadataStep initialized"); + } + + @Override + public String id() { + return "schema-field-from-schema-metadata-v1"; + } + + @VisibleForTesting + @Nullable + public String getUrnLike() { + return "urn:li:" + DATASET_ENTITY_NAME + ":%"; + } + + /** + * Returns whether the upgrade should be skipped. Uses previous run history or the environment + * variable SKIP_GENERATE_SCHEMA_FIELDS_FROM_SCHEMA_METADATA to determine whether to skip. + */ + public boolean skip(UpgradeContext context) { + if (Boolean.parseBoolean(System.getenv("SKIP_GENERATE_SCHEMA_FIELDS_FROM_SCHEMA_METADATA"))) { + log.info( + "Environment variable SKIP_GENERATE_SCHEMA_FIELDS_FROM_SCHEMA_METADATA is set to true. Skipping."); + return true; + } + + Optional prevResult = + context.upgrade().getUpgradeResult(opContext, getUpgradeIdUrn(), entityService); + + return prevResult + .filter( + result -> + DataHubUpgradeState.SUCCEEDED.equals(result.getState()) + || DataHubUpgradeState.ABORTED.equals(result.getState())) + .isPresent(); + } + + protected Urn getUpgradeIdUrn() { + return BootstrapStep.getUpgradeUrn(id()); + } + + @Override + public Function executable() { + log.info("Starting GenerateSchemaFieldsFromSchemaMetadataStep"); + return (context) -> { + // Resume state + Optional prevResult = + context.upgrade().getUpgradeResult(opContext, getUpgradeIdUrn(), entityService); + String resumeUrn = + prevResult + .filter( + result -> + DataHubUpgradeState.IN_PROGRESS.equals(result.getState()) + && result.getResult() != null + && result.getResult().containsKey(LAST_URN_KEY)) + .map(result -> result.getResult().get(LAST_URN_KEY)) + .orElse(null); + if (resumeUrn != null) { + log.info("{}: Resuming from URN: {}", getUpgradeIdUrn(), resumeUrn); + } + + // re-using for configuring the sql scan + RestoreIndicesArgs args = + new RestoreIndicesArgs() + .aspectNames(REQUIRED_ASPECTS) + .batchSize(batchSize) + .lastUrn(resumeUrn) + .urnBasedPagination(resumeUrn != null) + .limit(limit); + + if (getUrnLike() != null) { + args = args.urnLike(getUrnLike()); + } + + try (PartitionedStream stream = aspectDao.streamAspectBatches(args)) { + stream + .partition(args.batchSize) + .forEach( + batch -> { + log.info("Processing batch of size {}.", batchSize); + + AspectsBatch aspectsBatch = + AspectsBatchImpl.builder() + .retrieverContext(opContext.getRetrieverContext().get()) + .items( + batch + .flatMap( + ebeanAspectV2 -> + EntityUtils.toSystemAspectFromEbeanAspects( + opContext.getRetrieverContext().get(), + Set.of(ebeanAspectV2)) + .stream()) + .map( + systemAspect -> + ChangeItemImpl.builder() + .changeType(ChangeType.UPSERT) + .urn(systemAspect.getUrn()) + .entitySpec(systemAspect.getEntitySpec()) + .aspectName(systemAspect.getAspectName()) + .aspectSpec(systemAspect.getAspectSpec()) + .recordTemplate(systemAspect.getRecordTemplate()) + .auditStamp(systemAspect.getAuditStamp()) + .systemMetadata( + withAppSource(systemAspect.getSystemMetadata())) + .build( + opContext + .getRetrieverContext() + .get() + .getAspectRetriever())) + .collect(Collectors.toList())) + .build(); + + // re-ingest the aspects to trigger side effects + entityService.ingestAspects(opContext, aspectsBatch, true, false); + + // record progress + Urn lastUrn = + aspectsBatch.getItems().stream() + .reduce((a, b) -> b) + .map(ReadItem::getUrn) + .orElse(null); + if (lastUrn != null) { + log.info("{}: Saving state. Last urn:{}", getUpgradeIdUrn(), lastUrn); + context + .upgrade() + .setUpgradeResult( + opContext, + getUpgradeIdUrn(), + entityService, + DataHubUpgradeState.IN_PROGRESS, + Map.of(LAST_URN_KEY, lastUrn.toString())); + } + + if (batchDelayMs > 0) { + log.info("Sleeping for {} ms", batchDelayMs); + try { + Thread.sleep(batchDelayMs); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + }); + } + + BootstrapStep.setUpgradeResult(opContext, getUpgradeIdUrn(), entityService); + context.report().addLine("State updated: " + getUpgradeIdUrn()); + + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.SUCCEEDED); + }; + } + + private static SystemMetadata withAppSource(@Nullable SystemMetadata systemMetadata) { + SystemMetadata withAppSourceSystemMetadata = null; + try { + withAppSourceSystemMetadata = + systemMetadata != null + ? new SystemMetadata(systemMetadata.copy().data()) + : new SystemMetadata(); + } catch (CloneNotSupportedException e) { + throw new RuntimeException(e); + } + StringMap properties = withAppSourceSystemMetadata.getProperties(); + StringMap map = properties != null ? new StringMap(properties.data()) : new StringMap(); + map.put(APP_SOURCE, SYSTEM_UPDATE_SOURCE); + + withAppSourceSystemMetadata.setProperties(map); + return withAppSourceSystemMetadata; + } +} diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/schemafield/MigrateSchemaFieldDocIds.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/schemafield/MigrateSchemaFieldDocIds.java new file mode 100644 index 00000000000000..6cfd0d8c032ec1 --- /dev/null +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/schemafield/MigrateSchemaFieldDocIds.java @@ -0,0 +1,50 @@ +package com.linkedin.datahub.upgrade.system.schemafield; + +import com.google.common.collect.ImmutableList; +import com.linkedin.datahub.upgrade.UpgradeStep; +import com.linkedin.datahub.upgrade.system.NonBlockingSystemUpgrade; +import com.linkedin.gms.factory.search.BaseElasticSearchComponentsFactory; +import com.linkedin.metadata.entity.EntityService; +import io.datahubproject.metadata.context.OperationContext; +import java.util.List; +import javax.annotation.Nonnull; +import lombok.extern.slf4j.Slf4j; + +/** Migrate from URN document ids to hash based ids */ +@Slf4j +public class MigrateSchemaFieldDocIds implements NonBlockingSystemUpgrade { + private final List _steps; + + public MigrateSchemaFieldDocIds( + @Nonnull OperationContext opContext, + BaseElasticSearchComponentsFactory.BaseElasticSearchComponents elasticSearchComponents, + EntityService entityService, + boolean enabled, + Integer batchSize, + Integer batchDelayMs, + Integer limit) { + if (enabled) { + _steps = + ImmutableList.of( + new MigrateSchemaFieldDocIdsStep( + opContext, + elasticSearchComponents, + entityService, + batchSize, + batchDelayMs, + limit)); + } else { + _steps = ImmutableList.of(); + } + } + + @Override + public String id() { + return this.getClass().getName(); + } + + @Override + public List steps() { + return _steps; + } +} diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/schemafield/MigrateSchemaFieldDocIdsStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/schemafield/MigrateSchemaFieldDocIdsStep.java new file mode 100644 index 00000000000000..ab35b42bcc8481 --- /dev/null +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/schemafield/MigrateSchemaFieldDocIdsStep.java @@ -0,0 +1,291 @@ +package com.linkedin.datahub.upgrade.system.schemafield; + +import static com.linkedin.metadata.Constants.SCHEMA_FIELD_ENTITY_NAME; +import static com.linkedin.metadata.utils.GenericRecordUtils.entityResponseToSystemAspectMap; + +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.upgrade.UpgradeContext; +import com.linkedin.datahub.upgrade.UpgradeStep; +import com.linkedin.datahub.upgrade.UpgradeStepResult; +import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.gms.factory.search.BaseElasticSearchComponentsFactory; +import com.linkedin.metadata.aspect.SystemAspect; +import com.linkedin.metadata.boot.BootstrapStep; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.metadata.utils.AuditStampUtils; +import com.linkedin.upgrade.DataHubUpgradeResult; +import com.linkedin.upgrade.DataHubUpgradeState; +import io.datahubproject.metadata.context.OperationContext; +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.Arrays; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.function.Function; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.opensearch.action.bulk.BulkProcessor; +import org.opensearch.action.bulk.BulkRequest; +import org.opensearch.action.bulk.BulkResponse; +import org.opensearch.action.delete.DeleteRequest; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.search.SearchScrollRequest; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.RestHighLevelClient; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.script.Script; +import org.opensearch.search.Scroll; +import org.opensearch.search.SearchHit; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.search.sort.SortOrder; + +/** + * Performs a migration from one document id convention to another. + * + *

This step identifies documents using the old convention, if any, and generates MCLs for + * aspects for updating the data in the new document. + * + *

Finally, a DELETE is executed on the legacy document id. + */ +@Slf4j +public class MigrateSchemaFieldDocIdsStep implements UpgradeStep { + + private final OperationContext opContext; + private final EntityRegistry entityRegistry; + private final RestHighLevelClient elasticsearchClient; + private final BulkProcessor bulkProcessor; + private final String indexName; + private final EntityService entityService; + private final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(5L)); + private final int batchSize; + private final int batchDelayMs; + private final int limit; + + public MigrateSchemaFieldDocIdsStep( + OperationContext opContext, + BaseElasticSearchComponentsFactory.BaseElasticSearchComponents elasticSearchComponents, + EntityService entityService, + int batchSize, + int batchDelayMs, + int limit) { + this.opContext = opContext; + this.entityRegistry = opContext.getEntityRegistry(); + this.elasticsearchClient = elasticSearchComponents.getSearchClient(); + this.entityService = entityService; + this.batchSize = batchSize; + this.batchDelayMs = batchDelayMs; + this.limit = limit; + this.indexName = + elasticSearchComponents.getIndexConvention().getEntityIndexName(SCHEMA_FIELD_ENTITY_NAME); + this.bulkProcessor = buildBuildProcessor(); + log.info("MigrateSchemaFieldDocIdsStep initialized"); + } + + @Override + public String id() { + return "schema-field-doc-id-v1"; + } + + /** + * Returns whether the upgrade should be skipped. Uses previous run history or the environment + * variable SKIP_MIGRATE_SCHEMA_FIELDS_DOC_ID to determine whether to skip. + */ + public boolean skip(UpgradeContext context) { + if (Boolean.parseBoolean(System.getenv("SKIP_MIGRATE_SCHEMA_FIELDS_DOC_ID"))) { + log.info("Environment variable SKIP_MIGRATE_SCHEMA_FIELDS_DOC_ID is set to true. Skipping."); + return true; + } + + Optional prevResult = + context.upgrade().getUpgradeResult(opContext, getUpgradeIdUrn(), entityService); + + return prevResult + .filter( + result -> + DataHubUpgradeState.SUCCEEDED.equals(result.getState()) + || DataHubUpgradeState.ABORTED.equals(result.getState())) + .isPresent(); + } + + protected Urn getUpgradeIdUrn() { + return BootstrapStep.getUpgradeUrn(id()); + } + + @Override + public Function executable() { + return (context) -> { + final SearchRequest searchRequest = buildSearchRequest(); + String scrollId = null; + int migratedCount = 0; + + try { + do { + log.info( + "Upgrading batch of schemaFields {}-{}", migratedCount, migratedCount + batchSize); + scrollId = updateDocId(searchRequest, scrollId); + migratedCount += batchSize; + + if (limit > 0 && migratedCount >= limit) { + log.info("Exiting early due to limit."); + break; + } + + if (batchDelayMs > 0) { + log.info("Sleeping for {} ms", batchDelayMs); + try { + Thread.sleep(batchDelayMs); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + } while (scrollId != null); + } catch (Exception e) { + throw new RuntimeException(e); + } finally { + bulkProcessor.flush(); + } + + BootstrapStep.setUpgradeResult(context.opContext(), getUpgradeIdUrn(), entityService); + + return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.SUCCEEDED); + }; + } + + private BulkProcessor buildBuildProcessor() { + return BulkProcessor.builder( + (request, bulkListener) -> + elasticsearchClient.bulkAsync(request, RequestOptions.DEFAULT, bulkListener), + new BulkProcessor.Listener() { + @Override + public void beforeBulk(long executionId, BulkRequest request) { + log.debug("Deleting {} legacy schemaField documents", request.numberOfActions()); + } + + @Override + public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { + log.debug( + "Delete executed {} failures", response.hasFailures() ? "with" : "without"); + } + + @Override + public void afterBulk(long executionId, BulkRequest request, Throwable failure) { + log.warn("Error while executing legacy schemaField documents", failure); + } + }) + .setBulkActions(batchSize) + .build(); + } + + private SearchRequest buildSearchRequest() { + Script oldDocumentIdQuery = + new Script( + Script.DEFAULT_SCRIPT_TYPE, + "painless", + "doc['_id'][0].indexOf('urn%3Ali%3AschemaField%3A%28urn%3Ali%3Adataset%3A') > -1", + Map.of()); + QueryBuilder queryBuilder = QueryBuilders.scriptQuery(oldDocumentIdQuery); + + SearchRequest searchRequest = new SearchRequest(indexName); + searchRequest.scroll(scroll); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(queryBuilder); + searchSourceBuilder.size(batchSize); + searchSourceBuilder.sort("urn", SortOrder.ASC); + searchRequest.source(searchSourceBuilder); + + return searchRequest; + } + + private String updateDocId(final SearchRequest searchRequest, final String scrollId) + throws IOException, URISyntaxException { + final SearchResponse searchResponse; + + if (scrollId == null) { + searchResponse = elasticsearchClient.search(searchRequest, RequestOptions.DEFAULT); + } else { + SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); + scrollRequest.scroll(scroll); + searchResponse = elasticsearchClient.scroll(scrollRequest, RequestOptions.DEFAULT); + } + + final SearchHit[] searchHits = searchResponse.getHits().getHits(); + final String nextScrollId = searchResponse.getScrollId(); + + if (searchHits.length > 0) { + Set documentIds = + Arrays.stream(searchHits).map(SearchHit::getId).collect(Collectors.toSet()); + Set batchUrns = + Arrays.stream(searchHits) + .map(hit -> hit.getSourceAsMap().get("urn").toString()) + .map(UrnUtils::getUrn) + .collect(Collectors.toSet()); + + log.info("Sending MCLs for {} entities", batchUrns.size()); + emitMCLs(batchUrns); + log.info("Removing old document ids for {} documents", documentIds.size()); + deleteDocumentIds(documentIds); + + return nextScrollId; + } + + return null; + } + + private void emitMCLs(Set batchUrns) throws URISyntaxException { + Set batchAspects = + entityResponseToSystemAspectMap( + entityService.getEntitiesV2( + opContext, + SCHEMA_FIELD_ENTITY_NAME, + batchUrns, + opContext.getEntityAspectNames(SCHEMA_FIELD_ENTITY_NAME), + false), + entityRegistry) + .values() + .stream() + .flatMap(m -> m.values().stream()) + .collect(Collectors.toSet()); + + Set> futures = + batchAspects.stream() + .map( + systemAspect -> + entityService + .alwaysProduceMCLAsync( + opContext, + systemAspect.getUrn(), + systemAspect.getUrn().getEntityType(), + systemAspect.getAspectName(), + systemAspect.getAspectSpec(), + null, + systemAspect.getRecordTemplate(), + null, + systemAspect.getSystemMetadata(), + AuditStampUtils.createDefaultAuditStamp(), + ChangeType.UPSERT) + .getFirst()) + .collect(Collectors.toSet()); + + futures.forEach( + f -> { + try { + f.get(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + }); + } + + private void deleteDocumentIds(Set documentIds) { + documentIds.forEach(docId -> bulkProcessor.add(new DeleteRequest(indexName, docId))); + } +} diff --git a/datahub-upgrade/src/main/resources/application.properties b/datahub-upgrade/src/main/resources/application.properties index b884c92f74bd48..847c264dfac38c 100644 --- a/datahub-upgrade/src/main/resources/application.properties +++ b/datahub-upgrade/src/main/resources/application.properties @@ -3,3 +3,4 @@ management.health.neo4j.enabled=false ingestion.enabled=false spring.main.allow-bean-definition-overriding=true entityClient.impl=restli +metadataChangeProposal.throttle.updateIntervalMs=0 \ No newline at end of file diff --git a/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/DatahubUpgradeNoSchemaRegistryTest.java b/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/DatahubUpgradeNoSchemaRegistryTest.java index 8c9b72b0d88e5e..68dfd71ac10044 100644 --- a/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/DatahubUpgradeNoSchemaRegistryTest.java +++ b/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/DatahubUpgradeNoSchemaRegistryTest.java @@ -7,12 +7,13 @@ import static org.testng.Assert.assertNotNull; import com.linkedin.datahub.upgrade.system.SystemUpdate; -import com.linkedin.gms.factory.kafka.schemaregistry.SchemaRegistryConfig; import com.linkedin.metadata.boot.kafka.MockSystemUpdateDeserializer; import com.linkedin.metadata.boot.kafka.MockSystemUpdateSerializer; +import com.linkedin.metadata.config.kafka.KafkaConfiguration; import com.linkedin.metadata.dao.producer.KafkaEventProducer; import com.linkedin.metadata.entity.EntityServiceImpl; import com.linkedin.mxe.Topics; +import com.linkedin.upgrade.DataHubUpgradeState; import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient; import io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException; import io.datahubproject.metadata.context.OperationContext; @@ -54,7 +55,7 @@ public class DatahubUpgradeNoSchemaRegistryTest extends AbstractTestNGSpringCont @Autowired @Named("schemaRegistryConfig") - private SchemaRegistryConfig schemaRegistryConfig; + private KafkaConfiguration.SerDeKeyValueConfig schemaRegistryConfig; @Test public void testSystemUpdateInit() { @@ -63,13 +64,17 @@ public void testSystemUpdateInit() { @Test public void testSystemUpdateKafkaProducerOverride() throws RestClientException, IOException { - assertEquals(schemaRegistryConfig.getDeserializer(), MockSystemUpdateDeserializer.class); - assertEquals(schemaRegistryConfig.getSerializer(), MockSystemUpdateSerializer.class); + assertEquals( + schemaRegistryConfig.getValue().getDeserializer(), + MockSystemUpdateDeserializer.class.getName()); + assertEquals( + schemaRegistryConfig.getValue().getSerializer(), + MockSystemUpdateSerializer.class.getName()); assertEquals(kafkaEventProducer, duheKafkaEventProducer); assertEquals(entityService.getProducer(), duheKafkaEventProducer); MockSystemUpdateSerializer serializer = new MockSystemUpdateSerializer(); - serializer.configure(schemaRegistryConfig.getProperties(), false); + serializer.configure(schemaRegistryConfig.getProperties(null), false); SchemaRegistryClient registry = serializer.getSchemaRegistryClient(); assertEquals( registry.getId( @@ -79,7 +84,7 @@ public void testSystemUpdateKafkaProducerOverride() throws RestClientException, @Test public void testSystemUpdateSend() { - UpgradeStepResult.Result result = + DataHubUpgradeState result = systemUpdate.steps().stream() .filter(s -> s.id().equals("DataHubStartupStep")) .findFirst() diff --git a/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/DatahubUpgradeNonBlockingTest.java b/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/DatahubUpgradeNonBlockingTest.java index 154b1de71f46cd..55a52f072a0caf 100644 --- a/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/DatahubUpgradeNonBlockingTest.java +++ b/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/DatahubUpgradeNonBlockingTest.java @@ -10,9 +10,9 @@ import com.linkedin.datahub.upgrade.impl.DefaultUpgradeManager; import com.linkedin.datahub.upgrade.system.SystemUpdateNonBlocking; import com.linkedin.datahub.upgrade.system.vianodes.ReindexDataJobViaNodesCLL; -import com.linkedin.gms.factory.kafka.schemaregistry.SchemaRegistryConfig; import com.linkedin.metadata.boot.kafka.MockSystemUpdateDeserializer; import com.linkedin.metadata.boot.kafka.MockSystemUpdateSerializer; +import com.linkedin.metadata.config.kafka.KafkaConfiguration; import com.linkedin.metadata.dao.producer.KafkaEventProducer; import com.linkedin.metadata.entity.AspectDao; import com.linkedin.metadata.entity.EntityService; @@ -47,7 +47,7 @@ public class DatahubUpgradeNonBlockingTest extends AbstractTestNGSpringContextTe @Autowired @Named("schemaRegistryConfig") - private SchemaRegistryConfig schemaRegistryConfig; + private KafkaConfiguration.SerDeKeyValueConfig schemaRegistryConfig; @Autowired @Named("duheKafkaEventProducer") @@ -66,8 +66,12 @@ public void testSystemUpdateNonBlockingInit() { assertNotNull(systemUpdateNonBlocking); // Expected system update configuration and producer - assertEquals(schemaRegistryConfig.getDeserializer(), MockSystemUpdateDeserializer.class); - assertEquals(schemaRegistryConfig.getSerializer(), MockSystemUpdateSerializer.class); + assertEquals( + schemaRegistryConfig.getValue().getDeserializer(), + MockSystemUpdateDeserializer.class.getName()); + assertEquals( + schemaRegistryConfig.getValue().getSerializer(), + MockSystemUpdateSerializer.class.getName()); assertEquals(duheKafkaEventProducer, kafkaEventProducer); assertEquals(entityService.getProducer(), duheKafkaEventProducer); } diff --git a/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/UpgradeCliApplicationTest.java b/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/UpgradeCliApplicationTest.java index dc4c3073ee351c..8b6899b4c78866 100644 --- a/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/UpgradeCliApplicationTest.java +++ b/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/UpgradeCliApplicationTest.java @@ -1,12 +1,18 @@ package com.linkedin.datahub.upgrade; -import static org.testng.AssertJUnit.*; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; import com.linkedin.datahub.upgrade.restoreindices.RestoreIndices; import com.linkedin.datahub.upgrade.system.BlockingSystemUpgrade; +import com.linkedin.metadata.dao.throttle.NoOpSensor; +import com.linkedin.metadata.dao.throttle.ThrottleSensor; import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; import javax.inject.Named; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.testng.AbstractTestNGSpringContextTests; @@ -28,6 +34,10 @@ public class UpgradeCliApplicationTest extends AbstractTestNGSpringContextTests @Autowired private ESIndexBuilder esIndexBuilder; + @Qualifier("kafkaThrottle") + @Autowired + private ThrottleSensor kafkaThrottle; + @Test public void testRestoreIndicesInit() { /* @@ -46,4 +56,10 @@ public void testBuildIndicesInit() { assertFalse( esIndexBuilder.getElasticSearchConfiguration().getBuildIndices().isAllowDocCountMismatch()); } + + @Test + public void testNoThrottle() { + assertEquals( + new NoOpSensor(), kafkaThrottle, "No kafka throttle controls expected in datahub-upgrade"); + } } diff --git a/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/schemafield/GenerateSchemaFieldsFromSchemaMetadataStepTest.java b/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/schemafield/GenerateSchemaFieldsFromSchemaMetadataStepTest.java new file mode 100644 index 00000000000000..3a2728b4e1d3d6 --- /dev/null +++ b/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/schemafield/GenerateSchemaFieldsFromSchemaMetadataStepTest.java @@ -0,0 +1,110 @@ +package com.linkedin.datahub.upgrade.schemafield; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyInt; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.linkedin.datahub.upgrade.UpgradeContext; +import com.linkedin.datahub.upgrade.UpgradeStepResult; +import com.linkedin.datahub.upgrade.system.schemafield.GenerateSchemaFieldsFromSchemaMetadataStep; +import com.linkedin.metadata.aspect.SystemAspect; +import com.linkedin.metadata.entity.AspectDao; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.entity.ebean.EbeanAspectV2; +import com.linkedin.metadata.entity.ebean.PartitionedStream; +import com.linkedin.metadata.entity.restoreindices.RestoreIndicesArgs; +import com.linkedin.schema.SchemaMetadata; +import com.linkedin.upgrade.DataHubUpgradeState; +import io.datahubproject.metadata.context.OperationContext; +import io.datahubproject.metadata.context.RetrieverContext; +import java.util.Optional; +import java.util.stream.Stream; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +public class GenerateSchemaFieldsFromSchemaMetadataStepTest { + + @Mock private OperationContext mockOpContext; + + @Mock private EntityService mockEntityService; + + @Mock private AspectDao mockAspectDao; + + @Mock private RetrieverContext mockRetrieverContext; + + private GenerateSchemaFieldsFromSchemaMetadataStep step; + + @BeforeEach + public void setup() { + MockitoAnnotations.openMocks(this); + step = + new GenerateSchemaFieldsFromSchemaMetadataStep( + mockOpContext, mockEntityService, mockAspectDao, 10, 100, 1000); + when(mockOpContext.getRetrieverContext()).thenReturn(Optional.of(mockRetrieverContext)); + } + + /** Test to verify the correct step ID is returned. */ + @Test + public void testId() { + assertEquals("schema-field-from-schema-metadata-v1", step.id()); + } + + /** Test to verify the skip logic based on the environment variable. */ + @Test + public void testSkip() { + UpgradeContext mockContext = mock(UpgradeContext.class); + System.setProperty("SKIP_GENERATE_SCHEMA_FIELDS_FROM_SCHEMA_METADATA", "true"); + assertTrue(step.skip(mockContext)); + + System.setProperty("SKIP_GENERATE_SCHEMA_FIELDS_FROM_SCHEMA_METADATA", "false"); + assertFalse(step.skip(mockContext)); + } + + /** Test to verify the correct URN pattern is returned. */ + @Test + public void testGetUrnLike() { + assertEquals("urn:li:dataset:%", step.getUrnLike()); + } + + /** + * Test to verify the executable function processes batches correctly and returns a success + * result. + */ + @Test + public void testExecutable() { + UpgradeContext mockContext = mock(UpgradeContext.class); + + EbeanAspectV2 mockAspect = mock(EbeanAspectV2.class); + PartitionedStream mockStream = mock(PartitionedStream.class); + when(mockAspectDao.streamAspectBatches(any(RestoreIndicesArgs.class))).thenReturn(mockStream); + + when(mockStream.partition(anyInt())).thenReturn(Stream.of(Stream.of(mockAspect))); + + SystemAspect mockSystemAspect = mock(SystemAspect.class); + when(mockSystemAspect.getAspectName()).thenReturn("schemaMetadata"); + when(mockSystemAspect.getAspect(SchemaMetadata.class)).thenReturn(new SchemaMetadata()); + + // when(mockRetrieverContext.getAspectRetriever()).thenReturn(mockSystemAspect); + + ArgumentCaptor argsCaptor = + ArgumentCaptor.forClass(RestoreIndicesArgs.class); + + UpgradeStepResult result = step.executable().apply(mockContext); + assertEquals(DataHubUpgradeState.SUCCEEDED, result.result()); + + verify(mockAspectDao).streamAspectBatches(argsCaptor.capture()); + assertEquals("schemaMetadata", argsCaptor.getValue().aspectName()); + assertEquals(10, argsCaptor.getValue().batchSize()); + assertEquals(1000, argsCaptor.getValue().limit()); + } + + // Additional tests can be added to cover more scenarios and edge cases +} diff --git a/datahub-web-react/package.json b/datahub-web-react/package.json index f641706c7661e8..dcaef6004d7022 100644 --- a/datahub-web-react/package.json +++ b/datahub-web-react/package.json @@ -45,7 +45,7 @@ "dayjs": "^1.11.7", "deepmerge": "^4.2.2", "diff": "^5.0.0", - "dompurify": "^2.3.8", + "dompurify": "^2.5.4", "dotenv": "^8.2.0", "faker": "5.5.3", "graphql": "^15.5.0", @@ -132,7 +132,7 @@ "less": "^4.2.0", "prettier": "^2.8.8", "source-map-explorer": "^2.5.2", - "vite": "^4.5.3", + "vite": "^4.5.5", "vite-plugin-babel-macros": "^1.0.6", "vite-plugin-static-copy": "^0.17.0", "vite-plugin-svgr": "^4.1.0", diff --git a/datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearchResults.tsx b/datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearchResults.tsx index 26c90edd82b696..0749ff369c1251 100644 --- a/datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearchResults.tsx +++ b/datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearchResults.tsx @@ -10,6 +10,8 @@ import { SearchFiltersSection } from '../../../../../search/SearchFiltersSection import { EntitySearchResults, EntityActionProps } from './EntitySearchResults'; import MatchingViewsLabel from './MatchingViewsLabel'; import { ANTD_GRAY } from '../../../constants'; +import { useIsShowSeparateSiblingsEnabled } from '../../../../../useAppConfig'; +import { combineSiblingsInSearchResults } from '../../../../../search/utils/combineSiblingsInSearchResults'; const SearchBody = styled.div` height: 100%; @@ -129,6 +131,12 @@ export const EmbeddedListSearchResults = ({ onLineageClick, isLineageTab = false, }: Props) => { + const showSeparateSiblings = useIsShowSeparateSiblingsEnabled(); + const combinedSiblingSearchResults = combineSiblingsInSearchResults( + showSeparateSiblings, + searchResponse?.searchResults, + ); + const pageStart = searchResponse?.start || 0; const pageSize = searchResponse?.count || 0; const totalResults = searchResponse?.total || 0; @@ -169,9 +177,9 @@ export const EmbeddedListSearchResults = ({ )} {!loading && !isServerOverloadError && ( ({ + combinedSiblingSearchResults?.map((searchResult) => ({ // when we add impact analysis, we will want to pipe the path to each element to the result this // eslint-disable-next-line @typescript-eslint/dot-notation degree: searchResult['degree'], diff --git a/datahub-web-react/src/app/entity/shared/propagation/PropagationDetails.tsx b/datahub-web-react/src/app/entity/shared/propagation/PropagationDetails.tsx index 646f47134938c4..b5629d806f7197 100644 --- a/datahub-web-react/src/app/entity/shared/propagation/PropagationDetails.tsx +++ b/datahub-web-react/src/app/entity/shared/propagation/PropagationDetails.tsx @@ -69,7 +69,7 @@ export default function PropagationDetails({ sourceDetail }: Props) { Learn more diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/SchemaTab.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/SchemaTab.tsx index f9d7c8db025690..52c141282c3456 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/SchemaTab.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/SchemaTab.tsx @@ -79,7 +79,10 @@ export const SchemaTab = ({ properties }: { properties?: any }) => { const hasProperties = useMemo( () => entityWithSchema?.schemaMetadata?.fields?.some( - (schemaField) => !!schemaField.schemaFieldEntity?.structuredProperties?.properties?.length, + (schemaField) => + !!schemaField.schemaFieldEntity?.structuredProperties?.properties?.filter( + (prop) => prop.structuredProperty.exists, + )?.length, ), [entityWithSchema], ); diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/components/SchemaFieldDrawer/FieldProperties.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/components/SchemaFieldDrawer/FieldProperties.tsx index 689a191f469f53..9a0da20f22dfde 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/components/SchemaFieldDrawer/FieldProperties.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/components/SchemaFieldDrawer/FieldProperties.tsx @@ -34,14 +34,16 @@ interface Props { export default function FieldProperties({ expandedField }: Props) { const { schemaFieldEntity } = expandedField; const { refetch } = useGetEntityWithSchema(true); + const properties = + schemaFieldEntity?.structuredProperties?.properties?.filter((prop) => prop.structuredProperty.exists) || []; - if (!schemaFieldEntity?.structuredProperties?.properties?.length) return null; + if (!schemaFieldEntity || !properties.length) return null; return ( <> Properties - {schemaFieldEntity.structuredProperties.properties.map((structuredProp) => { + {properties.map((structuredProp) => { const isRichText = structuredProp.structuredProperty.definition.valueType?.info.type === StdDataType.RichText; const valuesData = mapStructuredPropertyValues(structuredProp); diff --git a/datahub-web-react/src/app/entity/shared/tabs/Properties/useStructuredProperties.tsx b/datahub-web-react/src/app/entity/shared/tabs/Properties/useStructuredProperties.tsx index 5600d7c3e8498e..86365b8232905c 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Properties/useStructuredProperties.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Properties/useStructuredProperties.tsx @@ -27,23 +27,26 @@ export function mapStructuredPropertyValues(structuredPropertiesEntry: Structure function getStructuredPropertyRows(entityData?: GenericEntityProperties | null) { const structuredPropertyRows: PropertyRow[] = []; - entityData?.structuredProperties?.properties?.forEach((structuredPropertiesEntry) => { - const { displayName, qualifiedName } = structuredPropertiesEntry.structuredProperty.definition; - structuredPropertyRows.push({ - displayName: displayName || qualifiedName, - qualifiedName, - values: mapStructuredPropertyValues(structuredPropertiesEntry), - dataType: structuredPropertiesEntry.structuredProperty.definition.valueType, - structuredProperty: structuredPropertiesEntry.structuredProperty, - type: - structuredPropertiesEntry.values[0] && structuredPropertiesEntry.values[0].__typename - ? { - type: typeNameToType[structuredPropertiesEntry.values[0].__typename].type, - nativeDataType: typeNameToType[structuredPropertiesEntry.values[0].__typename].nativeDataType, - } - : undefined, + entityData?.structuredProperties?.properties + ?.filter((prop) => prop.structuredProperty.exists) + .forEach((structuredPropertiesEntry) => { + const { displayName, qualifiedName } = structuredPropertiesEntry.structuredProperty.definition; + structuredPropertyRows.push({ + displayName: displayName || qualifiedName, + qualifiedName, + values: mapStructuredPropertyValues(structuredPropertiesEntry), + dataType: structuredPropertiesEntry.structuredProperty.definition.valueType, + structuredProperty: structuredPropertiesEntry.structuredProperty, + type: + structuredPropertiesEntry.values[0] && structuredPropertiesEntry.values[0].__typename + ? { + type: typeNameToType[structuredPropertiesEntry.values[0].__typename].type, + nativeDataType: + typeNameToType[structuredPropertiesEntry.values[0].__typename].nativeDataType, + } + : undefined, + }); }); - }); return structuredPropertyRows; } diff --git a/datahub-web-react/src/app/settings/features/Features.tsx b/datahub-web-react/src/app/settings/features/Features.tsx index 1d0a0bb469cf86..7f39fcc6d72855 100644 --- a/datahub-web-react/src/app/settings/features/Features.tsx +++ b/datahub-web-react/src/app/settings/features/Features.tsx @@ -89,7 +89,8 @@ export const Features = () => { }, ], isNew: true, - learnMoreLink: 'https://datahubproject.io/docs/automations/docs-propagation', + learnMoreLink: + 'https://datahubproject.io/docs/automations/docs-propagation?utm_source=datahub_core&utm_medium=docs&utm_campaign=features', }, ]; diff --git a/datahub-web-react/src/graphql/fragments.graphql b/datahub-web-react/src/graphql/fragments.graphql index e5bbb5f0dc29d5..17ad6f881b0abd 100644 --- a/datahub-web-react/src/graphql/fragments.graphql +++ b/datahub-web-react/src/graphql/fragments.graphql @@ -758,16 +758,6 @@ fragment schemaFieldFields on SchemaField { } } } - parent { - urn - type - ...entityDisplayNameFields - ... on Dataset { - platform { - ...platformFields - } - } - } } } @@ -1314,6 +1304,7 @@ fragment structuredPropertyFields on StructuredPropertyEntity { fragment structuredPropertiesFields on StructuredPropertiesEntry { structuredProperty { + exists ...structuredPropertyFields } values { diff --git a/datahub-web-react/yarn.lock b/datahub-web-react/yarn.lock index 4268acf155de1d..8d5899d9891f18 100644 --- a/datahub-web-react/yarn.lock +++ b/datahub-web-react/yarn.lock @@ -4510,7 +4510,14 @@ brace-expansion@^1.1.7: balanced-match "^1.0.0" concat-map "0.0.1" -braces@^3.0.2, braces@~3.0.2: +braces@^3.0.3: + version "3.0.3" + resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.3.tgz#490332f40919452272d55a8480adc0c441358789" + integrity sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA== + dependencies: + fill-range "^7.1.1" + +braces@~3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.2.tgz#3454e1a462ee8d599e236df336cd9ea4f8afe107" integrity sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A== @@ -5428,10 +5435,10 @@ domino@^2.1.6: resolved "https://registry.yarnpkg.com/domino/-/domino-2.1.6.tgz#fe4ace4310526e5e7b9d12c7de01b7f485a57ffe" integrity sha512-3VdM/SXBZX2omc9JF9nOPCtDaYQ67BGp5CoLpIQlO2KCAPETs8TcDHacF26jXadGbvUteZzRTeos2fhID5+ucQ== -dompurify@^2.3.8: - version "2.3.8" - resolved "https://registry.yarnpkg.com/dompurify/-/dompurify-2.3.8.tgz#224fe9ae57d7ebd9a1ae1ac18c1c1ca3f532226f" - integrity sha512-eVhaWoVibIzqdGYjwsBWodIQIaXFSB+cKDf4cfxLMsK0xiud6SE+/WCVx/Xw/UwQsa4cS3T2eITcdtmTg2UKcw== +dompurify@^2.5.4: + version "2.5.4" + resolved "https://registry.yarnpkg.com/dompurify/-/dompurify-2.5.4.tgz#347e91070963b22db31c7c8d0ce9a0a2c3c08746" + integrity sha512-l5NNozANzaLPPe0XaAwvg3uZcHtDBnziX/HjsY1UcDj1MxTK8Dd0Kv096jyPK5HRzs/XM5IMj20dW8Fk+HnbUA== dot-case@^3.0.4: version "3.0.4" @@ -5452,9 +5459,9 @@ dotenv@^8.2.0: integrity sha512-IrPdXQsk2BbzvCBGBOTmmSH5SodmqZNt4ERAZDmW4CT+tL8VtvinqywuANaFu4bOMWki16nqf0e4oC0QIaDr/g== dset@^3.1.2: - version "3.1.3" - resolved "https://registry.yarnpkg.com/dset/-/dset-3.1.3.tgz#c194147f159841148e8e34ca41f638556d9542d2" - integrity sha512-20TuZZHCEZ2O71q9/+8BwKwZ0QtD9D8ObhrihJPr+vLLYlSuAU3/zL4cSlgbfeoGHTjCSJBa7NGcrF9/Bx/WJQ== + version "3.1.4" + resolved "https://registry.yarnpkg.com/dset/-/dset-3.1.4.tgz#f8eaf5f023f068a036d08cd07dc9ffb7d0065248" + integrity sha512-2QF/g9/zTaPDc3BjNcVTGoBbXBgYfMTTceLaYcFJ/W9kggFUkhxD/hMEeuLKbugyef9SqAx8cpgwlIP/jinUTA== duplexer@^0.1.2: version "0.1.2" @@ -6052,6 +6059,13 @@ fill-range@^7.0.1: dependencies: to-regex-range "^5.0.1" +fill-range@^7.1.1: + version "7.1.1" + resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.1.1.tgz#44265d3cac07e3ea7dc247516380643754a05292" + integrity sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg== + dependencies: + to-regex-range "^5.0.1" + filter-obj@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/filter-obj/-/filter-obj-1.1.0.tgz#9b311112bc6c6127a16e016c6c5d7f19e0805c5b" @@ -7654,11 +7668,11 @@ micromark@^2.11.3, micromark@~2.11.0, micromark@~2.11.3: parse-entities "^2.0.0" micromatch@^4.0.4, micromatch@^4.0.5: - version "4.0.5" - resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.5.tgz#bc8999a7cbbf77cdc89f132f6e467051b49090c6" - integrity sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA== + version "4.0.8" + resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.8.tgz#d66fa18f3a47076789320b9b1af32bd86d9fa202" + integrity sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA== dependencies: - braces "^3.0.2" + braces "^3.0.3" picomatch "^2.3.1" mime-db@1.52.0: @@ -8201,9 +8215,9 @@ path-root@^0.1.1: path-root-regex "^0.1.0" path-to-regexp@^1.7.0: - version "1.8.0" - resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-1.8.0.tgz#887b3ba9d84393e87a0a0b9f4cb756198b53548a" - integrity sha512-n43JRhlUKUAlibEJhPeir1ncUID16QnEjNpwzNdO3Lm4ywrBpBZ5oLD0I6br9evr1Y9JTqwRtAh7JLoOzAQdVA== + version "1.9.0" + resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-1.9.0.tgz#5dc0753acbf8521ca2e0f137b4578b917b10cf24" + integrity sha512-xIp7/apCFJuUHdDLWe8O1HIkb0kQrOMb/0u6FXQjemHn/ii5LrIzU6bdECnsiTF/GjZkMEKg1xdiZwNqDYlZ6g== dependencies: isarray "0.0.1" @@ -10596,10 +10610,10 @@ vite-plugin-svgr@^4.1.0: "@svgr/core" "^8.1.0" "@svgr/plugin-jsx" "^8.1.0" -"vite@^3.0.0 || ^4.0.0 || ^5.0.0-0", "vite@^3.1.0 || ^4.0.0 || ^5.0.0-0", vite@^4.5.3: - version "4.5.3" - resolved "https://registry.yarnpkg.com/vite/-/vite-4.5.3.tgz#d88a4529ea58bae97294c7e2e6f0eab39a50fb1a" - integrity sha512-kQL23kMeX92v3ph7IauVkXkikdDRsYMGTVl5KY2E9OY4ONLvkHf04MDTbnfo6NKxZiDLWzVpP5oTa8hQD8U3dg== +"vite@^3.0.0 || ^4.0.0 || ^5.0.0-0", "vite@^3.1.0 || ^4.0.0 || ^5.0.0-0", vite@^4.5.5: + version "4.5.5" + resolved "https://registry.yarnpkg.com/vite/-/vite-4.5.5.tgz#639b9feca5c0a3bfe3c60cb630ef28bf219d742e" + integrity sha512-ifW3Lb2sMdX+WU91s3R0FyQlAyLxOzCSCP37ujw0+r5POeHPwe6udWVIElKQq8gk3t7b8rkmvqC6IHBpCff4GQ== dependencies: esbuild "^0.18.10" postcss "^8.4.27" diff --git a/docker/datahub-ingestion-base/Dockerfile b/docker/datahub-ingestion-base/Dockerfile index 8280b9685099f6..08cf2efdcb6a19 100644 --- a/docker/datahub-ingestion-base/Dockerfile +++ b/docker/datahub-ingestion-base/Dockerfile @@ -29,6 +29,7 @@ ENV UV_INDEX_URL=${PIP_MIRROR_URL} RUN apt-get update && apt-get upgrade -y \ && apt-get install -y -qq \ + lsb-release \ python3 \ python3-pip \ python3-venv \ @@ -42,19 +43,20 @@ RUN apt-get update && apt-get upgrade -y \ krb5-user \ krb5-config \ libkrb5-dev \ + librdkafka-dev \ wget \ + curl \ zip \ unzip \ ldap-utils \ unixodbc \ libodbc2 \ - && python -m pip install --no-cache --upgrade pip uv>=0.1.10 wheel setuptools \ + && python -m pip install --no-cache --upgrade pip 'uv>=0.1.10' wheel setuptools \ && apt-get clean \ && rm -rf /var/lib/{apt,dpkg,cache,log}/ -COPY --from=dockerize-binary /usr/local/bin/dockerize /usr/local/bin +COPY --from=powerman/dockerize:0.19 /usr/local/bin/dockerize /usr/local/bin -COPY ./docker/datahub-ingestion-base/base-requirements.txt requirements.txt COPY ./docker/datahub-ingestion-base/entrypoint.sh /entrypoint.sh RUN addgroup --gid 1000 datahub && \ @@ -66,7 +68,14 @@ ENV REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt ENV VIRTUAL_ENV=/datahub-ingestion/.venv ENV PATH="${VIRTUAL_ENV}/bin:$PATH" RUN python3 -m venv $VIRTUAL_ENV && \ - uv pip install --no-cache -r requirements.txt + uv pip install --no-cache --upgrade pip setuptools wheel + +# Note: Normally uv will create hardlinks from the cache directory to the venv. +# In our docker files, we normally use `RUN --mount=type=cache,... uv pip install ...`, +# which means the cache directory is on a separate filesystem. uv will emit a warning: +# Failed to hardlink files; falling back to full copy. This may lead to degraded performance. +# If the cache and target directories are on different filesystems, hardlinking may not be supported. +# If this is intentional, set `export UV_LINK_MODE=copy` or use `--link-mode=copy` to suppress this warning. ENTRYPOINT [ "/entrypoint.sh" ] diff --git a/docker/datahub-ingestion-base/base-requirements.txt b/docker/datahub-ingestion-base/base-requirements.txt deleted file mode 100644 index fa07b4184a6bc0..00000000000000 --- a/docker/datahub-ingestion-base/base-requirements.txt +++ /dev/null @@ -1,385 +0,0 @@ -# Generated requirements file. Run ./regenerate-base-requirements.sh to regenerate. -acryl-datahub-classify==0.0.11 -acryl-PyHive==0.6.16 -acryl-sqlglot==25.3.1.dev3 -aenum==3.1.15 -aiohappyeyeballs==2.3.2 -aiohttp==3.10.0 -aiosignal==1.3.1 -alembic==1.13.2 -altair==4.2.0 -anyio==4.4.0 -apache-airflow==2.9.3 -apache-airflow-providers-common-io==1.3.2 -apache-airflow-providers-common-sql==1.14.2 -apache-airflow-providers-fab==1.2.2 -apache-airflow-providers-ftp==3.10.0 -apache-airflow-providers-http==4.12.0 -apache-airflow-providers-imap==3.6.1 -apache-airflow-providers-smtp==1.7.1 -apache-airflow-providers-sqlite==3.8.1 -apispec==6.6.1 -appnope==0.1.4 -argcomplete==3.4.0 -argon2-cffi==23.1.0 -argon2-cffi-bindings==21.2.0 -asgiref==3.8.1 -asn1crypto==1.5.1 -asttokens==2.4.1 -async-timeout==4.0.3 -asynch==0.2.4 -attrs==23.2.0 -avro==1.11.3 -avro-gen3==0.7.13 -azure-common==1.1.28 -azure-core==1.29.4 -azure-identity==1.14.1 -azure-storage-blob==12.21.0 -azure-storage-file-datalake==12.16.0 -Babel==2.15.0 -backoff==2.2.1 -beautifulsoup4==4.12.3 -bleach==6.1.0 -blinker==1.8.2 -blis==0.7.11 -boto3==1.34.151 -botocore==1.34.151 -bracex==2.4 -cached-property==1.5.2 -cachelib==0.9.0 -cachetools==5.4.0 -catalogue==2.0.10 -cattrs==23.2.3 -certifi==2024.7.4 -cffi==1.16.0 -chardet==5.2.0 -charset-normalizer==3.3.2 -ciso8601==2.3.1 -click==8.1.7 -click-default-group==1.2.4 -click-spinner==0.1.10 -clickclick==20.10.2 -clickhouse-driver==0.2.8 -clickhouse-sqlalchemy==0.2.4 -cloudpathlib==0.18.1 -cloudpickle==3.0.0 -colorama==0.4.6 -colorlog==4.8.0 -comm==0.2.2 -confection==0.1.5 -ConfigUpdater==3.2 -confluent-kafka==2.5.0 -connexion==2.14.2 -cron-descriptor==1.4.3 -croniter==3.0.3 -cryptography==42.0.8 -cx_Oracle==8.3.0 -cymem==2.0.8 -databricks-dbapi==0.6.0 -databricks-sdk==0.29.0 -databricks-sql-connector==2.9.6 -dataflows-tabulator==1.54.3 -db-dtypes==1.2.0 -debugpy==1.8.2 -decorator==5.1.1 -defusedxml==0.7.1 -deltalake==0.17.4 -Deprecated==1.2.14 -dill==0.3.8 -dnspython==2.6.1 -docker==7.1.0 -docutils==0.21.2 -ecdsa==0.19.0 -elasticsearch==7.13.4 -email_validator==2.2.0 -entrypoints==0.4 -et-xmlfile==1.1.0 -exceptiongroup==1.2.2 -executing==2.0.1 -expandvars==0.12.0 -fastavro==1.9.5 -fastjsonschema==2.20.0 -filelock==3.15.4 -Flask==2.2.5 -flatdict==4.0.1 -frozenlist==1.4.1 -fsspec==2023.12.2 -future==1.0.0 -GeoAlchemy2==0.15.2 -gitdb==4.0.11 -GitPython==3.1.43 -google-api-core==2.19.1 -google-auth==2.32.0 -google-cloud-appengine-logging==1.4.5 -google-cloud-audit-log==0.2.5 -google-cloud-bigquery==3.25.0 -google-cloud-core==2.4.1 -google-cloud-datacatalog==3.20.0 -google-cloud-datacatalog-lineage==0.2.2 -google-cloud-logging==3.5.0 -google-crc32c==1.5.0 -google-re2==1.1.20240702 -google-resumable-media==2.7.1 -googleapis-common-protos==1.63.2 -gql==3.5.0 -graphql-core==3.2.3 -great-expectations==0.15.50 -greenlet==3.0.3 -grpc-google-iam-v1==0.13.1 -grpcio==1.65.2 -grpcio-status==1.62.2 -grpcio-tools==1.62.2 -gssapi==1.8.3 -gunicorn==22.0.0 -h11==0.14.0 -httpcore==1.0.5 -httpx==0.27.0 -humanfriendly==10.0 -idna==3.7 -ijson==3.3.0 -importlib_metadata==7.2.1 -importlib_resources==6.4.0 -inflection==0.5.1 -ipaddress==1.0.23 -ipykernel==6.17.1 -ipython==8.21.0 -ipython-genutils==0.2.0 -ipywidgets==8.1.3 -isodate==0.6.1 -itsdangerous==2.2.0 -jedi==0.19.1 -Jinja2==3.1.4 -jmespath==1.0.1 -JPype1==1.5.0 -jsonlines==4.0.0 -jsonpatch==1.33 -jsonpointer==3.0.0 -jsonref==1.1.0 -jsonschema==4.23.0 -jsonschema-specifications==2023.12.1 -jupyter-server==1.16.0 -jupyter_client==7.4.9 -jupyter_core==4.12.0 -jupyterlab_pygments==0.3.0 -jupyterlab_widgets==3.0.11 -langcodes==3.4.0 -language_data==1.2.0 -lark==1.1.4 -lazy-object-proxy==1.10.0 -leb128==1.0.8 -limits==3.13.0 -linear-tsv==1.1.0 -linkify-it-py==2.0.3 -lkml==1.3.5 -lockfile==0.12.2 -looker-sdk==23.0.0 -lxml==5.2.2 -lz4==4.3.3 -makefun==1.15.4 -Mako==1.3.5 -marisa-trie==1.2.0 -markdown-it-py==3.0.0 -MarkupSafe==2.1.5 -marshmallow==3.21.3 -marshmallow-oneofschema==3.1.1 -marshmallow-sqlalchemy==0.28.2 -matplotlib-inline==0.1.7 -mdit-py-plugins==0.4.1 -mdurl==0.1.2 -methodtools==0.4.7 -mistune==3.0.2 -mixpanel==4.10.1 -mlflow-skinny==2.15.0 -mmhash3==3.0.1 -more-itertools==10.3.0 -moto==4.2.14 -msal==1.22.0 -msal-extensions==1.1.0 -multidict==6.0.5 -murmurhash==1.0.10 -mypy-extensions==1.0.0 -nbclassic==1.1.0 -nbclient==0.6.3 -nbconvert==7.16.4 -nbformat==5.10.4 -nest-asyncio==1.6.0 -networkx==3.3 -notebook==6.5.7 -notebook_shim==0.2.4 -numpy==1.26.4 -oauthlib==3.2.2 -okta==1.7.0 -openlineage-airflow==1.18.0 -openlineage-integration-common==1.18.0 -openlineage-python==1.18.0 -openlineage_sql==1.18.0 -openpyxl==3.1.5 -opentelemetry-api==1.26.0 -opentelemetry-exporter-otlp==1.26.0 -opentelemetry-exporter-otlp-proto-common==1.26.0 -opentelemetry-exporter-otlp-proto-grpc==1.26.0 -opentelemetry-exporter-otlp-proto-http==1.26.0 -opentelemetry-proto==1.26.0 -opentelemetry-sdk==1.26.0 -opentelemetry-semantic-conventions==0.47b0 -ordered-set==4.1.0 -packaging==24.1 -pandas==2.1.4 -pandocfilters==1.5.1 -parse==1.20.2 -parso==0.8.4 -pathspec==0.12.1 -pendulum==3.0.0 -pexpect==4.9.0 -phonenumbers==8.13.0 -platformdirs==4.2.2 -pluggy==1.5.0 -portalocker==2.10.1 -preshed==3.0.9 -prison==0.2.1 -progressbar2==4.4.2 -prometheus_client==0.20.0 -prompt_toolkit==3.0.47 -proto-plus==1.24.0 -protobuf==4.25.4 -psutil==6.0.0 -psycopg2-binary==2.9.9 -ptyprocess==0.7.0 -pure-sasl==0.6.2 -pure_eval==0.2.3 -py-partiql-parser==0.5.0 -pyarrow==17.0.0 -pyarrow-hotfix==0.6 -pyasn1==0.6.0 -pyasn1_modules==0.4.0 -pyathena==2.25.2 -pycountry==24.6.1 -pycparser==2.22 -pycryptodome==3.20.0 -pydantic==1.10.17 -pydash==8.0.3 -pydruid==0.6.9 -Pygments==2.18.0 -pyiceberg==0.4.0 -pymongo==4.8.0 -PyMySQL==1.1.1 -pyOpenSSL==24.2.1 -pyparsing==3.0.9 -pyspnego==0.11.1 -python-daemon==3.0.1 -python-dateutil==2.9.0.post0 -python-jose==3.3.0 -python-ldap==3.4.4 -python-liquid==1.12.1 -python-nvd3==0.16.0 -python-slugify==8.0.4 -python-stdnum==1.20 -python-tds==1.15.0 -python-utils==3.8.2 -pytz==2024.1 -PyYAML==6.0.1 -pyzmq==26.0.3 -redash-toolbelt==0.1.9 -redshift-connector==2.1.2 -referencing==0.35.1 -regex==2024.7.24 -requests==2.32.3 -requests-file==2.1.0 -requests-gssapi==1.3.0 -requests-toolbelt==1.0.0 -requests_ntlm==1.3.0 -responses==0.25.3 -rfc3339-validator==0.1.4 -rfc3986==2.0.0 -rich==13.7.1 -rich-argparse==1.5.2 -rpds-py==0.19.1 -rsa==4.9 -rstr==3.2.2 -ruamel.yaml==0.17.17 -s3transfer==0.10.2 -schwifty==2024.6.1 -scipy==1.14.0 -scramp==1.4.5 -Send2Trash==1.8.3 -sentry-sdk==2.12.0 -setproctitle==1.3.3 -shellingham==1.5.4 -simple-salesforce==1.12.6 -six==1.16.0 -slack-sdk==3.18.1 -smart-open==7.0.4 -smmap==5.0.1 -sniffio==1.3.1 -snowflake-connector-python==3.12.0 -snowflake-sqlalchemy==1.6.1 -sortedcontainers==2.4.0 -soupsieve==2.5 -spacy==3.7.5 -spacy-legacy==3.0.12 -spacy-loggers==1.0.5 -sql_metadata==2.12.0 -SQLAlchemy==1.4.44 -sqlalchemy-bigquery==1.11.0 -sqlalchemy-cockroachdb==1.4.4 -SQLAlchemy-JSONField==1.0.2 -sqlalchemy-pytds==0.3.5 -sqlalchemy-redshift==0.8.14 -SQLAlchemy-Utils==0.41.2 -sqlglotrs==0.2.7 -sqllineage==1.3.8 -sqlparse==0.4.4 -srsly==2.4.8 -stack-data==0.6.3 -strictyaml==1.7.3 -tableauserverclient==0.25 -tableschema==1.20.11 -tabulate==0.9.0 -tenacity==9.0.0 -teradatasql==20.0.0.14 -teradatasqlalchemy==20.0.0.1 -termcolor==2.4.0 -terminado==0.18.1 -text-unidecode==1.3 -thinc==8.2.5 -thrift==0.16.0 -thrift-sasl==0.4.3 -time-machine==2.14.2 -tinycss2==1.3.0 -toml==0.10.2 -tomlkit==0.13.0 -toolz==0.12.1 -tornado==6.4.1 -tqdm==4.66.4 -traitlets==5.2.1.post0 -trino==0.329.0 -typer==0.12.3 -typing-inspect==0.9.0 -typing_extensions==4.12.2 -tzdata==2024.1 -tzlocal==5.2 -uc-micro-py==1.0.3 -ujson==5.10.0 -unicodecsv==0.14.1 -universal_pathlib==0.2.2 -urllib3==1.26.19 -vertica-python==1.4.0 -vertica-sqlalchemy-dialect==0.0.8.2 -vininfo==1.8.0 -wasabi==1.1.3 -wcmatch==8.5.2 -wcwidth==0.2.13 -weasel==0.4.1 -webencodings==0.5.1 -websocket-client==1.8.0 -Werkzeug==2.2.3 -widgetsnbextension==4.0.11 -wirerope==0.4.7 -wrapt==1.16.0 -WTForms==3.1.2 -xlrd==2.0.1 -xmltodict==0.13.0 -yarl==1.9.4 -zeep==4.2.1 -zipp==3.19.2 -zstd==1.5.5.1 diff --git a/docker/datahub-ingestion-base/build.gradle b/docker/datahub-ingestion-base/build.gradle index 3f65fd7410acec..ef482de9256a33 100644 --- a/docker/datahub-ingestion-base/build.gradle +++ b/docker/datahub-ingestion-base/build.gradle @@ -12,7 +12,7 @@ ext { docker_target = project.getProperties().getOrDefault("dockerTarget", "slim") docker_version = "${version}${docker_target == 'slim' ? '-slim' : ''}" - revision = 5 // increment to trigger rebuild + revision = 7 // increment to trigger rebuild } docker { diff --git a/docker/datahub-ingestion-base/regenerate-base-requirements.sh b/docker/datahub-ingestion-base/regenerate-base-requirements.sh deleted file mode 100755 index 13d74922d9013b..00000000000000 --- a/docker/datahub-ingestion-base/regenerate-base-requirements.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - -# This script is used to regenerate the base-requirements.txt file - -set -euxo pipefail -cd "$( dirname "${BASH_SOURCE[0]}" )" - -SCRIPT_NAME=$(basename "$0") -DATAHUB_DIR=$(pwd)/../.. - -# Create a virtualenv. -VENV_DIR=$(mktemp -d) -python -c "import sys; assert sys.version_info >= (3, 9), 'Python 3.9 or higher is required.'" -python -m venv $VENV_DIR -source $VENV_DIR/bin/activate -pip install --upgrade pip uv setuptools wheel -echo "Using virtualenv at $VENV_DIR" - -# Install stuff. -pushd $DATAHUB_DIR/metadata-ingestion -uv pip install -e '.[all]' -e '../metadata-ingestion-modules/airflow-plugin/[plugin-v2]' -popd - -# Generate the requirements file. -# Removing Flask deps due as per https://github.com/datahub-project/datahub/pull/6867/files -# Removing py4j and PyJWT due to https://github.com/datahub-project/datahub/pull/6868/files -# Removing pyspark and pydeequ because we don't want them in the slim image, so they can be added separately. -# TODO: It's unclear if these removals are still actually needed. -echo "# Generated requirements file. Run ./$SCRIPT_NAME to regenerate." > base-requirements.txt -pip freeze \ - | grep -v -E "^-e" \ - | grep -v -E "^uv==" \ - | grep -v "Flask-" \ - | grep -v -E "(py4j|PyJWT)==" \ - | grep -v -E "(pyspark|pydeequ)==" \ - >> base-requirements.txt diff --git a/docker/datahub-ingestion/Dockerfile b/docker/datahub-ingestion/Dockerfile index 1a1e57f42c76f4..ee0333e1cb1d1f 100644 --- a/docker/datahub-ingestion/Dockerfile +++ b/docker/datahub-ingestion/Dockerfile @@ -21,11 +21,11 @@ ARG PIP_MIRROR_URL RUN if [ "${PIP_MIRROR_URL}" != "https://pypi.python.org/simple" ] ; then pip config set global.index-url ${PIP_MIRROR_URL} ; fi ENV UV_INDEX_URL=${PIP_MIRROR_URL} -COPY --chown=datahub ./metadata-ingestion /datahub-ingestion -COPY --chown=datahub ./metadata-ingestion-modules/airflow-plugin /datahub-ingestion/airflow-plugin +COPY --chown=datahub ./metadata-ingestion /metadata-ingestion +COPY --chown=datahub ./metadata-ingestion-modules/airflow-plugin /metadata-ingestion/airflow-plugin ARG RELEASE_VERSION -WORKDIR /datahub-ingestion +WORKDIR /metadata-ingestion RUN sed -i.bak "s/__version__ = \"1\!0.0.0.dev0\"/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" src/datahub/__init__.py && \ sed -i.bak "s/__version__ = \"1\!0.0.0.dev0\"/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" airflow-plugin/src/datahub_airflow_plugin/__init__.py && \ cat src/datahub/__init__.py | grep __version__ && \ @@ -33,7 +33,8 @@ RUN sed -i.bak "s/__version__ = \"1\!0.0.0.dev0\"/__version__ = \"$(echo $RELEAS FROM base AS slim-install -RUN uv pip install --no-cache -e ".[base,datahub-rest,datahub-kafka,snowflake,bigquery,redshift,mysql,postgres,hive,clickhouse,glue,dbt,looker,lookml,tableau,powerbi,superset,datahub-business-glossary]" +RUN --mount=type=cache,target=/datahub-ingestion/.cache/uv,uid=1000,gid=1000 \ + UV_LINK_MODE=copy uv pip install -e ".[base,datahub-rest,datahub-kafka,snowflake,bigquery,redshift,mysql,postgres,hive,clickhouse,glue,dbt,looker,lookml,tableau,powerbi,superset,datahub-business-glossary]" FROM base AS full-install-build @@ -43,9 +44,10 @@ RUN apt-get update && apt-get install -y -qq maven USER datahub COPY ./docker/datahub-ingestion/pyspark_jars.sh . -RUN uv pip install --no-cache -e ".[base,all]" "./airflow-plugin[plugin-v2]" && \ +RUN --mount=type=cache,target=/datahub-ingestion/.cache/uv,uid=1000,gid=1000 \ + UV_LINK_MODE=copy uv pip install -e ".[base,all]" "./airflow-plugin[plugin-v2]" && \ + ./pyspark_jars.sh && \ datahub --version -RUN ./pyspark_jars.sh FROM base AS full-install @@ -57,4 +59,6 @@ FROM base AS dev-install FROM ${APP_ENV}-install AS final +WORKDIR /datahub-ingestion + USER datahub diff --git a/docker/datahub-ingestion/Dockerfile-slim-only b/docker/datahub-ingestion/Dockerfile-slim-only index f17c8df63ae9d3..6ade262f2feded 100644 --- a/docker/datahub-ingestion/Dockerfile-slim-only +++ b/docker/datahub-ingestion/Dockerfile-slim-only @@ -11,18 +11,21 @@ ARG PIP_MIRROR_URL RUN if [ "${PIP_MIRROR_URL}" != "https://pypi.python.org/simple" ] ; then pip config set global.index-url ${PIP_MIRROR_URL} ; fi ENV UV_INDEX_URL=${PIP_MIRROR_URL} -COPY --chown=datahub ./metadata-ingestion /datahub-ingestion +COPY --chown=datahub ./metadata-ingestion /metadata-ingestion ARG RELEASE_VERSION -WORKDIR /datahub-ingestion +WORKDIR /metadata-ingestion RUN sed -i.bak "s/__version__ = \"1\!0.0.0.dev0\"/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" src/datahub/__init__.py && \ cat src/datahub/__init__.py FROM base as slim-install -RUN uv pip install --no-cache -e ".[base,datahub-rest,datahub-kafka,snowflake,bigquery,redshift,mysql,postgres,hive,clickhouse,glue,dbt,looker,lookml,tableau,powerbi,superset,datahub-business-glossary]" && \ +RUN --mount=type=cache,target=/datahub-ingestion/.cache/uv,uid=1000,gid=1000 \ + UV_LINK_MODE=copy uv pip install -e ".[base,datahub-rest,datahub-kafka,snowflake,bigquery,redshift,mysql,postgres,hive,clickhouse,glue,dbt,looker,lookml,tableau,powerbi,superset,datahub-business-glossary]" && \ datahub --version FROM slim-install as final +WORKDIR /datahub-ingestion + USER datahub diff --git a/docker/datahub-ingestion/build.gradle b/docker/datahub-ingestion/build.gradle index 3161fa2353e224..113a6dcf0a1bd4 100644 --- a/docker/datahub-ingestion/build.gradle +++ b/docker/datahub-ingestion/build.gradle @@ -12,7 +12,7 @@ ext { docker_target = project.getProperties().getOrDefault("dockerTarget", "slim") docker_version = "${version}${docker_target == 'slim' ? '-slim' : ''}" - revision = 5 // increment to trigger rebuild + revision = 8 // increment to trigger rebuild } dependencies { diff --git a/docker/kafka-setup/kafka-config.sh b/docker/kafka-setup/kafka-config.sh index 4d5698ccc3856f..cc714fa53b84df 100644 --- a/docker/kafka-setup/kafka-config.sh +++ b/docker/kafka-setup/kafka-config.sh @@ -8,7 +8,8 @@ : ${DATAHUB_ANALYTICS_ENABLED:=true} -export KAFKA_HEAP_OPTS="-Xmx64M" +: ${KAFKA_HEAP_OPTS:=-Xmx64M} +export KAFKA_HEAP_OPTS CONNECTION_PROPERTIES_PATH=/tmp/connection.properties diff --git a/docker/monitoring/grafana/dashboards/datahub_dashboard.json b/docker/monitoring/grafana/dashboards/datahub_dashboard.json index 3b932756e5e117..352f84829b0923 100644 --- a/docker/monitoring/grafana/dashboards/datahub_dashboard.json +++ b/docker/monitoring/grafana/dashboards/datahub_dashboard.json @@ -24,7 +24,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 95, + "id": 56, "links": [], "liveNow": false, "panels": [ @@ -65,6 +65,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -78,6 +79,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -136,10 +138,12 @@ "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "increase(metrics_com_linkedin_metadata_resources_entity_EntityResource_get_Count{}[1m])/60", + "expr": "increase(metrics_com_linkedin_metadata_resources_entity_EntityV2Resource_get_Count{}[1m])/60", "interval": "", "legendFormat": "Get QPS", + "range": true, "refId": "A" }, { @@ -147,11 +151,13 @@ "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "increase(metrics_com_linkedin_metadata_resources_entity_EntityResource_get_failed_Count{}[1m])/60", + "expr": "increase(metrics_com_linkedin_metadata_resources_entity_EntityV2Resource_get_failed_Count{}[1m])/60", "hide": false, "interval": "", "legendFormat": "Get Failure", + "range": true, "refId": "B" }, { @@ -159,11 +165,13 @@ "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "increase(metrics_com_linkedin_metadata_resources_entity_EntityResource_batchGet_Count{}[1m])/60", + "expr": "increase(metrics_com_linkedin_metadata_resources_entity_EntityV2Resource_batchGet_Count{}[1m])/60", "hide": false, "interval": "", "legendFormat": "BatchGet QPS", + "range": true, "refId": "C" }, { @@ -171,11 +179,13 @@ "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "increase(metrics_com_linkedin_metadata_resources_entity_EntityResource_batchGet_failed_Count{}[1m])/60", + "expr": "increase(metrics_com_linkedin_metadata_resources_entity_EntityV2Resource_batchGet_failed_Count{}[1m])/60", "hide": false, "interval": "", "legendFormat": "BatchGet Failure", + "range": true, "refId": "D" } ], @@ -193,6 +203,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -206,6 +217,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -264,10 +276,12 @@ "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "metrics_com_linkedin_metadata_resources_entity_EntityResource_get_Mean{}", + "expr": "metrics_com_linkedin_metadata_resources_entity_EntityV2Resource_get_Mean{}", "interval": "", "legendFormat": "Get Avg", + "range": true, "refId": "A" }, { @@ -275,11 +289,13 @@ "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "metrics_com_linkedin_metadata_resources_entity_EntityResource_get_75thPercentile{}", + "expr": "metrics_com_linkedin_metadata_resources_entity_EntityV2Resource_get_75thPercentile{}", "hide": false, "interval": "", "legendFormat": "Get P75", + "range": true, "refId": "B" }, { @@ -287,11 +303,13 @@ "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "metrics_com_linkedin_metadata_resources_entity_EntityResource_get_95thPercentile{}", + "expr": "metrics_com_linkedin_metadata_resources_entity_EntityV2Resource_get_95thPercentile{}", "hide": false, "interval": "", "legendFormat": "Get P95", + "range": true, "refId": "C" }, { @@ -299,11 +317,13 @@ "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "metrics_com_linkedin_metadata_resources_entity_EntityResource_batchGet_Mean{}", + "expr": "metrics_com_linkedin_metadata_resources_entity_EntityV2Resource_batchGet_Mean{}", "hide": false, "interval": "", "legendFormat": "BatchGet Avg", + "range": true, "refId": "D" }, { @@ -311,11 +331,13 @@ "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "metrics_com_linkedin_metadata_resources_entity_EntityResource_batchGet_75thPercentile{}", + "expr": "metrics_com_linkedin_metadata_resources_entity_EntityV2Resource_batchGet_75thPercentile{}", "hide": false, "interval": "", "legendFormat": "BatchGet P75", + "range": true, "refId": "E" }, { @@ -323,11 +345,13 @@ "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "metrics_com_linkedin_metadata_resources_entity_EntityResource_batchGet_95thPercentile{}", + "expr": "metrics_com_linkedin_metadata_resources_entity_EntityV2Resource_batchGet_95thPercentile{}", "hide": false, "interval": "", "legendFormat": "BatchGet P95", + "range": true, "refId": "F" } ], @@ -371,6 +395,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -384,6 +409,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -442,20 +468,9 @@ "type": "prometheus", "uid": "${datasource}" }, - "exemplar": true, - "expr": "increase(metrics_com_linkedin_metadata_resources_entity_EntityResource_ingest_Count{}[1m])/60", - "hide": false, - "interval": "", - "legendFormat": "Ingest Count", - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, + "editorMode": "code", "exemplar": false, - "expr": "increase(metrics_com_linkedin_metadata_resources_entity_EntityResource_batchIngest_Count{}[1m])/60", + "expr": "increase(metrics_com_linkedin_metadata_resources_entity_AspectResource_ingestProposal_Count{}[1m])/60", "hide": false, "instant": false, "interval": "", @@ -467,24 +482,28 @@ "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "increase(metrics_com_linkedin_metadata_resources_entity_EntityResource_ingest_failed_Count[1m])/60", + "expr": "increase(metrics_com_linkedin_metadata_resources_entity_AspectResource_ingestProposal_success_Count{}[1m])/60", "hide": false, "interval": "", - "legendFormat": "Ingest Failure", - "refId": "C" + "legendFormat": "Ingest Count", + "range": true, + "refId": "E" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "builder", "exemplar": true, - "expr": "increase(metrics_com_linkedin_metadata_resources_entity_EntityResource_batchIngest_failed_Count[1m])/60", - "hide": false, + "expr": "increase(metrics_com_linkedin_metadata_resources_entity_AspectResource_ingestProposal_failed_Count{}[1m])/60", + "hide": true, "interval": "", - "legendFormat": "BatchIngest Failure", - "refId": "D" + "legendFormat": "Ingest Failure", + "range": true, + "refId": "C" } ], "title": "Ingest QPS", @@ -501,6 +520,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -514,6 +534,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -583,11 +604,13 @@ "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, "expr": "metrics_com_linkedin_metadata_resources_entity_EntityResource_ingest_75thPercentile{}", "hide": false, "interval": "", "legendFormat": "P75", + "range": true, "refId": "B" }, { @@ -595,11 +618,13 @@ "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "metrics_com_linkedin_metadata_resources_entity_EntityResource_ingest_95thPercentile{}", + "expr": "metrics_com_linkedin_metadata_resources_entity_AspectResource_ingestProposal_95thPercentile{}", "hide": false, "interval": "", "legendFormat": "P95", + "range": true, "refId": "C" } ], @@ -617,6 +642,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -630,6 +656,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -688,11 +715,13 @@ "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "metrics_com_linkedin_metadata_entity_EntityService_ingestAspectsToLocalDB_Mean{}", + "expr": "metrics_com_linkedin_metadata_entity_EntityServiceImpl_ingestAspectsToLocalDB_MeanRate{}", "hide": false, "interval": "", "legendFormat": "Ingest To DB", + "range": true, "refId": "B" }, { @@ -700,11 +729,13 @@ "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, "expr": "metrics_com_linkedin_metadata_entity_EntityService_produceMAE_Mean{}", - "hide": false, + "hide": true, "interval": "", "legendFormat": "Produce MAE", + "range": true, "refId": "C" } ], @@ -722,6 +753,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -735,6 +767,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -793,10 +826,12 @@ "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "metrics_com_linkedin_metadata_kafka_MetadataChangeLogProcessor_maeProcess_Mean", + "expr": "metrics_postEntity_MeanRate", "interval": "", "legendFormat": "Avg", + "range": true, "refId": "A" }, { @@ -804,11 +839,13 @@ "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "builder", "exemplar": true, "expr": "metrics_com_linkedin_metadata_kafka_MetadataAuditEventsProcessor_maeProcess_75thPercentile", - "hide": false, + "hide": true, "interval": "", "legendFormat": "P75", + "range": true, "refId": "B" }, { @@ -816,11 +853,13 @@ "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, "expr": "metrics_com_linkedin_metadata_kafka_MetadataAuditEventsProcessor_maeProcess_95thPercentile", - "hide": false, + "hide": true, "interval": "", "legendFormat": "P95", + "range": true, "refId": "C" } ], @@ -838,6 +877,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -851,6 +891,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -958,6 +999,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -971,6 +1013,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1077,6 +1120,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -1090,6 +1134,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1148,9 +1193,9 @@ "type": "prometheus", "uid": "${datasource}" }, - "editorMode": "builder", + "editorMode": "code", "exemplar": true, - "expr": "metrics_com_linkedin_metadata_kafka_MetadataChangeLogProcessor_MetadataTestHook_latency_Mean", + "expr": "metrics_com_linkedin_metadata_kafka_MetadataChangeLogProcessor_UpdateIndicesHook_latency_Mean", "legendFormat": "Avg", "range": true, "refId": "A" @@ -1160,9 +1205,9 @@ "type": "prometheus", "uid": "${datasource}" }, - "editorMode": "builder", + "editorMode": "code", "exemplar": true, - "expr": "metrics_com_linkedin_metadata_kafka_MetadataChangeLogProcessor_MetadataTestHook_latency_75thPercentile", + "expr": "metrics_com_linkedin_metadata_kafka_MetadataChangeLogProcessor_UpdateIndicesHook_latency_75thPercentile", "hide": false, "legendFormat": "P75", "range": true, @@ -1173,16 +1218,16 @@ "type": "prometheus", "uid": "${datasource}" }, - "editorMode": "builder", + "editorMode": "code", "exemplar": true, - "expr": "metrics_com_linkedin_metadata_kafka_MetadataChangeLogProcessor_MetadataTestHook_latency_95thPercentile", + "expr": "metrics_com_linkedin_metadata_kafka_MetadataChangeLogProcessor_UpdateIndicesHook_latency_95thPercentile", "hide": false, "legendFormat": "P95", "range": true, "refId": "C" } ], - "title": "Metadata Test Latency", + "title": "Metadata Update Latency", "type": "timeseries" }, { @@ -1196,6 +1241,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -1209,6 +1255,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1248,7 +1295,7 @@ "x": 7, "y": 26 }, - "id": 51, + "id": 56, "options": { "legend": { "calcs": [], @@ -1267,9 +1314,10 @@ "type": "prometheus", "uid": "${datasource}" }, - "editorMode": "builder", + "editorMode": "code", "exemplar": true, - "expr": "metrics_com_linkedin_metadata_kafka_MetadataChangeLogProcessor_NotificationGeneratorHook_latency_Mean", + "expr": "metrics_com_linkedin_metadata_resources_entity_EntityResource_delete_Mean{}", + "interval": "", "legendFormat": "Avg", "range": true, "refId": "A" @@ -1279,10 +1327,24 @@ "type": "prometheus", "uid": "${datasource}" }, - "editorMode": "builder", + "editorMode": "code", + "expr": "metrics_com_linkedin_metadata_resources_entity_EntityResource_delete_StdDev{}", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", "exemplar": true, - "expr": "metrics_com_linkedin_metadata_kafka_MetadataChangeLogProcessor_NotificationGeneratorHook_latency_75thPercentile", + "expr": "metrics_com_linkedin_metadata_resources_entity_EntityResource_delete_75thPercentile{}", "hide": false, + "interval": "", "legendFormat": "P75", "range": true, "refId": "B" @@ -1292,16 +1354,17 @@ "type": "prometheus", "uid": "${datasource}" }, - "editorMode": "builder", + "editorMode": "code", "exemplar": true, - "expr": "metrics_com_linkedin_metadata_kafka_MetadataChangeLogProcessor_NotificationGeneratorHook_latency_95thPercentile", + "expr": "metrics_com_linkedin_metadata_resources_entity_EntityResource_delete_95thPercentile{}", "hide": false, + "interval": "", "legendFormat": "P95", "range": true, "refId": "C" } ], - "title": "Notification Generator latency", + "title": "Delete Latency", "type": "timeseries" }, { @@ -1315,6 +1378,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -1328,6 +1392,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1434,6 +1499,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -1447,6 +1513,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1580,6 +1647,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -1593,6 +1661,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1651,23 +1720,13 @@ "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "increase(metrics_com_linkedin_metadata_resources_entity_EntityResource_search_Count{}[1m])/60", + "expr": "increase(metrics_com_linkedin_metadata_search_elasticsearch_query_ESSearchDAO_searchRequest_Count{}[1m])/60", "interval": "", "legendFormat": "QPS", + "range": true, "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "exemplar": true, - "expr": "increase(metrics_com_linkedin_metadata_resources_entity_EntityResource_search_failed_Count{}[1m])/60", - "hide": false, - "interval": "", - "legendFormat": "Failure", - "refId": "B" } ], "title": "Search QPS", @@ -1684,6 +1743,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -1697,6 +1757,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1755,10 +1816,12 @@ "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "metrics_com_linkedin_metadata_resources_entity_EntityResource_search_Mean{}", + "expr": "metrics_com_linkedin_metadata_search_elasticsearch_query_ESSearchDAO_searchRequest_Mean{}", "interval": "", "legendFormat": "Avg", + "range": true, "refId": "A" }, { @@ -1766,11 +1829,13 @@ "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "metrics_com_linkedin_metadata_resources_entity_EntityResource_search_75thPercentile{}", + "expr": "metrics_com_linkedin_metadata_search_elasticsearch_query_ESSearchDAO_searchRequest_75thPercentile{}", "hide": false, "interval": "", "legendFormat": "P75", + "range": true, "refId": "B" }, { @@ -1778,11 +1843,13 @@ "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "metrics_com_linkedin_metadata_resources_entity_EntityResource_search_95thPercentile{}", + "expr": "metrics_com_linkedin_metadata_search_elasticsearch_query_ESSearchDAO_searchRequest_95thPercentile{}", "hide": false, "interval": "", "legendFormat": "P95", + "range": true, "refId": "C" } ], @@ -1800,6 +1867,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -1813,6 +1881,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1943,6 +2012,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -1956,6 +2026,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2014,23 +2085,17 @@ "type": "prometheus", "uid": "${datasource}" }, + "disableTextWrap": false, + "editorMode": "code", "exemplar": true, - "expr": "increase(metrics_com_linkedin_metadata_resources_entity_EntityResource_browse_Count{}[1m])/60", + "expr": "increase(metrics_com_datahub_graphql_GraphQLController_browseV2_Count[1m]) / 60", + "fullMetaSearch": false, + "includeNullMetadata": true, "interval": "", "legendFormat": "QPS", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "exemplar": true, - "expr": "increase(metrics_com_linkedin_metadata_resources_entity_EntityResource_browse_failed_Count{}[1m])/60", - "hide": false, - "interval": "", - "legendFormat": "Failure", - "refId": "B" + "range": true, + "refId": "A", + "useBackend": false } ], "title": "Browse QPS", @@ -2047,6 +2112,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -2060,6 +2126,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2118,10 +2185,13 @@ "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "metrics_com_datahub_graphql_GraphQLController_browse_Mean{}", + "expr": "metrics_com_datahub_graphql_GraphQLController_browseV2_Mean{}", + "hide": false, "interval": "", "legendFormat": "Avg", + "range": true, "refId": "A" }, { @@ -2129,23 +2199,31 @@ "type": "prometheus", "uid": "${datasource}" }, + "disableTextWrap": false, + "editorMode": "code", "exemplar": true, - "expr": "metrics_com_datahub_graphql_GraphQLController_browse_75thPercentile{}", + "expr": "metrics_com_datahub_graphql_GraphQLController_browseV2_75thPercentile{}", + "fullMetaSearch": false, "hide": false, + "includeNullMetadata": true, "interval": "", "legendFormat": "P75", - "refId": "B" + "range": true, + "refId": "B", + "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "metrics_com_datahub_graphql_GraphQLController_browse_95thPercentile{}", + "expr": "metrics_com_datahub_graphql_GraphQLController_browseV2_95thPercentile{}", "hide": false, "interval": "", "legendFormat": "P95", + "range": true, "refId": "C" } ], @@ -2163,6 +2241,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -2176,6 +2255,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2306,6 +2386,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -2319,6 +2400,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2377,47 +2459,14 @@ "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "increase(metrics_com_linkedin_metadata_resources_lineage_Relationships_getLineage_Count{}[1m])/60", + "expr": "increase(metrics_com_linkedin_metadata_graph_elastic_ESGraphQueryDAO_esQuery_Count{}[1m])/60", + "hide": false, "interval": "", "legendFormat": "Relationships QPS", + "range": true, "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "exemplar": true, - "expr": "increase(metrics_com_linkedin_metadata_resources_lineage_Relationships_getLineage_Count{}[1m])/60", - "hide": false, - "interval": "", - "legendFormat": "Lineage QPS", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "exemplar": true, - "expr": "increase(metrics_com_linkedin_metadata_resources_lineage_Relationships_getLineage_failed_Count{}[1m])/60", - "hide": false, - "interval": "", - "legendFormat": "Relationships Failure", - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "exemplar": true, - "expr": "increase(metrics_com_linkedin_metadata_resources_lineage_Lineage_get_failed_Count{}[1m])/60", - "hide": false, - "interval": "", - "legendFormat": "Lineage Failure", - "refId": "D" } ], "title": "Graph QPS", @@ -2434,6 +2483,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -2447,6 +2497,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2505,10 +2556,12 @@ "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "metrics_com_linkedin_metadata_resources_lineage_Relationships_getLineage_Mean{}", + "expr": "metrics_com_linkedin_metadata_entity_validation_ValidationUtils_validateLineageResult_Mean{}", "interval": "", "legendFormat": "Avg", + "range": true, "refId": "A" }, { @@ -2516,11 +2569,13 @@ "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "metrics_com_linkedin_metadata_resources_lineage_Relationships_getLineage_75thPercentile{}", + "expr": "metrics_com_linkedin_metadata_entity_validation_ValidationUtils_validateLineageResult_75thPercentile{}", "hide": false, "interval": "", "legendFormat": "P75", + "range": true, "refId": "B" }, { @@ -2528,11 +2583,13 @@ "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "metrics_com_linkedin_metadata_resources_lineage_Relationships_getLineage_95thPercentile{}", + "expr": "metrics_com_linkedin_metadata_entity_validation_ValidationUtils_validateLineageResult_95thPercentile{}", "hide": false, "interval": "", "legendFormat": "P95", + "range": true, "refId": "C" } ], @@ -2576,6 +2633,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -2589,6 +2647,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2668,6 +2727,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -2681,6 +2741,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2760,6 +2821,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -2773,6 +2835,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2876,6 +2939,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -2889,6 +2953,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2992,6 +3057,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -3005,6 +3071,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -3108,6 +3175,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -3121,6 +3189,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -3224,6 +3293,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -3237,6 +3307,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -3331,16 +3402,15 @@ } ], "refresh": "5s", - "schemaVersion": 37, - "style": "dark", + "schemaVersion": 39, "tags": [], "templating": { "list": [ { "current": { - "selected": true, + "selected": false, "text": "Prometheus", - "value": "Prometheus" + "value": "prometheus" }, "hide": 0, "includeAll": false, @@ -3358,13 +3428,13 @@ ] }, "time": { - "from": "now-30m", + "from": "now-12h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "DataHub Dashboard", "uid": "x4fS54Vnk", - "version": 2, + "version": 6, "weekStart": "" -} +} \ No newline at end of file diff --git a/docker/profiles/docker-compose.actions.yml b/docker/profiles/docker-compose.actions.yml index c0a0fd59328715..87385801ede6b5 100644 --- a/docker/profiles/docker-compose.actions.yml +++ b/docker/profiles/docker-compose.actions.yml @@ -1,7 +1,7 @@ x-datahub-actions-service: &datahub-actions-service hostname: actions - image: ${DATAHUB_ACTIONS_IMAGE:-${DATAHUB_ACTIONS_REPO:-acryldata}/datahub-actions}:${ACTIONS_VERSION:-v0.0.14} + image: ${DATAHUB_ACTIONS_IMAGE:-${DATAHUB_ACTIONS_REPO:-acryldata}/datahub-actions}:${ACTIONS_VERSION:-v0.1.1} env_file: - datahub-actions/env/docker.env - ${DATAHUB_LOCAL_COMMON_ENV:-empty.env} @@ -87,4 +87,4 @@ services: - debug-elasticsearch depends_on: datahub-gms-debug-elasticsearch: - condition: service_healthy \ No newline at end of file + condition: service_healthy diff --git a/docs-website/adoptionStoriesIndexes.json b/docs-website/adoptionStoriesIndexes.json index d54dd6bcfa4f0a..1177d74dc6df95 100644 --- a/docs-website/adoptionStoriesIndexes.json +++ b/docs-website/adoptionStoriesIndexes.json @@ -22,10 +22,22 @@ "category": "Financial & Fintech", "description": "\"We found DataHub to provide excellent coverage for our needs. What we appreciate most about DataHub is its powerful API platform.\"

— Jean-Pierre Dijcks, Sr. Dir. Product Management at VISA

" }, + { + "name": "Notion", + "slug": "notion", + "imageUrl": "/img/logos/companies/notion.png", + "imageSize": "small", + "link": "https://blog.datahubproject.io/how-notion-uses-acryl-data-to-ensure-data-reliability-800427a9ba66", + "linkType": "blog", + "tagline": "How Notion Uses Acryl Data to Ensure Data Reliability", + "category": "B2B & B2C", + "platform": "cloud", + "description": "\"We rely on Acryl to gain insights and ensure our critical data is reliable. Acryl’s managed product takes DataHub to the next level through automation and emphasis on time-to-value.\"

— Ada Draginda, Senior Data Engineer at Notion

" + }, { "name": "Optum", "slug": "optum", - "imageUrl": "/img/logos/companies/optum.jpg", + "imageUrl": "/img/logos/companies/optum.png", "imageSize": "medium", "link": "https://opensource.optum.com/blog/2022/03/23/data-mesh-via-datahub", "linkType": "blog", @@ -99,6 +111,30 @@ "category": "Financial & Fintech", "description": "Discover how Checkout leverage DataHub for advanced data management and compliance, especially in managing sensitive data types." }, + { + "name": "MYOB", + "slug": "myob", + "imageUrl": "/img/logos/companies/myob.png", + "imageSize": "medium", + "link": "https://blog.datahubproject.io/how-myob-improved-data-reliability-for-dbt-and-snowflake-with-acryl-a1aa26285480", + "linkType": "blog", + "tagline": "How MYOB Improved Data Reliability for dbt and Snowflake", + "category": "Financial & Fintech", + "platform": "cloud", + "description": "\"Before bringing Acryl on board, MYOB’s data teams would see multiple breaking changes per week...Acryl has helped us significantly reduce the number of breaking changes, to the extent that they are no longer a burden on all teams.\"

— Asad Naveed, Engineering Manager at MYOB

" + }, + { + "name": "DPG Media", + "slug": "dpg-media", + "imageUrl": "/img/logos/companies/dpg-media.png", + "imageSize": "medium", + "link": "https://blog.datahubproject.io/how-acryl-data-helped-dpg-media-save-25-per-month-in-snowflake-costs-c29a1618a703", + "linkType": "blog", + "tagline": "How Acryl Data Helped DPG Media Save 25% Per Month in Snowflake Costs", + "category": "And More", + "platform": "cloud", + "description": "DPG Media used DataHub Cloud to identify and safely retire redundant assets from their data warehouse, reducing costs by 25%.

DataHub Cloud is used to bring federated governance to the data mesh architecture at DPG Media.
" + }, { "name": "MediaMarkt Saturn", "slug": "mediamarkt-saturn", diff --git a/docs-website/docusaurus.config.js b/docs-website/docusaurus.config.js index 1a40c986b31671..88a2c79a450eee 100644 --- a/docs-website/docusaurus.config.js +++ b/docs-website/docusaurus.config.js @@ -3,7 +3,7 @@ const isSaas = process.env.DOCUSAURUS_IS_SAAS === "true"; module.exports = { title: process.env.DOCUSAURUS_CONFIG_TITLE || "DataHub", - tagline: "A Metadata Platform for the Modern Data Stack", + tagline: "The #1 Open Source Metadata Platform", url: process.env.DOCUSAURUS_CONFIG_URL || "https://datahubproject.io", baseUrl: process.env.DOCUSAURUS_CONFIG_BASE_URL || "/", onBrokenLinks: "throw", @@ -23,6 +23,9 @@ module.exports = { src: "/scripts/rb2b.js", async: true, defer: true, + }, + { + src: "https://app.revenuehero.io/scheduler.min.js" } ], noIndex: isSaas, @@ -52,16 +55,16 @@ module.exports = { }, themeConfig: { - ...(!isSaas && { - announcementBar: { - id: "announcement", - content: - '

DataHub Cloud  Acryl Data delivers an easy to consume DataHub platform for the enterprise

Sign Up for DataHub Cloud →', - backgroundColor: "#070707", - textColor: "#ffffff", - isCloseable: false, - }, - }), + // ...(!isSaas && { + // announcementBar: { + // id: "announcement", + // content: + // '

DataHub Cloud  Acryl Data delivers an easy to consume DataHub platform for the enterprise

Sign Up for DataHub Cloud →', + // backgroundColor: "#070707", + // textColor: "#ffffff", + // isCloseable: false, + // }, + // }), navbar: { title: null, logo: { @@ -79,7 +82,7 @@ module.exports = { { to: "/cloud", activeBasePath: "cloud", - label: "Cloud", + html: "Cloud", position: "right", }, { diff --git a/docs-website/package.json b/docs-website/package.json index 58820fbf42b21b..e1130148864e5e 100644 --- a/docs-website/package.json +++ b/docs-website/package.json @@ -42,6 +42,7 @@ "docusaurus-graphql-plugin": "0.5.0", "docusaurus-plugin-sass": "^0.2.1", "dotenv": "^16.0.1", + "framer-motion": "^11.5.4", "markdown-link-check": "^3.11.2", "markprompt": "^0.1.7", "react": "^18.2.0", diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js index 20bed6099cdae3..fe1ee9e6236ab7 100644 --- a/docs-website/sidebars.js +++ b/docs-website/sidebars.js @@ -105,12 +105,12 @@ module.exports = { { label: "Documentation Propagation", type: "doc", - id: "docs/automation/docs-propagation", + id: "docs/automations/docs-propagation", }, { label: "Snowflake Tag Sync", type: "doc", - id: "docs/automation/snowflake-tag-propagation", + id: "docs/automations/snowflake-tag-propagation", className: "saasOnly", }, ], @@ -307,6 +307,7 @@ module.exports = { }, { "DataHub Cloud Release History": [ + "docs/managed-datahub/release-notes/v_0_3_5", "docs/managed-datahub/release-notes/v_0_3_4", "docs/managed-datahub/release-notes/v_0_3_3", "docs/managed-datahub/release-notes/v_0_3_2", diff --git a/docs-website/src/pages/_components/CaseStudy/case-study.module.scss b/docs-website/src/pages/_components/CaseStudy/case-study.module.scss new file mode 100644 index 00000000000000..e17f1f74656dc1 --- /dev/null +++ b/docs-website/src/pages/_components/CaseStudy/case-study.module.scss @@ -0,0 +1,209 @@ +.container { + display: flex; + flex-direction: column; + background: #F4F4F5; + font-family: "Manrope"; +} +.case_study { + display: flex; + flex-direction: column; + width: 100vw; + margin: 5rem auto; + + .case_study_heading { + text-align: center; + font-family: "Manrope"; + + div { + font-size: 2.5rem; + line-height: normal; + } + p { + font-size: 1.25rem; + margin-top: 10px; + } + } + + .card_row::-webkit-scrollbar { + display: none; + } + .card_row { + margin-top: 8px; + overflow-x: scroll; + width: 100vw; + scrollbar-width: none; + display: flex; + + .card_row_wrapper { + display: flex; + flex-direction: row; + align-items: center; + padding: 0 10vw; + flex: 1; + } + .cardLink { + color: #000; + + &:hover { + text-decoration: none; + } + } + + .card { + display: flex !important; + flex-shrink: 0; + justify-content: space-between; + flex-direction: column; + position: relative; + width: 340px !important; + height: 320px !important; + border-radius: 24px; + background: white; + padding: 8px; + margin-right: 20px; + transition: box-shadow .2s ease-in-out; + transition: opacity .2s ease-in-out; + + &:hover { + box-shadow: 0px 0px 4px 0px #0000001C; + opacity: .9; + } + + .card_image { + width: 100%; + height: 14rem; + + border-radius: 16px; + display: flex; + justify-content: center; + align-items: center; + position: relative; + overflow: hidden; + background-color: #25252e; + + img { + object-fit: center; + max-width: 40%; + max-height: 40%; + z-index: 10; + filter: brightness(2); + opacity: .9; + } + .cardImageBackground { + position: absolute; + height: 100%; + width: 100%; + z-index: 0; + background-size: cover; + opacity: .15; + } + } + + .card_tag { + position: absolute; + right: 1.2rem; + top: 1.2rem; + background-color: #edf2ff80; + color: #FFF; + font-size: 0.75rem; + font-family: 'Manrope'; + font-weight: 600; + padding: 0.1rem 0.5rem; + border-radius: 0.5rem; + z-index: 10; + } + .card_heading_div { + text-align: start; + padding: 12px; + + .card_heading { + font-family: "Manrope"; + font-size: 1.1rem; + font-weight: 600; + overflow: hidden; + white-space: nowrap; + text-overflow: ellipsis; + margin-bottom: 2px; + } + .card_para { + font-size: .9rem; + line-height: normal; + display: -webkit-box; + line-clamp: 2; + -webkit-line-clamp: 2; + -webkit-box-orient: vertical; + overflow: hidden; + text-overflow: ellipsis; + opacity: .75; + } + } + } + } + + .bottom_line { + cursor: pointer; + text-decoration: none; + display: flex; + align-items: center; + justify-content: center; + color: #12b0fb; + font-size: 1.1rem; + font-weight: 500; + margin-top: 50px; + margin-bottom: 10px; + + span { + line-height: 10px; + font-size: 1.5rem; + margin-left: 10px; + } + } +} + +@media (max-width: 800px) { + .case_study { + .case_study_heading { + text-align: center; + font-family: "Manrope"; + + div { + width: 80%; + margin: auto; + font-size: 2rem; + line-height: normal; + font-weight: 400; + } + p { + width: 80%; + margin: auto; + font-size: 1.1rem; + line-height: 1.5rem; + margin-top: 12px; + } + } + + .card_row { + margin-top: 24px; + + .card_row_wrapper { + padding: 0 5vw; + } + } + + .bottom_line { + cursor: pointer; + display: flex; + align-items: center; + justify-content: center; + color: #12b0fb; + font-size: 1.1rem; + font-weight: 500; + margin-top: 50px; + margin-bottom: 10px; + + span { + font-size: 1.5rem; + } + } + } +} diff --git a/docs-website/src/pages/_components/CaseStudy/caseStudyContent.js b/docs-website/src/pages/_components/CaseStudy/caseStudyContent.js new file mode 100644 index 00000000000000..524dbfd1b19d63 --- /dev/null +++ b/docs-website/src/pages/_components/CaseStudy/caseStudyContent.js @@ -0,0 +1,74 @@ +const caseStudyData = [ + { + title: "Enhancing Extensibility", + description: + "How Netflix is Collaborating with DataHub to Enhance its Extensibility.", + tag: "Technology", + backgroundImage: + "https://miro.medium.com/v2/resize:fit:1400/format:webp/1*iUnFujzVqMBpK41z32-Ctw.png", + image: "https://datahubproject.io/img/logos/companies/netflix.png", + link: "https://datahubproject.io/adoption-stories/#netflix", + }, + { + title: "Scaling Data Governance", + description: + "How VISA Uses DataHub to Scale Data Governance.", + tag: "Finance", + backgroundImage: + "https://miro.medium.com/v2/resize:fit:2000/format:webp/1*mtC4j8J-jumJKWg8RuR6xQ@2x.png", + image: "https://datahubproject.io/img/logos/companies/visa.png", + link: "https://datahubproject.io/adoption-stories/#visa", + }, + { + title: "Ensuring Data Reliability", + description: + "How Notion Uses DataHub Cloud to Ensure Data Reliability.", + tag: "Technology", + backgroundImage: + "https://www.notion.so/cdn-cgi/image/format=webp,width=1920/front-static/pages/product/super-duper/hero-illo.png", + image: "https://boost.space/wp-content/uploads/2023/08/notion.png", + link: "https://datahubproject.io/adoption-stories/#notion", + }, + { + title: "Working with Petabyte Scale Healthcare Data", + description: + "Enabling the Data Mesh via DataHub to do work on some of healthcare’s more impactful problems.", + tag: "Healthcare", + backgroundImage: + "https://opensource.optum.com/static/images/mesh-overview-e26ea2aaa8d3dbb1f1771b50f4e31449.png", + image: "/img/logos/companies/optum.png", + link: "https://datahubproject.io/adoption-stories/#optum", + }, + { + title: "Exploring Airtel's shift to a decentralized data architecture.", + description: + "Discover how DataHub has become integral to Airtel's datamesh architecture.", + tag: "Telecommunications", + backgroundImage: + "https://upload.wikimedia.org/wikipedia/commons/3/3a/Airtel_logo-01.png", + image: "/img/logos/companies/airtel.png", + link: "https://datahubproject.io/adoption-stories/#airtel", + }, + { + title: "Leveling Up Data Management.", + description: + "DataHub's role in improving data management, tracing data lineage, and ensuring quality.", + tag: "Gaming", + backgroundImage: + "https://upload.wikimedia.org/wikipedia/en/thumb/7/7b/Zynga.svg/1200px-Zynga.svg.png", + image: "/img/logos/companies/zynga.png", + link: "https://datahubproject.io/adoption-stories/#zynga", + }, + { + title: "And many more...", + description: + "See our adoption stories page for more.", + tag: "", + backgroundImage: + "", + image: "", + link: "https://datahubproject.io/adoption-stories/", + }, +]; + +export default caseStudyData; diff --git a/docs-website/src/pages/_components/CaseStudy/index.js b/docs-website/src/pages/_components/CaseStudy/index.js new file mode 100644 index 00000000000000..0362dec9a0d3ef --- /dev/null +++ b/docs-website/src/pages/_components/CaseStudy/index.js @@ -0,0 +1,54 @@ +import React from "react"; +import styles from "./case-study.module.scss"; +import Link from '@docusaurus/Link' +import { Carousel } from "antd"; +import caseStudyData from "./caseStudyContent"; + + +const CaseStudy = () => { + return ( +
+ {/* Section-1 */} +
+
+
See how industry leaders use Datahub
+

Across finance, healthcare, e-commerce, and countless more.

+
+ +
+
+ {caseStudyData.map((caseStudy) => ( + + + See all adoption stories → + +
+
+ ); +}; + +export default CaseStudy; diff --git a/docs-website/src/pages/_components/Community/community.module.scss b/docs-website/src/pages/_components/Community/community.module.scss new file mode 100644 index 00000000000000..37f1a00878f77a --- /dev/null +++ b/docs-website/src/pages/_components/Community/community.module.scss @@ -0,0 +1,280 @@ +.container { + display: flex; + flex-direction: column; + background: #fafafa; +} + +.community_section { + position: relative; + display: flex; + justify-content: space-between; + font-family: "Manrope"; + + .community_section_left { + padding: 60px 0; + width: 60%; + height: 100%; + background: #3e2644; + color: white; + display: flex; + align-items: center; + justify-content: center; + + .community_section_left_content { + margin: 2rem auto; + + .community_section_heading { + color: rgba($color: #fff, $alpha: 0.4); + font-size: 3.5rem; + line-height: normal; + font-weight: bold; + } + + p { + font-size: 1.2rem; + color: rgba($color: #fff, $alpha: 0.6); + font-weight: light; + margin-top: 10px; + } + + a { + cursor: pointer; + display: inline-block; + font-size: 1.2rem; + background-color: transparent; + padding: 5px 20px; + border-radius: 50px; + margin: 0 10px 0 0; + outline: none; + border: 1px solid white; + color: white; + text-decoration: none; + transition: background-color .2s ease-in-out; + &:hover { + background-color: #FFFFFF1A; + } + } + } + } + + .community_section_right { + width: 40%; + background: #000000; + flex: 1; + color: white; + display: flex; + justify-content: center; + flex-direction: column; + font-family: "Manrope"; + + .community_section_subText { + padding: 40px 60px 40px; + color: white; + + a { + text-decoration: none; + } + } + + .community_section_heading { + color: rgba($color: #fff, $alpha: 0.4); + font-size: 1.5rem; + line-height: normal; + font-weight: bold; + + a { + margin-top: 16px; + font-weight: 300; + cursor: pointer; + display: inline-flex; + font-size: 16px; + background-color: transparent; + padding: 8px 20px; + border-radius: 50px; + outline: none; + border: 0.89px solid #E9EAEE; + color: white; + align-items: center; + gap: 8px; + transition: background-color .2s ease-in-out; + &:hover { + background-color: #FFFFFF1A; + } + } + } + } +} + +.carouselContainer { + width: 100%; + background-color: #000; + min-height: 100px; + min-width: 400px; +} + +.slider { + background: #000; + height: 100px; + margin: auto; + overflow: hidden; + position: relative; + display: flex; + align-items: center; + width: 100%; + + &::before, + &::after { + position: absolute; + content: ''; + width: 90%; + height: 100%; + z-index: 9; + } + + &::before { + left: 0; + background: linear-gradient(90deg, rgba(0, 0, 0, 0.7) 0%, rgba(0, 0, 0, 0) 70%); + } + + &::after { + right: 0; + background: linear-gradient(90deg, rgba(0, 0, 0, 0) 70%, rgba(0, 0, 0, 0.7) 100%); + } +} + +.carouselContainer { + overflow: hidden; +} + +.slider { + position: relative; +} + +.slide_track { + display: flex; + width: calc((72px + 24px) * 6 * 2); + /* Width of the track */ + animation: scroll 15s linear infinite; +} + +.slide { + width: 72px; + height: 72px; + margin-left: 24px; + display: flex; + justify-content: center; + overflow: hidden; + flex-direction: column; + align-items: center; + position: relative; + background-size: cover; + background-repeat: no-repeat; +} + +@keyframes scroll { + 0% { + transform: translateX(0); + } + + 100% { + transform: translateX(-50%); + } +} + +.slideCrown { + position: absolute; + right: 5px; + top: 6px; + width: 18px; + height: 18px; + background-image: url("/img/slack/Crown.png"); + background-repeat: no-repeat; + background-size: contain; +} + +.slide::before { + width: 25px; + height: 25px; + border-radius: 25px; + right: 2px; + top: 2px; + position: absolute; + content: ''; + background: white; + display: flex; + align-items: center; + justify-content: center; +} + + +@keyframes slideIn { + 0% { + opacity: 0; + transform: translateY(20px); + } + + 100% { + opacity: 1; + transform: translateY(0); + } +} + +.numberContainer { + display: inline-block; + width: 11rem; + text-align: right; +} + +.numberChange { + display: inline-block; + animation: slideIn 0.5s ease-in-out; + width: 11rem; +} + + +@media only screen and (max-width: 800px) { + .community_section { + flex-direction: column; + + .community_section_left { + width: 100%; + padding: 20px 24px; + + .community_section_left_content { + + .community_section_heading { + font-size: 2rem; + + img { + height: 24px; + width: 24px; + } + } + p { + font-size: 1rem; + line-height: 1.2rem; + margin-top: 8px; + margin-bottom: 12px; + } + a { + padding: 4px 8px; + font-size: 1rem; + } + .numberChange, .numberContainer { + width: 6.5rem; + } + } + } + + .community_section_right { + width: 100%; + overflow: hidden; + .community_section_subText { + padding: 32px 52px 16px; + } + .carouselContainer { + margin-bottom: 24px; + } + } + } +} \ No newline at end of file diff --git a/docs-website/src/pages/_components/Community/index.js b/docs-website/src/pages/_components/Community/index.js new file mode 100644 index 00000000000000..a4f2b2304e51e4 --- /dev/null +++ b/docs-website/src/pages/_components/Community/index.js @@ -0,0 +1,120 @@ +import React, { useState, useRef, useEffect } from "react"; +import styles from "./community.module.scss"; +import useBaseUrl from "@docusaurus/useBaseUrl"; + +const TARGET_COUNT = 11535; +const INCREMENT = 1; + +const Community = () => { + const currentCountRef = useRef(TARGET_COUNT - 50); + const [count, setCount] = useState(currentCountRef.current); + const hasAnimatedRef = useRef(false); + const counterRef = useRef(null); + + const animateNumber = () => { + const makeTimeout = () => { + const distance = TARGET_COUNT - currentCountRef.current; + const isSlowCount = distance < 10; + const isMediumCount = distance < 20; + setTimeout(() => { + if (currentCountRef.current < TARGET_COUNT) { + currentCountRef.current += INCREMENT; + setCount(currentCountRef.current); + makeTimeout(); + } + }, isSlowCount ? Math.random() * 6000 : (isMediumCount ? 150 : 50)); + } + makeTimeout(); + }; + const handleScroll = () => { + if (hasAnimatedRef.current) return; + if (!counterRef.current) return; + + const { top } = counterRef.current.getBoundingClientRect(); + const windowHeight = window.innerHeight; + + if (top <= windowHeight) { + hasAnimatedRef.current = true; + animateNumber(); + } + }; + + const formattedCount = count.toLocaleString(); + + useEffect(() => { + window.addEventListener('scroll', handleScroll); + return () => { + window.removeEventListener('scroll', handleScroll); + } + }, []) + + return ( +
+
+
+
+
+ A community that's +
+ Slack + + + {formattedCount} + + {" "} + strong. +
+
+

+ Q&A.  Office Hours.  Think Tanks.  Job Postings. +

+ Join Slack +
+
+
+
+
+
+ With{" "} + GitHub{" "} + 500+ contributors
world-wide. +
+ Open GitHub +
+
+
+
+ {/* Duplicate the slides */} + {[...Array(2)].map((_, i) => ( + + {[1, 2, 3, 4, 5, 6].map((item, index) => ( +
+
+
+ ))} + + ))} +
+
+
+
+
+
+
+ ); +}; + +export default Community; diff --git a/docs-website/src/pages/_components/Ecosystem/ecosystem.module.scss b/docs-website/src/pages/_components/Ecosystem/ecosystem.module.scss new file mode 100644 index 00000000000000..499fe4c89eaa58 --- /dev/null +++ b/docs-website/src/pages/_components/Ecosystem/ecosystem.module.scss @@ -0,0 +1,225 @@ +.container { + display: flex; + flex-direction: column; + background: #fafafa; +} +.ecosystem_section { + position: relative; + background: white; + padding: 4vh 0; + padding-bottom: 6vh; + max-height: 1100px; + display: flex; + justify-content: center; + align-items: center; + // padding: 2rem 0rem; + // margin: 4rem 0rem 0rem 0rem; + + .ecosystem_section_content { + width: 70%; + min-width: 800px; + height: 100%; + display: flex; + flex-direction: column; + justify-content: center; + align-items: center; + padding: 2rem 0rem; + + .ecosystem_section_upper_content { + width: 100%; + display: flex; + flex-direction: column; + align-items: center; + text-align: center; + padding-top: 1rem; + + .ecosystem_section_heading { + font-size: 2.75rem; + line-height: 3.25rem; + font-weight: 500; + font-family: Manrope; + } + .ecosystem_section_subtitle { + font-size: 1.25rem; + margin-top: 2rem; + line-height: normal; + font-family: Manrope; + } + .bottom_line_cta { + color: #12b0fb; + font-size: 1rem; + font-weight: 500; + margin-top: 1.5rem; + margin-bottom: 1rem; + cursor: pointer; + } + } + .ecosystem_section_lower_content { + height: 380px; + margin-top: 48px; + display: flex; + flex-direction: row; + + .itemWrappers { + display: flex; + flex-direction: column; + margin-right: 8px; + } + .itemWrappersRow { + display: flex; + flex-direction: row; + height: 50%; + } + .itemWrappers>.itemWrappersRow:first-child { + padding-bottom: 4px; + } + .itemWrappers>.itemWrappersRow:last-child { + padding-top: 4px; + } + .itemWrappersRow>a:nth-child(2) { + margin-left: 8px; + } + + .diagramItem { + display: flex; + border-radius: 20px; + overflow: hidden; + aspect-ratio: 1; + height: 100%; + flex-basis: 45%; + background-color: #f6f8fb; + justify-content: center; + background-size: contain; + background-repeat: no-repeat; + background-position: center; + } + + } + + .item1, + .item2, + .item3, + .item4 { + border-radius: 20px; + display: flex; + align-items: end; + justify-content: start; + position: relative; + padding: 36px 32px; + overflow: hidden; + white-space: normal; + cursor: pointer; + transition: box-shadow 0.3s ease-in-out; + text-decoration: none; + + &:hover { + box-shadow: 0px 2px 12px 0px #00000040; + .itemLinkOpenArrow { + opacity: 1; + } + } + } + .itemLinkOpenArrow { + position: absolute; + top: 12px; right: 12px; + height: 36px; + width: 36px; + opacity: 0.75; + transition: opacity 0.3s ease-in-out; + } + + .item1 { + flex-basis: 40%; + background-color: #77B750; + } + + .item2 { + flex-grow: 1; + background-color: #FC5263; + } + + .item3 { + flex-grow: 1; + background-color: #EFB300; + } + + .item4 { + flex-basis: 40%; + background-color: #1890FF; + } + + /* Additional: Control image size */ + .item_content img { + width: 20px; + height: 20px; + // make it all white + filter: brightness(0) invert(1); + } + + /* Additional: Limit text height with ellipsis */ + .item_content span { + display: -webkit-box; + -webkit-line-clamp: 3; /* Show only 2 lines */ + -webkit-box-orient: vertical; + overflow: hidden; + text-overflow: ellipsis; + color: white; + font-size: 1.15rem; + line-height: 1.3rem; + } + } + + // Responsiveness + @media only screen and (max-width: 800px) { + .ecosystem_section_content { + width: 90%; + min-width: 0; + height: auto!important; + padding-bottom: 48px; + padding-top: 48px; + + .ecosystem_section_upper_content { + height: auto!important; + + .ecosystem_section_heading { + font-size: 1.75rem; + line-height: 2.25rem; + } + .ecosystem_section_subtitle { + font-size: 1rem; + line-height: 1.3rem; + margin-top: 1rem; + opacity: .75; + } + .bottom_line_cta { + margin-top: .5rem; + } + } + .ecosystem_section_lower_content { + height: auto; + margin-top: 8px; + flex-direction: column-reverse; + + .diagramItem { + margin-bottom: 12px; + } + + .item1, + .item2, + .item3, + .item4 { + padding: 24px 12px; + } + .item1, .item4 { + flex-basis: 45%; + } + .item_content span { + font-size: .8rem; + line-height: 1.1rem; + } + } + } + + } + +} diff --git a/docs-website/src/pages/_components/Ecosystem/index.js b/docs-website/src/pages/_components/Ecosystem/index.js new file mode 100644 index 00000000000000..4f33e01a7f4c73 --- /dev/null +++ b/docs-website/src/pages/_components/Ecosystem/index.js @@ -0,0 +1,95 @@ +import React from "react"; +import Link from "@docusaurus/Link"; +import styles from "./ecosystem.module.scss"; +import useBaseUrl from "@docusaurus/useBaseUrl"; + +const Ecosystem = () => { + return ( +
+
+
+
+
+ A single control plane for
your entire data ecosystem. +
+
+ DataHub is built on an extensible, scalable and secure foundation. +
It is hardened and ready for the toughest challenges large enterprises face. +
+ + See how Netflix uses DataHub at scale → + +
+
+
+
+ + +
+ right--v1 + + An architecture that's designed for true enterprise scale. + +
+ + + +
+ right--v1 + + 70+ native integrations,
growing every month. +
+
+ +
+
+ + +
+ right--v1 + + Execute ingestion in-VPC,
so your data never leaves +
the network. +
+
+ + + +
+ right--v1 + + Event-driven actions framework, rich APIs and SDKs. + +
+ +
+
+
+
+
+
+
+ ); +}; + +export default Ecosystem; diff --git a/docs-website/src/pages/_components/Hero/hero.module.scss b/docs-website/src/pages/_components/Hero/hero.module.scss index 97bdceaef69366..2484b259a32c28 100644 --- a/docs-website/src/pages/_components/Hero/hero.module.scss +++ b/docs-website/src/pages/_components/Hero/hero.module.scss @@ -1,44 +1,228 @@ .hero { + padding: 2vw 0; + :global { .button { margin-right: 1rem; } - .hero__alert { - margin-bottom: 2rem; - @media (min-width: 690px) { - display: flex; + + .container { + background-color: #1890FF; + border-radius: 50px; + max-width: 95vw !important; + background-image: url("data:image/svg+xml,%3Csvg width='1586' height='895' viewBox='0 0 1586 895' fill='none' xmlns='http://www.w3.org/2000/svg'%3E%3Cg opacity='0.3'%3E%3Cmask id='mask0_312_2970' style='mask-type:alpha' maskUnits='userSpaceOnUse' x='4' y='0' width='1578' height='887'%3E%3Crect x='1582' y='887' width='1578' height='887' rx='49.66' transform='rotate(180 1582 887)' fill='%231890FF'/%3E%3C/mask%3E%3Cg mask='url(%23mask0_312_2970)'%3E%3Cpath fill-rule='evenodd' clip-rule='evenodd' d='M1258.06 -51.9644L1255.94 -49.8714L1287.88 -18.3583H1362.29C1362.88 -16.9122 1364.32 -15.8921 1366 -15.8921C1368.21 -15.8921 1370 -17.659 1370 -19.8387C1370 -22.0183 1368.21 -23.7853 1366 -23.7853C1364.32 -23.7853 1362.88 -22.7648 1362.29 -21.3183H1289.12L1258.06 -51.9644ZM1586.5 -50.9179H1583.5V51.3203L1625.38 92.64H1662V89.68H1626.62L1586.5 50.0943V-50.9179ZM905.709 -29.2115C905.116 -27.7653 903.679 -26.7452 902 -26.7452C899.791 -26.7452 898 -28.5122 898 -30.6918C898 -32.8715 899.791 -34.6384 902 -34.6384C903.679 -34.6384 905.116 -33.618 905.709 -32.1715H1240.12L1322.62 49.2273H1565.53C1566.23 48.0475 1567.52 47.2537 1569 47.2537C1571.21 47.2537 1573 49.0206 1573 51.2003C1573 53.3799 1571.21 55.1469 1569 55.1469C1567.14 55.1469 1565.57 53.8893 1565.13 52.1873H1321.38L1238.88 -29.2115H905.709ZM1401.46 -21.8116H1458.5V-24.7716H1401.87C1401.43 -26.474 1399.86 -27.7319 1398 -27.7319C1395.79 -27.7319 1394 -25.9649 1394 -23.7853C1394 -21.6056 1395.79 -19.8387 1398 -19.8387C1399.48 -19.8387 1400.77 -20.6322 1401.46 -21.8116ZM921.709 29.0009H1042.88L1092.38 77.8402H1227.54C1228.23 79.0197 1229.52 79.8132 1231 79.8132C1233.21 79.8132 1235 78.0462 1235 75.8666C1235 73.6869 1233.21 71.92 1231 71.92C1229.14 71.92 1227.57 73.1779 1227.13 74.8802H1093.62L1044.12 26.041H921.709C921.116 24.5945 919.679 23.574 918 23.574C915.791 23.574 914 25.341 914 27.5206C914 29.7003 915.791 31.4672 918 31.4672C919.679 31.4672 921.116 30.4471 921.709 29.0009ZM1305.56 60.514L1303.44 62.607L1359.88 118.293H1501.5V115.333H1361.12L1305.56 60.514ZM1580.97 107.933C1580.72 109.88 1579.04 111.386 1577 111.386C1574.79 111.386 1573 109.619 1573 107.439C1573 105.26 1574.79 103.493 1577 103.493C1578.26 103.493 1579.39 104.071 1580.12 104.973H1662V107.933H1580.97ZM938.465 130.133C937.773 131.312 936.48 132.106 935 132.106C932.791 132.106 931 130.339 931 128.159C931 125.979 932.791 124.213 935 124.213C936.864 124.213 938.43 125.47 938.874 127.173H1075.5V130.133H938.465ZM1372.5 164.378C1371.03 163.793 1370 162.375 1370 160.719C1370 158.539 1371.79 156.772 1374 156.772C1376.21 156.772 1378 158.539 1378 160.719C1378 162.375 1376.97 163.793 1375.5 164.378V233.611L1401.12 258.891H1564.88L1584.48 239.552C1584.17 238.994 1584 238.356 1584 237.677C1584 235.498 1585.79 233.731 1588 233.731C1590.21 233.731 1592 235.498 1592 237.677C1592 239.857 1590.21 241.624 1588 241.624C1587.58 241.624 1587.18 241.561 1586.8 241.445L1566.12 261.851H1399.88L1372.5 234.838V164.378ZM1283 178.479H1131.5V181.439H1283V178.479ZM1585 202.158H1490.38L1465.38 226.825H1415V229.785H1466.62L1491.62 205.118H1585V202.158ZM1337.13 232.745H1257V235.705H1337.54C1338.23 236.884 1339.52 237.677 1341 237.677C1343.21 237.677 1345 235.91 1345 233.731C1345 231.551 1343.21 229.784 1341 229.784C1339.14 229.784 1337.57 231.042 1337.13 232.745ZM1423.5 250.504H1517.5V247.544H1423.5V250.504ZM1587.87 317.597H1662V314.637H1587.47C1586.77 313.457 1585.48 312.663 1584 312.663C1581.79 312.663 1580 314.43 1580 316.61C1580 318.789 1581.79 320.556 1584 320.556C1585.86 320.556 1587.43 319.299 1587.87 317.597ZM1466 333.876H1541.38L1564.88 357.063H1639.5V354.103H1566.12L1542.62 330.916H1466V333.876ZM1640.5 522.327H1569.4L1539.4 550.446H1474.66C1474.1 549.279 1472.9 548.473 1471.5 548.473C1469.57 548.473 1468 550.019 1468 551.926C1468 553.833 1469.57 555.379 1471.5 555.379C1472.9 555.379 1474.1 554.573 1474.66 553.406H1540.6L1570.6 525.286H1640.5V522.327ZM1660 536.633H1590.88L1568.5 558.713V585.009C1566.53 585.252 1565 586.913 1565 588.925C1565 591.105 1566.79 592.872 1569 592.872C1571.21 592.872 1573 591.105 1573 588.925C1573 587.679 1572.41 586.568 1571.5 585.844V559.939L1592.12 539.593H1660V536.633ZM1481.66 567.219H1538.5V601.052C1537.03 601.638 1536 603.056 1536 604.712C1536 606.892 1537.79 608.659 1540 608.659C1542.21 608.659 1544 606.892 1544 604.712C1544 603.056 1542.97 601.638 1541.5 601.052V564.259H1481.66C1481.1 563.093 1479.9 562.286 1478.5 562.286C1476.57 562.286 1475 563.832 1475 565.739C1475 567.646 1476.57 569.193 1478.5 569.193C1479.9 569.193 1481.1 568.386 1481.66 567.219ZM1591.5 567.219H1660V564.259H1591.5V567.219ZM1609.5 588.432H1606.5V631.845H1660V628.885H1609.5V588.432ZM-70 622.472H99.7621L122.5 651.589V726.357C121.034 726.942 120 728.36 120 730.017C120 732.196 121.791 733.963 124 733.963C126.209 733.963 128 732.196 128 730.017C128 728.36 126.966 726.942 125.5 726.357V650.581L101.238 619.512H-70V622.472ZM1566.62 622.472H1398.88L1372.5 648.498V717.477C1371.03 718.063 1370 719.481 1370 721.137C1370 723.317 1371.79 725.083 1374 725.083C1376.21 725.083 1378 723.317 1378 721.137C1378 719.481 1376.97 718.063 1375.5 717.477V649.724L1400.12 625.432H1565.38L1588.18 647.931C1588.06 648.303 1588 648.7 1588 649.111C1588 651.291 1589.79 653.058 1592 653.058C1594.21 653.058 1596 651.291 1596 649.111C1596 646.932 1594.21 645.165 1592 645.165C1591.31 645.165 1590.67 645.336 1590.1 645.637L1566.62 622.472ZM1343 660.951C1341.14 660.951 1339.57 659.693 1339.13 657.991H1193.11L1145.61 703.87H958.5V700.911H1144.39L1191.89 655.031H1339.54C1340.23 653.852 1341.52 653.058 1343 653.058C1345.21 653.058 1347 654.825 1347 657.004C1347 659.184 1345.21 660.951 1343 660.951ZM1409 652.505C1410.7 652.505 1412.16 653.553 1412.73 655.031H1470.12L1495.62 680.191H1581.29C1581.88 678.745 1583.32 677.724 1585 677.724C1587.21 677.724 1589 679.491 1589 681.671C1589 683.85 1587.21 685.617 1585 685.617C1583.32 685.617 1581.88 684.597 1581.29 683.151H1494.38L1468.88 657.991H1412.68C1412.08 659.406 1410.66 660.398 1409 660.398C1406.79 660.398 1405 658.631 1405 656.451C1405 654.272 1406.79 652.505 1409 652.505ZM1619 653.058C1620.48 653.058 1621.77 653.852 1622.46 655.031H1660V657.991H1622.87C1622.43 659.693 1620.86 660.951 1619 660.951C1616.79 660.951 1615 659.184 1615 657.004C1615 654.825 1616.79 653.058 1619 653.058ZM382 753.696C383.679 753.696 385.116 754.717 385.709 756.163H573.5V759.123H385.709C385.116 760.569 383.679 761.59 382 761.59C379.791 761.59 378 759.823 378 757.643C378 755.463 379.791 753.696 382 753.696ZM1073 761.59C1071.32 761.59 1069.88 760.569 1069.29 759.123H943.621L890.621 811.416H835V808.456H889.379L942.379 756.163H1069.29C1069.88 754.717 1071.32 753.696 1073 753.696C1075.21 753.696 1077 755.463 1077 757.643C1077 759.823 1075.21 761.59 1073 761.59ZM1497.54 767.51H1353.88L1297.94 822.702L1300.06 824.795L1355.12 770.469H1497.13C1497.57 772.172 1499.14 773.429 1501 773.429C1503.21 773.429 1505 771.662 1505 769.483C1505 767.303 1503.21 765.536 1501 765.536C1499.52 765.536 1498.23 766.33 1497.54 767.51ZM1577 774.416C1578.68 774.416 1580.12 775.436 1580.71 776.883H1660V779.843H1580.71C1580.12 781.289 1578.68 782.309 1577 782.309C1574.79 782.309 1573 780.542 1573 778.363C1573 776.183 1574.79 774.416 1577 774.416ZM474.5 789.709H524.903L597.403 857.295H688V854.335H598.597L526.097 786.749H474.5V789.709ZM266.709 811.416H405.879L452.379 857.295H572.535C573.227 858.474 574.519 859.268 576 859.268C578.209 859.268 580 857.501 580 855.321C580 853.142 578.209 851.375 576 851.375C574.136 851.375 572.57 852.633 572.126 854.335H453.621L407.121 808.456H266.709C266.116 807.009 264.679 805.989 263 805.989C260.791 805.989 259 807.756 259 809.936C259 812.115 260.791 813.882 263 813.882C264.679 813.882 266.116 812.862 266.709 811.416ZM803.291 808.456C803.884 807.009 805.321 805.989 807 805.989C809.209 805.989 811 807.756 811 809.936C811 812.115 809.209 813.882 807 813.882C805.321 813.882 803.884 812.862 803.291 811.416H688V808.456H803.291ZM1232 813.882C1230.32 813.882 1228.88 812.862 1228.29 811.416H1094.62L1044.62 860.748H923.709C923.116 862.194 921.679 863.215 920 863.215C917.791 863.215 916 861.448 916 859.268C916 857.089 917.791 855.322 920 855.322C921.679 855.322 923.116 856.342 923.709 857.788H1043.38L1093.38 808.456H1228.29C1228.88 807.009 1230.32 805.989 1232 805.989C1234.21 805.989 1236 807.756 1236 809.936C1236 812.115 1234.21 813.882 1232 813.882ZM1577.5 829.175H1320.88L1233.88 915.014H906.123C905.39 914.112 904.263 913.534 903 913.534C900.791 913.534 899 915.301 899 917.481C899 919.66 900.791 921.427 903 921.427C905.04 921.427 906.723 919.921 906.969 917.974H1235.12L1322.12 832.135H1577.5V829.175ZM-66.5 833.122H169.879L254.016 916.136C254.005 916.254 254 916.373 254 916.494C254 918.674 255.791 920.44 258 920.44C260.209 920.44 262 918.674 262 916.494C262 914.314 260.209 912.547 258 912.547C257.054 912.547 256.184 912.872 255.499 913.414L171.121 830.162H-66.5V833.122ZM1366 904.161H1298.38L1265.94 936.167L1268.06 938.26L1299.62 907.121H1366V904.161ZM1396.5 907.121H1462.42L1496.99 938.306L1499.01 936.121L1463.58 904.161H1396.5V907.121ZM271 912.547C272.679 912.547 274.116 913.568 274.709 915.014H592.5V917.974H274.709C274.116 919.42 272.679 920.44 271 920.44C268.791 920.44 267 918.674 267 916.494C267 914.314 268.791 912.547 271 912.547ZM1660.5 792.669H1629.88L1583.5 838.429V934.254H1586.5V839.655L1631.12 795.629H1660.5V792.669ZM1515 639.245C1517.21 639.245 1519 637.478 1519 635.298C1519 633.119 1517.21 631.352 1515 631.352C1512.79 631.352 1511 633.119 1511 635.298C1511 637.478 1512.79 639.245 1515 639.245ZM1619 228.798C1621.21 228.798 1623 227.031 1623 224.851C1623 222.671 1621.21 220.904 1619 220.904C1616.79 220.904 1615 222.671 1615 224.851C1615 227.031 1616.79 228.798 1619 228.798ZM1606 308.716C1608.21 308.716 1610 306.949 1610 304.77C1610 302.59 1608.21 300.823 1606 300.823C1603.79 300.823 1602 302.59 1602 304.77C1602 306.949 1603.79 308.716 1606 308.716ZM1573 292.93C1573 295.11 1571.21 296.877 1569 296.877C1566.79 296.877 1565 295.11 1565 292.93C1565 290.75 1566.79 288.983 1569 288.983C1571.21 288.983 1573 290.75 1573 292.93ZM1545 284.05C1547.21 284.05 1549 282.283 1549 280.103C1549 277.924 1547.21 276.157 1545 276.157C1542.79 276.157 1541 277.924 1541 280.103C1541 282.283 1542.79 284.05 1545 284.05ZM1471 320.556C1471 322.736 1469.21 324.503 1467 324.503C1464.79 324.503 1463 322.736 1463 320.556C1463 318.377 1464.79 316.61 1467 316.61C1469.21 316.61 1471 318.377 1471 320.556ZM1112 826.709C1113.48 826.709 1114.77 825.915 1115.46 824.735H1137V821.775H1115.87C1115.43 820.073 1113.86 818.815 1112 818.815C1109.79 818.815 1108 820.582 1108 822.762C1108 824.942 1109.79 826.709 1112 826.709ZM916 809.936C916 812.115 914.209 813.882 912 813.882C909.791 813.882 908 812.115 908 809.936C908 807.756 909.791 805.989 912 805.989C914.209 805.989 916 807.756 916 809.936ZM1012 783.789H949V786.749H1012V783.789ZM1182 698.937H1221.5V701.897H1182V698.937ZM1112 723.604H1155V726.563H1112V723.604ZM1260.5 723.604H1346.5V726.563H1260.5V723.604Z' fill='white'/%3E%3C/g%3E%3C/g%3E%3C/svg%3E%0A"); + background-size: cover; + background-position: center; + background-repeat: no-repeat; + height: 76vh; + min-height: 600px; + } + + // .hero__alert { + // margin-bottom: 2rem; + + // @media (min-width: 690px) { + // display: flex; + // align-items: center; + // justify-content: space-between; + // } + + // .button { + // text-decoration: none; + // margin: 0.5rem 0 0 0; + // display: block; + // white-space: nowrap; + + // @media (min-width: 690px) { + // margin: 0 0 0 0.5rem; + // } + // } + // } + + .hero__img, + .hero__text { + color: white; + } + .hero__img { + width: 45%; + } + + .hero__text { + width: 50%; + display: flex; + flex-direction: column; + justify-content: center; + align-items: center; + + >* { + margin-left: 30%; + text-align: left; + width: 90%; + } + + .hero__title { + font-size: 3.5rem; + line-height: normal; + font-family: Manrope; + } + + .hero__subtitle { + font-size: 1.5rem; + line-height: 1.8rem; + font-family: Manrope; + margin-top: 2rem; + font-weight: 300; + display: inline-flex; align-items: center; - justify-content: space-between; + + span { + background: white; + display: inline-flex; + align-items: center; + border-radius: 4px; + margin-left: 4px; + height: 1.8rem; + padding-left: 2px; + padding-right: 6px; + font-family: 'Manrope'; + font-style: normal; + font-weight: 400; + font-size: 1.3rem; + + span { + padding: 0; + + + // background: linear-gradient(90deg, #3B20D9 0%, #C713C6 31%, #DD567D 64.5%, #D7AC27 100%); + background: linear-gradient(90deg, #3B20D9 0%, #533FD1 100%); + -webkit-background-clip: text; + -webkit-text-fill-color: transparent; + background-clip: text; + text-fill-color: transparent; + } + } } - .button { - text-decoration: none; - margin: 0.5rem 0 0 0; - display: block; - white-space: nowrap; - @media (min-width: 690px) { - margin: 0 0 0 0.5rem; + + .hero__cta { + margin-top: 2rem; + + a { + cursor: pointer; + display: inline-block; + font-size: 1rem; + background-color: white; + padding: 5px 20px; + border-radius: 50px; + margin: 0 10px 0 0; + font-weight: 600; + text-decoration: none; + transition: background-color .2s ease-in-out; + &:hover { + background-color: #FFFFFFEE; + } + } + + .cta__secondary { + background-color: rgba($color: #fff, $alpha: .2); + color: white; + border: 1px solid white; + transition: background-color .2s ease-in-out; + &:hover { + background-color: rgba($color: #fff, $alpha: .3); + } } } } - .hero__image { - margin: 2rem 0; + + .hero__footer_cta { + margin-top: 2rem; + font-weight: 500; + cursor: pointer; + text-decoration: none; + display: inline-block; + font-size: 1.2rem; + color: white; } - .hero__alert { - margin-bottom: 2rem; - @media (min-width: 690px) { - display: flex; + + + .hero__img { + display: flex; + align-items: center; + justify-content: center; + + img { + width: 80%; + } + } + + .hero__content { + padding: 2rem 0; + height: 100%; + display: flex; + } + + // .hero__alert { + // margin-bottom: 2rem; + + // @media (min-width: 690px) { + // display: flex; + // align-items: center; + // justify-content: space-between; + // } + + // .button { + // text-decoration: none; + // margin: 0.5rem 0 0 0; + // display: block; + // white-space: nowrap; + + // @media (min-width: 690px) { + // margin: 0 0 0 0.5rem; + // } + // } + // } + + + + + @media only screen and (max-width: 800px) { + .hero__content { + flex-direction: column; align-items: center; - justify-content: space-between; + justify-content: center; } - .button { - text-decoration: none; - margin: 0.5rem 0 0 0; - display: block; - white-space: nowrap; - @media (min-width: 690px) { - margin: 0 0 0 0.5rem; + .hero__img { + width: 100%; + margin-top: 12px; + } + .hero__text { + width: 100%; + padding-left: 24px; + align-items: start; + >* { + margin-left: 0; + width: auto; + } + .hero__title { + font-size: 2rem; + } + .hero__subtitle { + margin-top: .5rem; + font-size: 1rem; + span { + font-size: .8rem; + height: 1.2rem; + padding-left: 0px; + padding-right: 4px; + } + } + .hero__cta { + margin-top: 12px; + } + .hero__footer_cta { + margin-top: 12px; } } } } -} +} \ No newline at end of file diff --git a/docs-website/src/pages/_components/Hero/index.js b/docs-website/src/pages/_components/Hero/index.js index 12e41e2ecd1766..2369c4668bfcd9 100644 --- a/docs-website/src/pages/_components/Hero/index.js +++ b/docs-website/src/pages/_components/Hero/index.js @@ -1,53 +1,90 @@ -import React from "react"; +import React, { useEffect } from "react"; import clsx from "clsx"; import Link from "@docusaurus/Link"; import useBaseUrl from "@docusaurus/useBaseUrl"; -import Image from "@theme/IdealImage"; -import { useColorMode } from "@docusaurus/theme-common"; -import { QuestionCircleOutlined } from "@ant-design/icons"; +// import Image from "@theme/IdealImage"; +// import { useColorMode } from "@docusaurus/theme-common"; +// import { QuestionCircleOutlined } from "@ant-design/icons"; import styles from "./hero.module.scss"; -import CodeBlock from "@theme/CodeBlock"; -import TownhallButton from "../TownhallButton"; -import { Section } from "../Section"; +import { animate, motion, useMotionValue, useTransform } from "framer-motion"; +// import CodeBlock from "@theme/CodeBlock"; +// import TownhallButton from "../TownhallButton"; +// import { Section } from "../Section"; -const HeroAnnouncement = ({ message, linkUrl, linkText }) => ( -
- {message} - {linkUrl && ( - - {linkText} - - )} -
-); +// const HeroAnnouncement = ({ message, linkUrl, linkText }) => ( +//
+// {message} +// {linkUrl && ( +// +// {linkText} +// +// )} +//
+// ); -const Hero = ({}) => { - const { colorMode } = useColorMode(); +const SOLUTION_TEXTS = ["AI Governance", "Data Discovery", "AI Collaboration", "Data Governance", "Data Democratization", "Data Observability"]; + +const Hero = ({ onOpenTourModal }) => { + // const { colorMode } = useColorMode(); + const textIndex = useMotionValue(0); + const baseText = useTransform(textIndex, (latest) => SOLUTION_TEXTS[latest] || ""); + const count = useMotionValue(0); + const rounded = useTransform(count, (latest) => Math.round(latest)); + const displayText = useTransform(rounded, (latest) => + baseText.get().slice(0, latest) + ); + const updatedThisRound = useMotionValue(true); + + useEffect(() => { + animate(count, 60, { + type: "tween", + duration: 1.5, + ease: "easeIn", + repeat: Infinity, + repeatType: "reverse", + repeatDelay: 0.1, + onUpdate(latest) { + if (updatedThisRound.get() === true && latest > 0) { + updatedThisRound.set(false); + } else if (updatedThisRound.get() === false && latest === 0) { + textIndex.set((textIndex.get() + 1) % SOLUTION_TEXTS.length); + updatedThisRound.set(true); + } + }, + }); + }, []); return (
- {/* HeroAnnouncement goes here */}
-
-

The #1 Open Source Metadata Platform

-

- DataHub is an extensible data catalog that enables data discovery, data observability and federated governance to help tame the - complexity of your data ecosystem. -

-

- Built with ❤️ by {" "} - - Acryl Data - {" "} - and LinkedIn. -

- - Get Started → +
+
+ The #1 open source
metadata platform. +
+
+ A unified platform for + + {displayText} + +
+
+ + Book a Demo + + + Product Tour + +
+ + Get started with Core → - - Join our Slack - - +
+
+
@@ -55,4 +92,4 @@ const Hero = ({}) => { ); }; -export default Hero; +export default Hero; \ No newline at end of file diff --git a/docs-website/src/pages/_components/QuickstartContent/index.js b/docs-website/src/pages/_components/QuickstartContent/index.js index 8c942a6a2e440b..bb801e5a3c96d4 100644 --- a/docs-website/src/pages/_components/QuickstartContent/index.js +++ b/docs-website/src/pages/_components/QuickstartContent/index.js @@ -1,50 +1,103 @@ -import React from "react"; +import React, { useEffect, useRef, useState } from "react"; import clsx from "clsx"; -import Link from "@docusaurus/Link"; +// import Link from "@docusaurus/Link"; import useBaseUrl from "@docusaurus/useBaseUrl"; -import Image from "@theme/IdealImage"; -import { useColorMode } from "@docusaurus/theme-common"; -import { QuestionCircleOutlined } from "@ant-design/icons"; +// import Image from "@theme/IdealImage"; +// import { useColorMode } from "@docusaurus/theme-common"; +// import { QuestionCircleOutlined } from "@ant-design/icons"; import styles from "./quickstartcontent.module.scss"; -import CodeBlock from "@theme/CodeBlock"; -import TownhallButton from "../TownhallButton"; -import { Section } from "../Section"; - +// import CodeBlock from "@theme/CodeBlock"; +// import TownhallButton from "../TownhallButton"; +// import { Section } from "../Section"; +import quickstartContent from "./quickstartContent"; +import { motion, useScroll, useTransform} from 'framer-motion'; const QuickstartContent = ({}) => { - const { colorMode } = useColorMode(); + const scrollableElement = useRef(null) + const { scrollYProgress } = useScroll({ + target: scrollableElement, + offset: ["start end", "end end"] + }) + const scaleBar = useTransform(scrollYProgress, [0, 0.2, .9, 1], [0, 0, .8, 1]); + const opacityBar = useTransform(scrollYProgress, [0, 0.2, 0.4], [0, 0, 1]); + return ( -
-
-
-

Get Started Now

-

Run the following command to get started with DataHub.

-
- - python3 -m pip install --upgrade pip wheel setuptools
- python3 -m pip install --upgrade acryl-datahub
- datahub docker quickstart -
-
- - DataHub Quickstart Guide - - - Deploying With Kubernetes - -
-
-
- - Learn -
- What is DataHub? - How is DataHub architected? - See DataHub in action -
+
+ +
The only platform you need.
+
+ Unified Discovery, Observability, and Governance for Data and AI.
+
+
+ + {quickstartContent.map((data, idx) => ( + +
+
{data.heading}
+
{data.title}
+
+

+ {/* Learn More → */} +

+
+
+ +
+
+ ))}
+
); }; -export default QuickstartContent; +export default QuickstartContent; \ No newline at end of file diff --git a/docs-website/src/pages/_components/QuickstartContent/quickstartContent.js b/docs-website/src/pages/_components/QuickstartContent/quickstartContent.js new file mode 100644 index 00000000000000..f5afc09871977b --- /dev/null +++ b/docs-website/src/pages/_components/QuickstartContent/quickstartContent.js @@ -0,0 +1,25 @@ +const quickstartData = [ + { + heading: "Discovery", + title: "Make data democratization a reality", + description: + "Enable everyone in your organization to effortlessly discover trustworthy data, with experiences tailored for each persona.Eliminate breaking changes with detailed cross-platform and column-level lineage. Build confidence in your data with a comprehensive view of business, operational, and technical context, all in one place.", + image: "/img/quickstart_discovery.png", + }, + { + heading: "Observability", + title: "Build trust in your data", + description: + "Effortlessly detect data quality issues with automated checks and AI-driven anomaly detection. Notify your team where they work when issues arise and keep stakeholders in the loop with centralized incident tracking. Spend minutes, not days, resolving issues with detailed lineage, documentation, and ownership information all in one place.", + image: "/img/quickstart_observability.png", + }, + { + heading: "Governance", + title: "Minimize compliance risk, effortlessly", + description: + "Ensure every data asset is accounted for and responsibility governed by defining and enforcing documentation standards. Automate your governance program to automatically classify assets as they evolve over time. Minimize redundant, manual work with GenAI documentation, AI-driven classification, smart propagation, and more.", + image: "/img/quickstart_governance.png", + }, + ]; + + export default quickstartData; \ No newline at end of file diff --git a/docs-website/src/pages/_components/QuickstartContent/quickstartcontent.module.scss b/docs-website/src/pages/_components/QuickstartContent/quickstartcontent.module.scss index e1badca6d2e348..e31f0ec0e1bbbd 100644 --- a/docs-website/src/pages/_components/QuickstartContent/quickstartcontent.module.scss +++ b/docs-website/src/pages/_components/QuickstartContent/quickstartcontent.module.scss @@ -1,67 +1,262 @@ -.container { - margin-bottom: 2rem; +// .container { +// margin-bottom: 2rem; +// } + +// .button { +// text-decoration: none; +// margin: 0.5rem 0 0 0; +// white-space: nowrap; +// @media (min-width: 690px) { +// margin: 0 0 0 0.5rem; +// } +// } + +// .quickstartContent { +// text-align: center; +// padding: 2rem 0; +// height: 100%; +// margin: 2rem 0; +// background: #34394d; +// border-radius: var(--ifm-card-border-radius); +// } + +// .quickstartTitle { +// color: #fafafa; +// } + +// .quickstartSubtitle { +// font-size: 1.1rem; +// color: gray; +// } + +// .quickstartCodeblock { +// text-align: left; +// padding: 0 20vh; +// } + +// .quickLinks { +// display: flex; +// align-items: center; +// justify-content: space-between; +// padding: 1rem; +// font-weight: bold; +// margin-bottom: -2.5vh; +// @media (min-width: 768px) { +// flex-direction: row; +// } + +// > * { +// padding: 0.5rem 1rem; +// display: inline-block; + +// @media (min-width: 768px) { +// padding: 0 1rem; +// } +// } +// } + +// .quickLinksLabel { +// display: flex; +// align-items: center; +// svg { +// width: 24px; +// height: 24px; +// color: var(--ifm-text-color) !important; +// margin-right: 0.5rem; +// } +// } + + +@media only screen and (max-width: 800px) { + .quickstart { + margin: 6rem auto!important; + } } +.quickstart { + width: 80vw; + min-height: 100vh; + margin: 12rem auto; + display: flex; + flex-direction: column; + + :global { + .quickstart__header { + text-align: center; + margin-bottom: 100px; + + .quickstart__title, + .quickstart__subtitle { + line-height: normal; + font-family: Manrope; + } + .quickstart__title { + font-size: 3.5rem; + } -.button { - text-decoration: none; - margin: 0.5rem 0 0 0; - white-space: nowrap; - @media (min-width: 690px) { - margin: 0 0 0 0.5rem; + .quickstart__subtitle { + font-size: 1.8rem; + margin-top: 10px; + } } -} -.quickstartContent { - text-align: center; - padding: 2rem 0; - height: 100%; - margin: 2rem 0; - background: #34394d; - border-radius: var(--ifm-card-border-radius); -} + .quickstart__bar { + width: 10px; + height: 100%; + position: absolute; -.quickstartTitle { - color: #fafafa; -} + background: linear-gradient(180deg, #29B5E8 18.71%, #FFBF4C 143.37%); + border-radius: 10px; + transform-origin: top; + // transition: transform 0.3s; + // animation: progress 0.3s linear; -.quickstartSubtitle { - font-size: 1.1rem; - color: gray; -} + /* Inside auto layout */ + display: block; + } -.quickstartCodeblock { - text-align: left; - padding: 0 20vh; -} + // @keyframes progress { + // 0% { + // height: 0; + // } + // 100% { + // height: 100%; + // } + // } -.quickLinks { - display: flex; - align-items: center; - justify-content: space-between; - padding: 1rem; - font-weight: bold; - margin-bottom: -2.5vh; - @media (min-width: 768px) { - flex-direction: row; - } + .quickstart__container { + position: relative; + } + + .quickstart__content { + display: flex; + margin-bottom: 3rem; + width: 100%; + + .quickstart__text { + width: 60%; + min-width: 600px; + padding-right: 40px; + display: flex; + justify-content: center; + flex-direction: column; + + div { + padding-left: 10%; + } + + .quickstart__text__label { + + /* Label */ + font-family: 'Manrope'; + font-style: normal; + font-weight: 500; + font-size: 1rem; + line-height: 1.2rem; + letter-spacing: 0.4em; + text-transform: uppercase; + + color: #999EB3; + } + + .quickstart__text__head { + + /* H4 | Semibold */ + font-family: 'Manrope'; + font-style: normal; + font-weight: 600; + font-size: 2rem; + line-height: 2rem; + + color: #171B2B; + margin-top: 1rem; + } + + .quickstart__text__desc { + + margin-top: 1rem; + font-family: 'Manrope'; + font-style: normal; + font-weight: 400; + font-size: 1.1rem; + line-height: 1.75rem; + /* Text|Light */ + color: #838BA8; + + p { + margin-bottom: 1rem; + span { + margin-bottom: .75rem; + display: block; + } + } - > * { - padding: 0.5rem 1rem; - display: inline-block; + .learn_more { + color: #1890FF; + font-weight: 500; + } + } + } - @media (min-width: 768px) { - padding: 0 1rem; + .quickstart__img { + display: flex; + align-items: flex-start; + justify-content: center; + min-width: 400px; + width: 40%; + + img { + width: 100%; + } + } } - } -} -.quickLinksLabel { - display: flex; - align-items: center; - svg { - width: 24px; - height: 24px; - color: var(--ifm-text-color) !important; - margin-right: 0.5rem; + + @media only screen and (max-width: 800px) { + .quickstart__bar { + width: 4px; + left: -4px; + } + .quickstart__header { + width: 95%; + margin-bottom: 40px; + + .quickstart__title { + font-size: 2rem; + } + .quickstart__subtitle { + font-size: 1.25rem; + } + } + + .quickstart__content { + flex-direction: column; + + .quickstart__text { + min-width: 0; + width: 100%; + padding-left: 20px; + padding-right: 0; + + div { + padding-left: 0; + } + + .quickstart__text__label { + font-size: .8rem; + } + .quickstart__text__head { + font-size: 1.4rem; + } + .quickstart__text__desc { + font-size: .9rem; + line-height: 1.3rem; + } + } + .quickstart__img { + min-width: 0; + width: 100%; + } + } + } } } diff --git a/docs-website/src/pages/_components/SocialMedia/index.js b/docs-website/src/pages/_components/SocialMedia/index.js new file mode 100644 index 00000000000000..9c5188bca06314 --- /dev/null +++ b/docs-website/src/pages/_components/SocialMedia/index.js @@ -0,0 +1,242 @@ +import React, { useEffect, useRef, useState } from "react"; +import useBaseUrl from "@docusaurus/useBaseUrl"; +import Link from "@docusaurus/Link"; +import styles from "./socialmedia.module.scss"; +import "swiper/css"; +import "swiper/css/pagination"; +import { + ArrowUpOutlined, + LinkedinOutlined, + ReadOutlined, + MediumWorkmarkOutlined, + YoutubeOutlined, +} from "@ant-design/icons"; +import { Carousel } from "antd"; + + +const VideoItem = ({ isActive, src }) => { + const ref = useRef(); + + useEffect(() => { + if (isActive) { + ref.current?.play(); + } else { + ref.current?.pause(); + } + }, [isActive]) + + return
diff --git a/docs-website/src/pages/adoption-stories/styles.module.scss b/docs-website/src/pages/adoption-stories/styles.module.scss index d08b48a011de07..514811b0cfa9a1 100644 --- a/docs-website/src/pages/adoption-stories/styles.module.scss +++ b/docs-website/src/pages/adoption-stories/styles.module.scss @@ -4,4 +4,4 @@ align-items: center; gap: 10px; flex-wrap: wrap; -} \ No newline at end of file +} diff --git a/docs-website/src/pages/cloud/CompanyLogos/customersData.json b/docs-website/src/pages/cloud/CompanyLogos/customersData.json index e8a7470eab4702..8255fa33b77a3c 100644 --- a/docs-website/src/pages/cloud/CompanyLogos/customersData.json +++ b/docs-website/src/pages/cloud/CompanyLogos/customersData.json @@ -69,7 +69,7 @@ "asset": { "_ref": "https://cdn.sanity.io/images/cqo9wkgf/production/b446f595b4b13a72ee82a285924715f950e012ca-540x270.png" }, - "alt": "DPG Megia" + "alt": "DPG Media" } }, { diff --git a/docs-website/src/pages/cloud/DemoForm/hubspotFormStyles.css b/docs-website/src/pages/cloud/DemoForm/hubspotFormStyles.css new file mode 100644 index 00000000000000..9d0488bf8064ba --- /dev/null +++ b/docs-website/src/pages/cloud/DemoForm/hubspotFormStyles.css @@ -0,0 +1,74 @@ +.hs-input { + width: 100% !important; +} + +.hs-form input, +.hs-form select, +.hs-form textarea { + border: .5px solid #DDD; + background-color: #FFF; + padding: .75rem 1rem; + margin: 0.4rem auto; + border-radius: 8px; + font-size: 16px; +} + +.hs-form ::placeholder { + font-style: Manrope; +} + +.hs-form input[type="submit"] { + background-color: #1990FF; + color: #fff; + border: none; + padding: 10px 20px; /* Custom padding */ + cursor: pointer; + border-radius: 8px; /* Rounded corners */ + font-size: 16px; + font-weight: 600; + width: 100px; +} + +.hs-form input[type="submit"]:hover { + background-color: #0056b3; /* Button hover color */ +} + +/* hide the label */ + +.hs-form label span { + display: none; +} + +/* error labels */ + +.hs-form .hs-error-msgs { + font-size: 15px; + color: red; + list-style-type: none; + padding: 0; + text-align: left; +} + +.hs-form .error { + border: red 1.5px solid !important; +} + +/* customize placeholder style */ + +.hs-form input::placeholder, +.hs-form textarea::placeholder, +.hs-form textarea { + font-size: 16px; + font-weight: 400; + font-family: sans-serif; + color: gray; +} + +.hs-form .hs_firstname { + padding-right: 0.5rem; +} + +.hs-form .hs_lastname { + padding-left: 0.5rem; +} + diff --git a/docs-website/src/pages/cloud/DemoForm/index.jsx b/docs-website/src/pages/cloud/DemoForm/index.jsx new file mode 100644 index 00000000000000..28777e722e962d --- /dev/null +++ b/docs-website/src/pages/cloud/DemoForm/index.jsx @@ -0,0 +1,94 @@ +import React, { useEffect } from 'react'; +import clsx from "clsx"; +import Link from "@docusaurus/Link"; +import styles from "./styles.module.scss"; +import ScrollingCustomers from '../CompanyLogos'; +import './hubspotFormStyles.css'; + +const DemoForm = ({ formId }) => { + useEffect(() => { + const formContainerId = `hubspotForm-${formId}`; + + const initializeHubspotForm = () => { + if (!document.querySelector(`#${formContainerId} .hs-form`)) { + window.hbspt.forms.create({ + region: "na1", + portalId: "14552909", + formId: "ed2447d6-e6f9-4771-8f77-825b114a9421", + target: `#${formContainerId}`, + }); + + setTimeout(() => { + const emailInput = document.querySelector(`#${formContainerId} .hs_email .input > input`); + const firstNameInput = document.querySelector(`#${formContainerId} .hs_firstname .input > input`); + const lastNameInput = document.querySelector(`#${formContainerId} .hs_lastname .input > input`); + const phoneInput = document.querySelector(`#${formContainerId} .hs_phone .input > input`); + const additionalInfoInput = document.querySelector(`#${formContainerId} .hs_additional_info .input > textarea`); + + if (emailInput) emailInput.placeholder = 'Company Email'; + if (firstNameInput) firstNameInput.placeholder = 'First Name'; + if (lastNameInput) lastNameInput.placeholder = 'Last Name'; + if (phoneInput) phoneInput.placeholder = 'Phone Number'; + if (additionalInfoInput) additionalInfoInput.placeholder = 'How can we help?'; + + const selectNoEElement = document.getElementById(`number_of_employees-ed2447d6-e6f9-4771-8f77-825b114a9421`); + if (selectNoEElement) { + const disabledOption = selectNoEElement.querySelector('option[disabled]'); + if (disabledOption) { + disabledOption.text = "Select Number of Employees"; + disabledOption.value = ""; + } + } + const selectfamiliarityElement = document.getElementById(`familiarity_with_acryl_datahub-ed2447d6-e6f9-4771-8f77-825b114a9421`); + if (selectfamiliarityElement) { + const disabledOption = selectfamiliarityElement.querySelector('option[disabled]'); + if (disabledOption) { + disabledOption.text = "How familiar are you with DataHub?"; + disabledOption.value = ""; + } + } + }, 1000); // Delay to ensure the form is fully loaded + + window.hero = new RevenueHero({ routerId: '982' }); + window.hero.schedule('hsForm_ed2447d6-e6f9-4771-8f77-825b114a9421'); + } + }; + + if (!window.hbspt) { + const script = document.createElement('script'); + script.src = "//js.hsforms.net/forms/embed/v2.js"; + script.async = true; + script.type = 'text/javascript'; + document.body.appendChild(script); + + script.onload = () => { + initializeHubspotForm(); + }; + } else { + initializeHubspotForm(); + } + + return () => { + const hubspotForm = document.querySelector(`#${formContainerId} .hs-form`); + if (hubspotForm) { + hubspotForm.remove(); + } + }; + }, [formId]); + + return ( +
+
+
+
Book a free Demo
+
+ Schedule a personalized demo and get a free a trial. +
+
+
{/* Use unique ID */} +
+
+ ); +}; + +export default DemoForm; diff --git a/docs-website/src/pages/cloud/DemoForm/styles.module.scss b/docs-website/src/pages/cloud/DemoForm/styles.module.scss new file mode 100644 index 00000000000000..4157a228ae739e --- /dev/null +++ b/docs-website/src/pages/cloud/DemoForm/styles.module.scss @@ -0,0 +1,55 @@ +.col { + padding: 0 2rem; +} + +.formContainer { + padding: 2rem; + margin: 0 auto; + background-color:#F3F3F3; + border: 1px solid #DDD; + align-items: center; + max-width: 540px; + border-radius: 16px; + + .formContent { + width: 90%; + margin: 0 auto; + } + + .formHeader { + justify-content: space-between; + text-align: left; + + .formTitle { + font-size: 24px; + font-weight: 600; + color: #000; + line-height: 28px; + margin-bottom: 12px; + } + + .formSubtitle { + font-size: 14px; + color: #666; + line-height: 14px; + margin-bottom: 8px; + } + } + +} + +.bookButton { + display: none; +} + +@media screen and (max-width: 999px) { + .bookButton { + display: block; + } + .formContainer { + display: none; + } + .productTourButton { + text-align: center!important; + } +} diff --git a/docs-website/src/pages/cloud/Hero/index.js b/docs-website/src/pages/cloud/Hero/index.js new file mode 100644 index 00000000000000..eeb982544ddca7 --- /dev/null +++ b/docs-website/src/pages/cloud/Hero/index.js @@ -0,0 +1,40 @@ +import React, { useEffect } from 'react'; +import clsx from "clsx"; +import Link from "@docusaurus/Link"; +import styles from "./styles.module.scss"; +import ScrollingCustomers from '../CompanyLogos'; +import DemoForm from '../DemoForm'; + +const Hero = () => { + return ( +
+
+
+
+
+

DataHub Cloud

+
+ Experience the premium version of DataHub +
+ with Observability and Governance built-in. +
+
+ + Book Demo + + + Live Product Tour → + + +
+
+ +
+
+
+
+
+ ); +}; + +export default Hero; diff --git a/docs-website/src/pages/cloud/Hero/styles.module.scss b/docs-website/src/pages/cloud/Hero/styles.module.scss new file mode 100644 index 00000000000000..5da028d0171557 --- /dev/null +++ b/docs-website/src/pages/cloud/Hero/styles.module.scss @@ -0,0 +1,74 @@ +.col { + padding: 0 2rem; +} + +.hero { + .button { + margin: 0rem 1rem 2rem 1rem; + } + + .hero__title { + font-size: 3rem; + text-align: left; + } + + .hero__secondtitle { + font-size: 2rem; + font-weight: 300; + margin-bottom: 2rem; + text-align: left; + } + + .hero__subtitle { + margin-bottom: 2rem; + font-size: 1.75rem; + line-height: 2.5rem; + margin-left: 0; + text-align: left; + } + + .buttonLightBlue { + color: #1990FF !important; + background: #EAF3FF; + border-color: #EAF3FF; + + &:hover { + background: #D6E7FF; + } + } + .productTourButton { + background-color: transparent; + border: 0; + color: #1990FF !important; + text-align: left; + } + +} + +.hero__cta { + margin: auto; + display: flex; + flex-direction: column; + justify-content: flex-start; +} + +.bookButton { + display: none; +} + +@media screen and (max-width: 999px) { + .bookButton { + display: block; + } + + .productTourButton { + text-align: center!important; + } +} + +@media screen and (min-width: 1000px){ + .productTourButton { + padding-left: 0!important; + margin-left: 0!important; + } +} \ No newline at end of file diff --git a/docs-website/src/pages/cloud/UnifiedTabs/index.js b/docs-website/src/pages/cloud/UnifiedTabs/index.js index 0c7ff50ce54471..c0fbc25a8de6bc 100644 --- a/docs-website/src/pages/cloud/UnifiedTabs/index.js +++ b/docs-website/src/pages/cloud/UnifiedTabs/index.js @@ -8,21 +8,21 @@ const TabbedComponent = () => { const tabs = [ { - title: 'Data Discovery', + title: 'Discovery', description: 'All the search and discovery features of DataHub Core you already love, enhanced.', icon: "/img/assets/data-discovery.svg", link: "https://www.acryldata.io/acryl-datahub", image: 'https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/saas/demo/discovery.webm', }, { - title: 'Data Observability', + title: 'Observability', description: 'Detect, resolve, and prevent data quality issues before they impact your business. Unify data health signals from all your data quality tools, including dbt tests and more.', icon: "/img/assets/data-ob.svg", link: "https://www.acryldata.io/observe", image: 'https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/saas/demo/observe.webm', }, { - title: 'Data Governance', + title: 'Governance', description: 'Powerful Automation, Reporting and Organizational tools to help you govern effectively.', icon: "/img/assets/data-governance.svg", link: "https://www.acryldata.io/acryl-datahub#governance", @@ -32,7 +32,7 @@ const TabbedComponent = () => { return (
-
One platform to rule them all
+
The only platform you need
{tabs.map((tab, index) => ( @@ -56,7 +56,7 @@ const TabbedComponent = () => { {activeTab === index && (
-
)} diff --git a/docs-website/src/pages/cloud/UnifiedTabs/styles.module.scss b/docs-website/src/pages/cloud/UnifiedTabs/styles.module.scss index 7549f743053552..752bd16471c0b8 100644 --- a/docs-website/src/pages/cloud/UnifiedTabs/styles.module.scss +++ b/docs-website/src/pages/cloud/UnifiedTabs/styles.module.scss @@ -21,7 +21,7 @@ &:before, &:after { content: " "; height: 2px; - width: calc((100vw - 468px - 120px)/2); + width: calc((100vw - 640px - 120px)/2); background: #EEE; display: block; position: absolute; diff --git a/docs-website/src/pages/cloud/index.js b/docs-website/src/pages/cloud/index.js index 00437c8a7640a7..8ceab8aa04e4a9 100644 --- a/docs-website/src/pages/cloud/index.js +++ b/docs-website/src/pages/cloud/index.js @@ -4,18 +4,16 @@ import Link from "@docusaurus/Link"; import useDocusaurusContext from "@docusaurus/useDocusaurusContext"; import Enterprise from "./Enterprise"; import { Section } from "../_components/Section"; -import ScrollingCustomers from "./CompanyLogos"; import clsx from "clsx"; import styles from "./styles.module.scss"; import UnifiedTabs from "./UnifiedTabs"; import FeatureCards from "./FeatureCards"; - +import Hero from "./Hero"; +import DemoForm from "./DemoForm"; function Home() { const context = useDocusaurusContext(); const { siteConfig = {} } = context; - // const { colorMode } = useColorMode(); - if (siteConfig.customFields.isSaas) { window.location.replace("/docs"); @@ -26,32 +24,8 @@ function Home() { title={'DataHub Cloud - Unify Data Observability, Governance and Discovery'} description="DataHub cloud is Managed DataHub with Data Observability and Data Governance built-in." > -
-
-
-
-

Try DataHub Cloud

-
- Introducing DataHub as a Managed Service -
with Data Observability and Data Governance built-in.
- {/* */} -
- - Book Demo - - - Product Tour - -
-
-
-
- -
+ +
@@ -62,24 +36,22 @@ function Home() {
-
-
-

Get your free trial.

-
Data Discovery, Data Quality and Data Governance unified.
- - - Book Demo - - - Product Tour - +
+
+

Start your free trial
today.

+
Unify Discovery, Observability and Governance
for data and AI.
+
+ + Book Demo + + + Live Product Tour → + +
- {/*
-
- An extension of the DataHub Core project.
- View Cloud Docs. - -
*/} +
+
+
diff --git a/docs-website/src/pages/cloud/styles.module.scss b/docs-website/src/pages/cloud/styles.module.scss index b805063750dd4b..66eafff4617db0 100644 --- a/docs-website/src/pages/cloud/styles.module.scss +++ b/docs-website/src/pages/cloud/styles.module.scss @@ -1,3 +1,8 @@ +.col { + padding: 0 2rem; +} + + .link { &:hover { text-decoration: none; @@ -9,44 +14,67 @@ background-color: #FAFAFA !important; } + .hero { - margin-top: 80px; - :global { - .button { - margin-right: 1rem; - } - } - .hero__title { - font-size: 4rem; - } - - .hero__secondtitle { - font-size: 2rem; - font-weight: 300; - margin-bottom: 2rem; - - } - - .hero__subtitle { - margin-bottom: 2rem; - font-size: 1.75rem; - line-height: 2.5rem; - } - - .buttonLightBlue { - color: #1990FF !important; - background: #EAF3FF; - border-color: #EAF3FF; - :hover { - background: #D6E7FF; - } - } - - .learnMore { - font-size: 1.25rem; - font-weight: 600; - margin-top: 0.5rem; - - } - } - + .button { + margin: 0rem 1rem 2rem 1rem; + } + + .hero__title { + font-size: 3rem; + text-align: left; + } + + .hero__secondtitle { + font-size: 2rem; + font-weight: 300; + margin-bottom: 2rem; + text-align: left; + } + + .hero__subtitle { + margin-bottom: 2rem; + font-size: 1.75rem; + line-height: 2.5rem; + margin-left: 0; + text-align: left; + } + + +.productTourButton { + background-color: transparent; + border: 0; + color: #1990FF !important; +} + + +} + +.hero__cta { + margin: auto; + display: flex; + flex-direction: column; + justify-content: flex-start; +} + +.bookButton { + display: none; +} + + +@media screen and (max-width: 999px) { + .bookButton, .productTourButton { + display: block; + } + + .productTourButton { + text-align: center!important; + } +} + +@media screen and (min-width: 1000px){ + .productTourButton { + padding-left: 0!important; + margin-left: 0!important; + } +} \ No newline at end of file diff --git a/docs-website/src/pages/index.js b/docs-website/src/pages/index.js index 2eed41b4ad1bd3..d538831ca3dca1 100644 --- a/docs-website/src/pages/index.js +++ b/docs-website/src/pages/index.js @@ -1,4 +1,4 @@ -import React from "react"; +import React, { useState } from "react"; import Layout from "@theme/Layout"; import Link from "@docusaurus/Link"; import useDocusaurusContext from "@docusaurus/useDocusaurusContext"; @@ -11,19 +11,20 @@ import { PlatformLogos } from "./_components/Logos"; import RoundedImage from "./_components/RoundedImage"; import { CompanyLogos } from "./_components/Logos"; import QuickstartContent from "./_components/QuickstartContent"; +import Testimonials from "./_components/Testimonials"; +import Ecosystem from "./_components/Ecosystem"; +import Community from "./_components/Community"; +import SocialMedia from "./_components/SocialMedia"; +import CaseStudy from "./_components/CaseStudy"; +import Trial from "./_components/Trial"; +import CloseButton from "@ant-design/icons/CloseCircleFilled"; -const example_recipe = ` -source: - type: "mysql" - config: - username: "datahub" - password: "datahub" - host_port: "localhost:3306" -sink: - type: "datahub-rest" - config: - server: 'http://localhost:8080'`.trim(); -const example_recipe_run = "datahub ingest -c recipe.yml"; +const companyIndexes = require("../../adoptionStoriesIndexes.json"); +const companies = companyIndexes.companies; +const keyCompanySlugs = ["netflix", "visa", "pinterest", "airtel", "optum"]; +const keyCompanies = keyCompanySlugs + .map((slug) => companies.find((co) => co.slug === slug)) + .filter((isDefined) => isDefined); function Home() { const context = useDocusaurusContext(); @@ -33,26 +34,63 @@ function Home() { window.location.replace("/docs"); } + const [isTourModalVisible, setIsTourModalVisible] = useState(false); + const onOpenTourModal = () => { + setIsTourModalVisible(true); + }; + const onCloseTourModal = () => { + setIsTourModalVisible(false); + }; return !siteConfig.customFields.isSaas ? ( - -
- -
+ {isTourModalVisible ? ( +
+
+ +
+