Merge branch 'master' into remote-ingest-mcp

simaov · Mar 6, 2024 · 6936d40 · 6936d40
2 parents 5ae8227 + 289a484
commit 6936d40
Show file tree

Hide file tree

Showing 580 changed files with 36,666 additions and 4,237 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -1,7 +1,11 @@
 **/node_modules/
 */build/
 */*/build/
-*/venv/
+**/venv/
+**/.tox/
+**/.mypy_cache/
+**/.pytest_cache/
+**/__pycache__/
 out
 **/*.class
 # Have to copy gradle/wrapper/gradle-wrapper.jar, can't exclude ALL jars

diff --git a/.github/actions/ci-optimization/action.yml b/.github/actions/ci-optimization/action.yml
@@ -35,6 +35,9 @@ outputs:
   elasticsearch-setup-change:
     description: "Elasticsearch setup docker change"
     value: ${{ steps.filter.outputs.elasticsearch-setup == 'true' }}
+  smoke-test-change:
+    description: "Smoke test change"
+    value: ${{ steps.filter.outputs.smoke-test == 'true' }}
 runs:
   using: "composite"
   steps:
@@ -76,4 +79,6 @@ runs:
           postgres-setup:
             - "docker/postgres-setup/**"
           elasticsearch-setup:
-            - "docker/elasticsearch-setup/**"
+            - "docker/elasticsearch-setup/**"
+          smoke-test:
+            - "smoke-test/**"
diff --git a/.github/scripts/docker_helpers.sh b/.github/scripts/docker_helpers.sh
@@ -24,7 +24,7 @@ function get_tag_full {
 }
 
 function get_python_docker_release_v {
-    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},0.0.0+docker.${SHORT_SHA},g" -e 's,refs/tags/v\(.*\),\1+docker,g' -e 's,refs/pull/\([0-9]*\).*,0.0.0+docker.pr\1,g')
+    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},1!0.0.0+docker.${SHORT_SHA},g" -e 's,refs/tags/v\(.*\),1!\1+docker,g' -e 's,refs/pull/\([0-9]*\).*,1!0.0.0+docker.pr\1,g')
 }
 
 function get_unique_tag {
@@ -37,4 +37,4 @@ function get_unique_tag_slim {
 
 function get_unique_tag_full {
     echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${SHORT_SHA}-full,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-full,g')
-}
+}
diff --git a/.github/workflows/airflow-plugin.yml b/.github/workflows/airflow-plugin.yml
@@ -59,7 +59,7 @@ jobs:
           distribution: "zulu"
           java-version: 17
       - uses: gradle/gradle-build-action@v2
-      - uses: actions/checkout@v3
+      - uses: acryldata/sane-checkout-action@v3
       - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
@@ -35,7 +35,7 @@ jobs:
       elasticsearch_setup_change: ${{ steps.ci-optimize.outputs.elasticsearch-setup-change == 'true' }}
     steps:
       - name: Check out the repo
-        uses: hsheth2/sane-checkout-action@v1
+        uses: acryldata/sane-checkout-action@v3
       - uses: ./.github/actions/ci-optimization
         id: ci-optimize
 
@@ -61,7 +61,7 @@ jobs:
         with:
           timezoneLinux: ${{ matrix.timezone }}
       - name: Check out the repo
-        uses: hsheth2/sane-checkout-action@v1
+        uses: acryldata/sane-checkout-action@v3
       - name: Set up JDK 17
         uses: actions/setup-java@v3
         with:
@@ -105,7 +105,7 @@ jobs:
     if: ${{ needs.setup.outputs.docker_change == 'true' }}
     steps:
       - name: Check out the repo
-        uses: hsheth2/sane-checkout-action@v1
+        uses: acryldata/sane-checkout-action@v3
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"

diff --git a/.github/workflows/check-datahub-jars.yml b/.github/workflows/check-datahub-jars.yml
@@ -27,7 +27,7 @@ jobs:
         command: ["datahub-client", "datahub-protobuf", "spark-lineage"]
     runs-on: ubuntu-latest
     steps:
-      - uses: hsheth2/sane-checkout-action@v1
+      - uses: acryldata/sane-checkout-action@v3
       - name: Set up JDK 17
         uses: actions/setup-java@v3
         with:

diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml
@@ -31,7 +31,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out the repo
-        uses: hsheth2/sane-checkout-action@v1
+        uses: acryldata/sane-checkout-action@v3
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"

diff --git a/.github/workflows/docker-ingestion-smoke.yml b/.github/workflows/docker-ingestion-smoke.yml
@@ -25,7 +25,7 @@ jobs:
       python_release_version: ${{ steps.python_release_version.outputs.release_version }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: acryldata/sane-checkout-action@v3
       - name: Compute Tag
         id: tag
         run: |
@@ -50,7 +50,7 @@ jobs:
     if: ${{ needs.setup.outputs.publish == 'true' }}
     steps:
       - name: Check out the repo
-        uses: hsheth2/sane-checkout-action@v1
+        uses: acryldata/sane-checkout-action@v3
       - name: Build and push
         uses: ./.github/actions/docker-custom-build-and-push
         with:

diff --git a/.github/workflows/docker-postgres-setup.yml b/.github/workflows/docker-postgres-setup.yml
@@ -27,7 +27,7 @@ jobs:
       publish: ${{ steps.publish.outputs.publish }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: acryldata/sane-checkout-action@v3
       - name: Compute Tag
         id: tag
         run: |
@@ -46,7 +46,7 @@ jobs:
     needs: setup
     steps:
       - name: Check out the repo
-        uses: hsheth2/sane-checkout-action@v1
+        uses: acryldata/sane-checkout-action@v3
       - name: Build and push
         uses: ./.github/actions/docker-custom-build-and-push
         with:

diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml
@@ -29,29 +29,8 @@ env:
   DATAHUB_INGESTION_IMAGE: "acryldata/datahub-ingestion"
 
 jobs:
-  check_lint:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out the repo
-        uses: hsheth2/sane-checkout-action@v1
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
-          cache: "pip"
-      - name: Set up JDK 17
-        uses: actions/setup-java@v3
-        with:
-          distribution: "zulu"
-          java-version: 17
-      - name: Ensure packages are correct
-        run: |
-          python ./.github/scripts/check_python_package.py
-      - name: Run lint on smoke test
-        run: |
-          ./gradlew :smoke-test:lint
   setup:
     runs-on: ubuntu-latest
-    needs: check_lint
     outputs:
       tag: ${{ steps.tag.outputs.tag }}
       slim_tag: ${{ steps.tag.outputs.slim_tag }}
@@ -76,7 +55,7 @@ jobs:
       elasticsearch_setup_change: ${{ steps.ci-optimize.outputs.elasticsearch-setup-change == 'true' }}
     steps:
       - name: Check out the repo
-        uses: hsheth2/sane-checkout-action@v1
+        uses: acryldata/sane-checkout-action@v3
       - name: Compute Tag
         id: tag
         run: |
@@ -100,6 +79,29 @@ jobs:
           echo "publish=${{ env.ENABLE_PUBLISH }}" >> $GITHUB_OUTPUT
       - uses: ./.github/actions/ci-optimization
         id: ci-optimize
+      - uses: actions/setup-python@v4
+        if: ${{ steps.ci-optimize.outputs.smoke-test-change == 'true' }}
+        with:
+          python-version: "3.10"
+          cache: "pip"
+      - uses: actions/cache@v4
+        if: ${{ steps.ci-optimize.outputs.smoke-test-change == 'true' }}
+        with:
+          path: |
+            ~/.cache/uv
+          key: ${{ runner.os }}-uv-${{ hashFiles('**/requirements.txt') }}
+      - name: Set up JDK 17
+        uses: actions/setup-java@v3
+        if: ${{ steps.ci-optimize.outputs.smoke-test-change == 'true' }}
+        with:
+          distribution: "zulu"
+          java-version: 17
+      - uses: gradle/gradle-build-action@v2
+      - name: Run lint on smoke test
+        if: ${{ steps.ci-optimize.outputs.smoke-test-change == 'true' }}
+        run: |
+          python ./.github/scripts/check_python_package.py
+          ./gradlew :smoke-test:lint
 
   gms_build:
     name: Build and Push DataHub GMS Docker Image
@@ -142,7 +144,7 @@ jobs:
     if: ${{ needs.setup.outputs.backend_change == 'true' || needs.setup.outputs.publish == 'true' }}
     steps:
       - name: Checkout # adding checkout step just to make trivy upload happy
-        uses: actions/checkout@v3
+        uses: acryldata/sane-checkout-action@v3
       - name: Download image
         uses: ishworkh/docker-image-artifact-download@v1
         if: ${{ needs.setup.outputs.publish != 'true' }}
@@ -206,7 +208,7 @@ jobs:
       actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
     steps:
       - name: Checkout # adding checkout step just to make trivy upload happy
-        uses: actions/checkout@v3
+        uses: acryldata/sane-checkout-action@v3
       - name: Download image
         uses: ishworkh/docker-image-artifact-download@v1
         if: ${{ needs.setup.outputs.publish != 'true' }}
@@ -242,7 +244,7 @@ jobs:
           java-version: 17
       - uses: gradle/gradle-build-action@v2
       - name: Check out the repo
-        uses: hsheth2/sane-checkout-action@v1
+        uses: acryldata/sane-checkout-action@v3
       - name: Pre-build artifacts for docker image
         run: |
           ./gradlew :metadata-jobs:mce-consumer-job:build -x test --parallel
@@ -270,7 +272,7 @@ jobs:
       actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
     steps:
       - name: Checkout # adding checkout step just to make trivy upload happy
-        uses: actions/checkout@v3
+        uses: acryldata/sane-checkout-action@v3
       - name: Download image
         uses: ishworkh/docker-image-artifact-download@v1
         if: ${{ needs.setup.outputs.publish != 'true' }}
@@ -306,7 +308,7 @@ jobs:
           java-version: 17
       - uses: gradle/gradle-build-action@v2
       - name: Check out the repo
-        uses: hsheth2/sane-checkout-action@v1
+        uses: acryldata/sane-checkout-action@v3
       - name: Pre-build artifacts for docker image
         run: |
           ./gradlew :datahub-upgrade:build -x test --parallel
@@ -334,7 +336,7 @@ jobs:
       actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
     steps:
       - name: Checkout # adding checkout step just to make trivy upload happy
-        uses: actions/checkout@v3
+        uses: acryldata/sane-checkout-action@v3
       - name: Download image
         uses: ishworkh/docker-image-artifact-download@v1
         if: ${{ needs.setup.outputs.publish != 'true' }}
@@ -370,7 +372,7 @@ jobs:
           java-version: 17
       - uses: gradle/gradle-build-action@v2
       - name: Check out the repo
-        uses: hsheth2/sane-checkout-action@v1
+        uses: acryldata/sane-checkout-action@v3
       - name: Pre-build artifacts for docker image
         run: |
           ./gradlew :datahub-frontend:dist -x test -x yarnTest -x yarnLint --parallel
@@ -430,7 +432,7 @@ jobs:
     if: ${{ needs.setup.outputs.kafka_setup_change == 'true' || needs.setup.outputs.publish == 'true' }}
     steps:
       - name: Check out the repo
-        uses: hsheth2/sane-checkout-action@v1
+        uses: acryldata/sane-checkout-action@v3
       - name: Build and push
         uses: ./.github/actions/docker-custom-build-and-push
         with:
@@ -451,7 +453,7 @@ jobs:
     if: ${{ needs.setup.outputs.mysql_setup_change == 'true' || needs.setup.outputs.publish == 'true' }}
     steps:
       - name: Check out the repo
-        uses: hsheth2/sane-checkout-action@v1
+        uses: acryldata/sane-checkout-action@v3
       - name: Build and push
         uses: ./.github/actions/docker-custom-build-and-push
         with:
@@ -472,7 +474,7 @@ jobs:
     if: ${{ needs.setup.outputs.elasticsearch_setup_change == 'true' || needs.setup.outputs.publish == 'true' }}
     steps:
       - name: Check out the repo
-        uses: hsheth2/sane-checkout-action@v1
+        uses: acryldata/sane-checkout-action@v3
       - name: Build and push
         uses: ./.github/actions/docker-custom-build-and-push
         with:
@@ -495,7 +497,7 @@ jobs:
     if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' }}
     steps:
       - name: Check out the repo
-        uses: hsheth2/sane-checkout-action@v1
+        uses: acryldata/sane-checkout-action@v3
       - uses: dorny/paths-filter@v2
         id: filter
         with:
@@ -528,7 +530,7 @@ jobs:
     if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' }}
     steps:
       - name: Check out the repo
-        uses: hsheth2/sane-checkout-action@v1
+        uses: acryldata/sane-checkout-action@v3
       - uses: dorny/paths-filter@v2
         id: filter
         with:
@@ -569,7 +571,7 @@ jobs:
     if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' }}
     steps:
       - name: Check out the repo
-        uses: hsheth2/sane-checkout-action@v1
+        uses: acryldata/sane-checkout-action@v3
       - uses: dorny/paths-filter@v2
         id: filter
         with:
@@ -618,7 +620,7 @@ jobs:
           java-version: 17
       - uses: gradle/gradle-build-action@v2
       - name: Check out the repo
-        uses: hsheth2/sane-checkout-action@v1
+        uses: acryldata/sane-checkout-action@v3
       - uses: dorny/paths-filter@v2
         id: filter
         with:
@@ -668,7 +670,7 @@ jobs:
     if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' }}
     steps:
       - name: Checkout # adding checkout step just to make trivy upload happy
-        uses: actions/checkout@v3
+        uses: acryldata/sane-checkout-action@v3
       - name: Download image Slim Image
         uses: ishworkh/docker-image-artifact-download@v1
         if: ${{ needs.datahub_ingestion_slim_build.outputs.needs_artifact_download == 'true' }}
@@ -707,7 +709,7 @@ jobs:
           java-version: 17
       - uses: gradle/gradle-build-action@v2
       - name: Check out the repo
-        uses: hsheth2/sane-checkout-action@v1
+        uses: acryldata/sane-checkout-action@v3
       - uses: dorny/paths-filter@v2
         id: filter
         with:
@@ -756,7 +758,7 @@ jobs:
     if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' }}
     steps:
       - name: Checkout # adding checkout step just to make trivy upload happy
-        uses: actions/checkout@v3
+        uses: acryldata/sane-checkout-action@v3
       - name: Download image Full Image
         uses: ishworkh/docker-image-artifact-download@v1
         if: ${{ needs.datahub_ingestion_full_build.outputs.needs_artifact_download == 'true' }}
@@ -823,7 +825,7 @@ jobs:
       - name: Disk Check
         run: df -h . && docker images
       - name: Check out the repo
-        uses: actions/checkout@v3
+        uses: acryldata/sane-checkout-action@v3
       - name: Set up JDK 17
         uses: actions/setup-java@v3
         with:
@@ -978,14 +980,14 @@ jobs:
         if: failure()
         run: |
           docker ps -a
-          docker logs datahub-gms >& gms-${{ matrix.test_strategy }}.log || true
-          docker logs datahub-actions >& actions-${{ matrix.test_strategy }}.log || true
-          docker logs datahub-mae-consumer >& mae-${{ matrix.test_strategy }}.log || true
-          docker logs datahub-mce-consumer >& mce-${{ matrix.test_strategy }}.log || true
-          docker logs broker >& broker-${{ matrix.test_strategy }}.log || true
-          docker logs mysql >& mysql-${{ matrix.test_strategy }}.log || true
-          docker logs elasticsearch >& elasticsearch-${{ matrix.test_strategy }}.log || true
-          docker logs datahub-frontend-react >& frontend-${{ matrix.test_strategy }}.log || true
+          docker logs datahub-datahub-gms-1 >& gms-${{ matrix.test_strategy }}.log || true
+          docker logs datahub-datahub-actions-1 >& actions-${{ matrix.test_strategy }}.log || true
+          docker logs datahub-datahub-mae-consumer-1 >& mae-${{ matrix.test_strategy }}.log || true
+          docker logs datahub-datahub-mce-consumer-1 >& mce-${{ matrix.test_strategy }}.log || true
+          docker logs datahub-broker-1 >& broker-${{ matrix.test_strategy }}.log || true
+          docker logs datahub-mysql-1 >& mysql-${{ matrix.test_strategy }}.log || true
+          docker logs datahub-elasticsearch-1 >& elasticsearch-${{ matrix.test_strategy }}.log || true
+          docker logs datahub-datahub-frontend-react-1 >& frontend-${{ matrix.test_strategy }}.log || true
       - name: Upload logs
         uses: actions/upload-artifact@v3
         if: failure()