feat(base-image): update base image to ubuntu

.github/workflows/datahub-actions-docker.yml

@@ -102,11 +102,12 @@ jobs:
       - name: Build and Push Image (slim)
         uses: docker/build-push-action@v3
         with:
-          file: ./docker/datahub-actions/Dockerfile.slim
+          file: ./docker/datahub-actions/Dockerfile
           platforms: linux/amd64,linux/arm64
           tags: ${{ steps.docker_meta_slim.outputs.tags }}
           push: ${{ needs.setup.outputs.publish == 'true' }}
           build-args:
+            "APP_ENV=prod-slim"
             "DOCKER_BASE_IMAGE=${{ steps.action_tag.outputs.tag }}"
   # image_scan:
   #   permissions:

.github/workflows/test-results.yml

@@ -9,6 +9,11 @@ jobs:
   unit-test-results:
     name: Unit Test Results
     runs-on: ubuntu-latest
+    permissions:
+      checks: write
+      pull-requests: write
+      contents: write
+      repository-projects: write
     if: github.event.workflow_run.conclusion != 'skipped'
 
     steps:
@@ -28,7 +33,7 @@ jobs:
           done
 
       - name: Publish Unit Test Results
-        uses: EnricoMi/publish-unit-test-result-action@v1
+        uses: EnricoMi/publish-unit-test-result-action@v2
         with:
           check_name: "Unit Test Results (${{ github.event.workflow_run.name }})"
           commit: ${{ github.event.workflow_run.head_sha }}

build.gradle

@@ -13,12 +13,15 @@
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
-
 buildscript {
   apply from: './repositories.gradle'
   buildscript.repositories.addAll(project.repositories)
 }
 
+plugins {
+    id 'com.palantir.docker' version '0.35.0' apply false
+}
+
 project.ext.externalDependency = [
 ]
 

datahub-actions/build.gradle

@@ -13,40 +13,57 @@
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
-
 plugins {
-    id 'base'
+  id 'base'
+  id 'com.palantir.docker'
 }
 
 ext {
   python_executable = 'python3'
   venv_name = 'venv'
+  docker_registry = 'acryldata'
+  docker_repo = 'datahub-actions'
+}
+
+if (!project.hasProperty("extra_pip_requirements")) {
+  ext.extra_pip_requirements = ""
 }
 
 task checkPythonVersion(type: Exec) {
-  commandLine python_executable, '-c', 'import sys; assert sys.version_info >= (3, 6)'
+  commandLine python_executable, '-c',
+          'import sys; sys.version_info >= (3, 8), f"Python version {sys.version_info[:2]} not allowed"'
 }
 
 task environmentSetup(type: Exec, dependsOn: checkPythonVersion) {
+  def sentinel_file = "${venv_name}/.venv_environment_sentinel"
   inputs.file file('setup.py')
-  outputs.dir("${venv_name}")
-  commandLine 'bash', '-c', "${python_executable} -m venv ${venv_name} && ${venv_name}/bin/python -m pip install --upgrade pip wheel setuptools"
+  outputs.file(sentinel_file)
+  commandLine 'bash', '-c',
+          "${python_executable} -m venv ${venv_name} && " +
+                  "${venv_name}/bin/python -m pip install --upgrade pip uv wheel 'setuptools>=63.0.0' && " +
+                  "touch ${sentinel_file}"
 }
 
 task installPackage(type: Exec, dependsOn: environmentSetup) {
+  def sentinel_file = "${venv_name}/.build_install_package_sentinel"
   inputs.file file('setup.py')
-  outputs.dir("${venv_name}")
-  commandLine "${venv_name}/bin/pip", 'install', '-e', '.'
+  outputs.file(sentinel_file)
+  commandLine 'bash', '-c',
+          "source ${venv_name}/bin/activate && set -x && " +
+                  "uv pip install -e . ${extra_pip_requirements} && " +
+                  "touch ${sentinel_file}"
 }
 
 task install(dependsOn: [installPackage])
 
 task installDev(type: Exec, dependsOn: [install]) {
+  def sentinel_file = "${venv_name}/.build_install_dev_sentinel"
   inputs.file file('setup.py')
-  outputs.dir("${venv_name}")
-  outputs.file("${venv_name}/.build_install_dev_sentinel")
-  commandLine 'bash', '-x', '-c',
-    "${venv_name}/bin/pip install -e .[dev] && touch ${venv_name}/.build_install_dev_sentinel"
+  outputs.file(sentinel_file)
+  commandLine 'bash', '-c',
+          "source ${venv_name}/bin/activate && set -x && " +
+                  "uv pip install -e .[dev] ${extra_pip_requirements} && " +
+                  "touch ${sentinel_file}"
 }
 
 task lint(type: Exec, dependsOn: installDev) {
@@ -108,6 +125,24 @@ task cleanPythonCache(type: Exec) {
     "find src -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete -o -type d -empty -delete"
 }
 
+docker {
+  name "${docker_registry}/${docker_repo}:v${version}"
+  version "v${version}"
+  dockerfile file("${rootProject.projectDir}/docker/datahub-actions/Dockerfile")
+  files fileTree(rootProject.projectDir) {
+    exclude "datahub-actions/scripts/**"
+    exclude "datahub-actions/build/**"
+    exclude "datahub-actions/venv/**"
+    exclude "datahub-actions/tests/**"
+    exclude "**/*.xml"
+    include ".dockerignore"
+    include "docker/**"
+    include "datahub-actions/**"
+  }
+  // buildArgs([APP_ENV: 'prod-slim'])
+}
+tasks.getByName('docker').dependsOn('build')
+
 build.dependsOn install
 check.dependsOn lint
 check.dependsOn testQuick

datahub-actions/setup.py

@@ -93,7 +93,7 @@ def get_long_description():
     "types-pytz",
     "types-dataclasses",
     "sqlalchemy-stubs",
-    "types-pkg_resources",
+    "types-setuptools",
     "types-six",
     "types-python-dateutil",
     "types-requests",

datahub-actions/tests/unit/test_helpers.py

@@ -51,6 +51,7 @@
         None,
         None,
         None,
+        None,
         AuditStampClass(0, "urn:li:corpuser:datahub"),
     )
 )

docker/datahub-actions/Dockerfile

@@ -15,39 +15,44 @@
 # Defining environment
 ARG APP_ENV=prod
 
-FROM acryldata/datahub-ingestion-base:head-full as prod-install
-COPY datahub-actions /actions-src
+FROM acryldata/datahub-ingestion-base:head-full AS base
 
 USER root
 
-RUN mkdir -p /etc/datahub/actions && mkdir -p /tmp/datahub/logs/actions/system
-RUN cd /actions-src && \
-    pip install "." && \
-    pip install '.[all]' && \
-    # This is required to fix security vulnerability in htrace-core4
-    rm -f /usr/local/lib/python3.10/site-packages/pyspark/jars/htrace-core4-4.1.0-incubating.jar
+COPY --chown=datahub:datahub ./docker/datahub-actions/start.sh /start_datahub_actions.sh
+COPY --chown=datahub:datahub ./docker/datahub-actions/readiness-check.sh /readiness-check.sh
 
-COPY ./docker/datahub-actions/start.sh /start_datahub_actions.sh
-COPY ./docker/datahub-actions/readiness-check.sh /readiness-check.sh
-RUN chmod a+x /start_datahub_actions.sh
+RUN chmod a+x /start_datahub_actions.sh && \
+    mkdir -p /etc/datahub/actions && \
+    mkdir -p /tmp/datahub/logs/actions/system && \
+    chown -R datahub:datahub /etc/datahub /tmp/datahub && \
+    apt-get update && \
+    apt-get install -y -qq default-jre && \
+    apt-get clean && \
+    rm -rf /var/lib/{apt,dpkg,cache,log}/
 
+COPY --chown=datahub:datahub datahub-actions /actions-src
 # Add other default configurations into this!
-RUN mkdir -p /etc/datahub/actions/conf && mkdir -p /etc/datahub/actions/system/conf
-COPY ./docker/config/*.yaml /etc/datahub/actions/system/conf
+COPY --chown=datahub:datahub ./docker/config /etc/datahub/actions/system/conf
 
-# datahub:datahub is created in datahub-ingestion-base image
-RUN chown datahub:datahub /etc/datahub && \
-    chown -R datahub:datahub /tmp/datahub
+USER datahub
+WORKDIR /actions-src
 
-# By transferring the root user's pip cache directory to the datahub
-# user, we can avoid the need for some redundant dependency downloads.
-RUN mkdir -p /home/datahub/.cache \
-    && mv /root/.cache/pip /home/datahub/.cache/pip \
-    && chown -R datahub:datahub /home/datahub/.cache/pip
+FROM base AS prod-slim-install
+
+# Effectively builds the image without the .cache (looks like it is duplicated but is not)
+RUN --mount=type=cache,target=/datahub-ingestion/.cache/uv,uid=1000,gid=1000 \
+    UV_LINK_MODE=copy uv pip install -e ".[all]"
+
+FROM base AS prod-install
+
+RUN UV_LINK_MODE=copy uv pip install -e ".[all]"
+
+FROM ${APP_ENV}-install AS final
+WORKDIR /datahub-ingestion
+
+# This is required to fix security vulnerability in htrace-core4
+RUN find . -name "htrace-core4-4.1.0-incubating.jar" -exec rm "{}" \;
 
-FROM ${APP_ENV}-install as final
-USER datahub
-RUN curl -s "https://get.sdkman.io" | bash
-RUN /bin/bash -c "source /$HOME/.sdkman/bin/sdkman-init.sh; sdk version; sdk install java 8.0.332-zulu"
 ENTRYPOINT [ ]
 CMD dockerize -wait ${DATAHUB_GMS_PROTOCOL:-http}://$DATAHUB_GMS_HOST:$DATAHUB_GMS_PORT/health -timeout 240s /start_datahub_actions.sh