#140 Using the pytest plugins

.github/workflows/build.yaml

@@ -3,9 +3,6 @@ name: Build and test project
 on:
   workflow_call:
     inputs:
-      slow-tests:
-        type: boolean
-        default: false
       python-version:
         type: string
         required: true
@@ -18,29 +15,9 @@ jobs:
     environment: aws
 
     steps:
-      - name: Set pytest markers
-        id: pytest-markers
-        if: ${{ ! inputs.slow-tests }}
-        run: echo slow-tests='-m "not slow"' >> "$GITHUB_OUTPUT"
-
       - name: SCM Checkout
         uses: actions/checkout@v4
 
-      - name: Free disk space
-        uses: jlumbroso/free-disk-space@main
-        with:
-          tool-cache: true
-          large-packages: false
-
-      - name: Free disk space by removing large directories
-        run: |
-          sudo rm -rf /usr/local/graalvm/
-          sudo rm -rf /usr/local/.ghcup/
-          sudo rm -rf /usr/local/share/powershell
-          sudo rm -rf /usr/local/share/chromium
-          sudo rm -rf /usr/local/lib/node_modules
-          sudo rm -rf /opt/ghc
-
       - name: Setup Python & Poetry Environment
         uses: ./.github/actions/prepare_poetry_env
         with:
@@ -60,19 +37,12 @@ jobs:
       - name: Run Lua unit tests and static code analyzer
         run: poetry run poe lua-tests
 
-      - name: Setup integration test environment
-        run: ./scripts/setup_integration_test.sh
+      - name: Run unit tests
+        run: poetry run pytest tests/unit_tests
 
-      - name: Poetry run pytest integration tests
+      - name: Run integration tests
         env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          AWS_ROLE: ${{ secrets.AWS_ROLE }}
-          AWS_DEFAULT_REGION: "eu-central-1"
           SAAS_HOST: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_HOST }}
           SAAS_ACCOUNT_ID: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_ACCOUNT_ID }}
           SAAS_PAT: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_PAT }}
-          PYTEST_ADDOPTS: '${{ steps.pytest-markers.outputs.slow-tests }}'
-        run: |
-          echo "PYTEST_ADDOPTS = $PYTEST_ADDOPTS"
-          poetry run pytest tests
+        run: poetry run pytest --backend=all tests/integration_tests

.github/workflows/build_slow.yaml

@@ -0,0 +1,44 @@
+name: Build and test project
+
+on:
+  workflow_call:
+    inputs:
+      python-version:
+        type: string
+        required: true
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        backend: ["onprem", "saas"]
+    runs-on: ubuntu-20.04
+    environment: aws
+    name: Running ci tests on ${{ matrix.backend }}
+
+    steps:
+      - name: SCM Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Python & Poetry Environment
+        uses: ./.github/actions/prepare_poetry_env
+        with:
+          python-version: ${{ inputs.python-version }}
+
+      - name: Build Poetry
+        run: poetry build
+
+      - name: Install project whl
+        run: pip install dist/*.whl
+
+      - name: Run tests
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_ROLE: ${{ secrets.AWS_ROLE }}
+          AWS_DEFAULT_REGION: "eu-central-1"
+          SAAS_HOST: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_HOST }}
+          SAAS_ACCOUNT_ID: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_ACCOUNT_ID }}
+          SAAS_PAT: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_PAT }}
+        run: poetry run pytest --backend=${{ matrix.backend }} --itde-nameserver="8.8.8.8" tests/ci_tests

.github/workflows/ci.yml

@@ -15,7 +15,6 @@ jobs:
     secrets: inherit
     with:
       python-version: ${{ matrix.python-version }}
-      slow-tests: false
 
   gate-1:
     name: Gate 1 - Regular CI
@@ -36,11 +35,10 @@ jobs:
 
   run-slow-tests:
     name: Run Slow or Expensive Tests if Requested
-    uses: ./.github/workflows/build.yaml
+    uses: ./.github/workflows/build_slow.yaml
     needs: [ slow-test-detection ]
     secrets: inherit
     with:
-      slow-tests: true
       python-version: "3.10"
 
   gate-2:

.github/workflows/release_droid_upload_github_release_assets.yml

@@ -31,7 +31,7 @@ jobs:
         run: pip install dist/*.whl
 
       - name: Release language container
-        run: ./release_language_container.sh "$RELEASE_DIR"
+        run: poetry run python3 -u "./exasol_sagemaker_extension/deployment/language_container.py" "$RELEASE_DIR"
 
       - name: Upload assets to the GitHub release draft
         uses: shogo82148/actions-upload-release-asset@v1

doc/changes/changelog.md

@@ -1,5 +1,6 @@
 # Changelog
 
+* [0.11.0](changes_0.11.0.md)
 * [0.10.0](changes_0.10.0.md)
 * [0.9.0](changes_0.9.0.md)
 * [0.8.0](changes_0.8.0.md)

doc/changes/changes_0.11.0.md

@@ -0,0 +1,11 @@
+# SageMaker Extension 0.11.0, released T.B.D.
+
+Code name: T.B.D.
+
+## Summary
+
+T.B.D.
+
+### Refactoring
+
+* #140: Using the pytest plugins for testing.

exasol_sagemaker_extension/deployment/language_container.py

@@ -0,0 +1,28 @@
+import sys
+import os
+from contextlib import contextmanager
+
+from exasol.python_extension_common.deployment.language_container_builder import (
+    LanguageContainerBuilder, find_path_backwards)
+
+CONTAINER_NAME = "exasol_sagemaker_extension_container"
+
+
+@contextmanager
+def language_container_factory():
+    with LanguageContainerBuilder(CONTAINER_NAME) as container_builder:
+        project_directory = find_path_backwards("pyproject.toml", __file__).parent
+        container_builder.prepare_flavor(project_directory)
+        yield container_builder
+
+
+def export_slc():
+    export_dir = sys.argv[1]
+    if not os.path.isdir(export_dir):
+        os.makedirs(export_dir)
+    with language_container_factory() as container_builder:
+        container_builder.export(export_dir)
+
+
+if __name__ == '__main__':
+    export_slc()