add and use exasol toolbox (#66)

closes #65 Co-authored-by: Nicola Coretti <[email protected]> Co-authored-by: Thomas Ubensee <[email protected]>
exasol · Oct 2, 2024 · 827fa59 · 827fa59
1 parent 5ad84bf
commit 827fa59
Show file tree

Hide file tree

Showing 125 changed files with 2,583 additions and 1,126 deletions.
diff --git a/.github/workflows/check_version.yaml b/.github/workflows/check_version.yaml
@@ -12,9 +12,9 @@ jobs:
       - name: SCM Checkout
         uses: actions/checkout@v3
       - name: Setup Python & Poetry Environment
-        uses: exasol/python-toolbox/.github/actions/python-environment@0.12.0
+        uses: exasol/python-toolbox/.github/actions/python-environment@0.15.0
         with:
           python-version: "3.10"
           poetry-version: '1.8.2'
       - name: Check Release
-        run: ./scripts/build/check_release.sh "python3.10"
+        run: poetry run python3 scripts/build/check_release.py
diff --git a/.github/workflows/checks.yaml b/.github/workflows/checks.yaml
@@ -0,0 +1,38 @@
+name: Checks
+
+on:
+  pull_request:
+
+jobs:
+
+  lint-job:
+    name: Linting and Type checks (Python-${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+
+    steps:
+      - name: SCM Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Python & Poetry Environment
+        uses: exasol/python-toolbox/.github/actions/[email protected]
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Run Unit Tests
+        run: poetry run nox -s unit-tests
+
+      - name: Run Lint
+        run: poetry run nox -s lint
+
+      - name: Run type-check
+        run: poetry run nox -s type-check
+
+      - name: Upload Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: ".lint-python-${{ matrix.python-version }}.txt"
+          path: .lint.txt
diff --git a/.github/workflows/run_ci_test.yaml → .github/workflows/ci.yaml b/.github/workflows/run_ci_test.yaml → .github/workflows/ci.yaml
@@ -1,4 +1,4 @@
-name: Run Unit Tests
+name: Run Integration Tests
 
 on:
   push:
@@ -14,9 +14,9 @@ jobs:
       - name: SCM Checkout
         uses: actions/checkout@v4
       - name: Setup Python & Poetry Environment
-        uses: exasol/python-toolbox/.github/actions/python-environment@0.12.0
+        uses: exasol/python-toolbox/.github/actions/python-environment@0.15.0
         with:
           python-version: "3.10"
           poetry-version: '1.8.2'
       - name: Run pytest
-        run: poetry run pytest
+        run: poetry run nox -s integration-tests
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -17,7 +17,7 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Setup Python & Poetry Environment
-        uses: exasol/python-toolbox/.github/actions/python-environment@0.12.0
+        uses: exasol/python-toolbox/.github/actions/python-environment@0.15.0
         with:
           python-version: "3.10"
           poetry-version: '1.8.2'

diff --git a/.github/workflows/shellcheck.yaml b/.github/workflows/shellcheck.yaml
diff --git a/.gitignore b/.gitignore
@@ -2,4 +2,6 @@
 dist
 .build_output
 .pytest_cache
-__pycache__
+__pycache__
+.lint.json
+.lint.txt
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -0,0 +1,36 @@
+default_stages: [ commit ]
+repos:
+
+  - repo: local
+    hooks:
+      - id: code-format
+        name: code-format
+        types: [ python ]
+        pass_filenames: false
+        language: system
+        entry: poetry run nox -s fix
+
+  - repo: local
+    hooks:
+      - id: type-check
+        name: type-check
+        types: [ python ]
+        pass_filenames: false
+        language: system
+        entry: poetry run nox -s type-check
+
+  - repo: local
+    hooks:
+      - id: lint
+        name: lint
+        types: [ python ]
+        pass_filenames: false
+        language: system
+        entry: poetry run nox -s lint
+
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: check-yaml
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
diff --git a/exasol_script_languages_container_ci/__init__.py b/exasol_script_languages_container_ci/__init__.py
@@ -1,5 +1,4 @@
-
 from exasol_script_languages_container_ci.cli.commands import (
     run_ci,
-    run_release
+    run_release,
 )
diff --git a/exasol_script_languages_container_ci/cli/commands/run_ci.py b/exasol_script_languages_container_ci/cli/commands/run_ci.py
@@ -9,40 +9,48 @@
 
 
 @cli.command()
-@click.option('--flavor', required=True, type=str,
-              help="Flavor name.")
-@click.option('--branch-name', required=True, type=str,
-              help="Branch name.")
-@click.option('--docker-user', required=True, type=str,
-              help="Docker user name")
-@click.option('--docker-password', required=True, type=str,
-              help="Docker password")
-@click.option('--docker-build-repository', required=True, type=str,
-              help="Docker build repository")
-@click.option('--docker-release-repository', required=True, type=str,
-              help="Docker release repository")
-@click.option('--commit-sha', required=True, type=str,
-              help="Commit SHA")
-@click.option('--config-file', required=True, type=click.Path(exists=True, file_okay=True, dir_okay=False),
-              help="The build config file (project specific)")
+@click.option("--flavor", required=True, type=str, help="Flavor name.")
+@click.option("--branch-name", required=True, type=str, help="Branch name.")
+@click.option("--docker-user", required=True, type=str, help="Docker user name")
+@click.option("--docker-password", required=True, type=str, help="Docker password")
+@click.option(
+    "--docker-build-repository", required=True, type=str, help="Docker build repository"
+)
+@click.option(
+    "--docker-release-repository",
+    required=True,
+    type=str,
+    help="Docker release repository",
+)
+@click.option("--commit-sha", required=True, type=str, help="Commit SHA")
+@click.option(
+    "--config-file",
+    required=True,
+    type=click.Path(exists=True, file_okay=True, dir_okay=False),
+    help="The build config file (project specific)",
+)
 @click.pass_context
-def run_ci(ctx: click.Context,
-           flavor: str,
-           branch_name: str,
-           docker_user: str,
-           docker_password: str,
-           docker_build_repository: str,
-           docker_release_repository: str,
-           commit_sha: str,
-           config_file: str):
+def run_ci(
+    ctx: click.Context,
+    flavor: str,
+    branch_name: str,
+    docker_user: str,
+    docker_password: str,
+    docker_build_repository: str,
+    docker_release_repository: str,
+    commit_sha: str,
+    config_file: str,
+):
     logging.basicConfig(level=logging.INFO)
     build_config = Config.parse_file(config_file)
-    ci(flavor=flavor,
-       branch_name=branch_name,
-       docker_user=docker_user,
-       docker_password=docker_password,
-       docker_build_repository=docker_build_repository,
-       docker_release_repository=docker_release_repository,
-       commit_sha=commit_sha,
-       build_config=build_config,
-       git_access=GitAccess())
+    ci(
+        flavor=flavor,
+        branch_name=branch_name,
+        docker_user=docker_user,
+        docker_password=docker_password,
+        docker_build_repository=docker_build_repository,
+        docker_release_repository=docker_release_repository,
+        commit_sha=commit_sha,
+        build_config=build_config,
+        git_access=GitAccess(),
+    )
diff --git a/exasol_script_languages_container_ci/cli/commands/run_release.py b/exasol_script_languages_container_ci/cli/commands/run_release.py
@@ -6,50 +6,69 @@
 from exasol_script_languages_container_ci.cli.cli import cli
 from exasol_script_languages_container_ci.lib.asset_uploader import AssetUploader
 from exasol_script_languages_container_ci.lib.config.config_data_model import Config
-from exasol_script_languages_container_ci.lib.github_release_asset_uploader import GithubReleaseAssetUploader
+from exasol_script_languages_container_ci.lib.github_release_asset_uploader import (
+    GithubReleaseAssetUploader,
+)
 from exasol_script_languages_container_ci.lib.release import release
 from exasol_script_languages_container_ci.lib.release_uploader import ReleaseUploader
 
 
 @cli.command()
-@click.option('--flavor', required=True, type=str,
-              help="Flavor name.")
-@click.option('--docker-user', required=True, type=str,
-              help="Docker user name")
-@click.option('--docker-password', required=True, type=str,
-              help="Docker password")
-@click.option('--docker-release-repository', required=True, type=str,
-              help="Docker release repository")
-@click.option('--config-file', required=True, type=click.Path(exists=True, file_okay=True, dir_okay=False),
-              help="The build config file (project specific)")
-@click.option('--source-repo-url', required=True, type=str,
-              help="The url of the repository. Usually set by AWS under env variable CODEBUILD_SOURCE_REPO_URL.")
-@click.option('--release-id', required=True, type=int,
-              help="The id of the release.")
-@click.option('--dry-run/--no-dry-run', default=False,
-              help="If true, runs release without pushing the container to the docker release repository."
-                   "If false, also pushes the container to the docker release repository.")
+@click.option("--flavor", required=True, type=str, help="Flavor name.")
+@click.option("--docker-user", required=True, type=str, help="Docker user name")
+@click.option("--docker-password", required=True, type=str, help="Docker password")
+@click.option(
+    "--docker-release-repository",
+    required=True,
+    type=str,
+    help="Docker release repository",
+)
+@click.option(
+    "--config-file",
+    required=True,
+    type=click.Path(exists=True, file_okay=True, dir_okay=False),
+    help="The build config file (project specific)",
+)
+@click.option(
+    "--source-repo-url",
+    required=True,
+    type=str,
+    help="The url of the repository. Usually set by AWS under env variable CODEBUILD_SOURCE_REPO_URL.",
+)
+@click.option("--release-id", required=True, type=int, help="The id of the release.")
+@click.option(
+    "--dry-run/--no-dry-run",
+    default=False,
+    help="If true, runs release without pushing the container to the docker release repository."
+    "If false, also pushes the container to the docker release repository.",
+)
 @click.pass_context
-def run_release(ctx: click.Context,
-                flavor: str,
-                docker_user: str,
-                docker_password: str,
-                docker_release_repository: str,
-                config_file: str,
-                source_repo_url: str,
-                release_id: int,
-                dry_run: bool):
+def run_release(
+    ctx: click.Context,
+    flavor: str,
+    docker_user: str,
+    docker_password: str,
+    docker_release_repository: str,
+    config_file: str,
+    source_repo_url: str,
+    release_id: int,
+    dry_run: bool,
+):
     logging.basicConfig(level=logging.INFO)
-    github_release_asset_uploader = GithubReleaseAssetUploader(os.getenv("GITHUB_TOKEN"))
+    github_release_asset_uploader = GithubReleaseAssetUploader(
+        os.getenv("GITHUB_TOKEN")
+    )
     asset_uploader = AssetUploader(release_asset_uploader=github_release_asset_uploader)
     release_uploader = ReleaseUploader(asset_uploader=asset_uploader)
     build_config = Config.parse_file(config_file)
-    release(flavor=flavor,
-            docker_user=docker_user,
-            docker_password=docker_password,
-            docker_release_repository=docker_release_repository,
-            build_config=build_config,
-            source_repo_url=source_repo_url,
-            release_id=release_id,
-            release_uploader=release_uploader,
-            is_dry_run=dry_run)
+    release(
+        flavor=flavor,
+        docker_user=docker_user,
+        docker_password=docker_password,
+        docker_release_repository=docker_release_repository,
+        build_config=build_config,
+        source_repo_url=source_repo_url,
+        release_id=release_id,
+        release_uploader=release_uploader,
+        is_dry_run=dry_run,
+    )
diff --git a/exasol_script_languages_container_ci/lib/asset_uploader.py b/exasol_script_languages_container_ci/lib/asset_uploader.py
@@ -2,25 +2,39 @@
 import logging
 from pathlib import Path
 
-from exasol_script_languages_container_ci.lib.github_release_asset_uploader import GithubReleaseAssetUploader
+from exasol_script_languages_container_ci.lib.github_release_asset_uploader import (
+    GithubReleaseAssetUploader,
+)
 
 
 class AssetUploader:
 
     def __init__(self, release_asset_uploader: GithubReleaseAssetUploader):
         self._release_asset_uploader = release_asset_uploader
 
-    def upload_assets(self,
-                      repo_id: str, release_id: int, content_type: str,
-                      artifact_path: str, file_suffix: str, label_prefix: str):
-        release_artifacts = glob.glob(f'{artifact_path}/*{file_suffix}')
+    def upload_assets(
+        self,
+        repo_id: str,
+        release_id: int,
+        content_type: str,
+        artifact_path: str,
+        file_suffix: str,
+        label_prefix: str,
+    ):
+        release_artifacts = glob.glob(f"{artifact_path}/*{file_suffix}")
         for release_artifact in release_artifacts:
             artifact_file_name = Path(release_artifact).name
             if artifact_file_name.endswith(file_suffix):
-                artifact_file_name = artifact_file_name[:-len(file_suffix)]
+                artifact_file_name = artifact_file_name[: -len(file_suffix)]
             else:
-                logging.error(f"Artifact file: {artifact_file_name} does not end with {file_suffix}. "
-                              f"Using {artifact_file_name} as label.")
-            self._release_asset_uploader.upload(archive_path=release_artifact,
-                                                label=f"{label_prefix} {artifact_file_name}",
-                                                repo_id=repo_id, release_id=release_id, content_type=content_type)
+                logging.error(
+                    f"Artifact file: {artifact_file_name} does not end with {file_suffix}. "
+                    f"Using {artifact_file_name} as label."
+                )
+            self._release_asset_uploader.upload(
+                archive_path=release_artifact,
+                label=f"{label_prefix} {artifact_file_name}",
+                repo_id=repo_id,
+                release_id=release_id,
+                content_type=content_type,
+            )