From 3f3491f9706eca452e8bffb7343ff7f61c11f70c Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Thu, 5 Dec 2024 16:56:43 +0100 Subject: [PATCH] Add back support for snap & rocks (not ST124 support) Adds support for core24 `platforms` in snapcraft.yaml --- .github/workflows/build_rock.md | 14 +++++++- .github/workflows/build_rock.yaml | 22 ++++++------ .github/workflows/build_snap.md | 33 ++++++++++++++++- .github/workflows/build_snap.yaml | 26 +++++++------- .../craft_tools/collect_platforms.py | 35 ++++++++++++++++--- python/cli/pyproject.toml | 4 +-- 6 files changed, 102 insertions(+), 32 deletions(-) diff --git a/.github/workflows/build_rock.md b/.github/workflows/build_rock.md index c933765b..89d2eaa4 100644 --- a/.github/workflows/build_rock.md +++ b/.github/workflows/build_rock.md @@ -8,4 +8,16 @@ jobs: build: name: Build rock uses: canonical/data-platform-workflows/.github/workflows/build_rock.yaml@v0.0.0 -``` \ No newline at end of file +``` + +### Supported `platforms` syntax in rockcraft.yaml +Only "shorthand notation" is supported + +Example rockcraft.yaml +```yaml +platforms: + amd64: + arm64: +``` + +`build-on` and `build-for` are not supported diff --git a/.github/workflows/build_rock.yaml b/.github/workflows/build_rock.yaml index a46fa4a3..2e295984 100644 --- a/.github/workflows/build_rock.yaml +++ b/.github/workflows/build_rock.yaml @@ -28,10 +28,10 @@ on: outputs: artifact-prefix: description: Rock packages are uploaded to GitHub artifacts beginning with this prefix - value: ${{ jobs.collect-bases.outputs.artifact-prefix-with-inputs }} + value: ${{ jobs.collect-platforms.outputs.artifact-prefix-with-inputs }} jobs: - collect-bases: + collect-platforms: name: Collect platforms for rock | ${{ inputs.path-to-rock-directory }} runs-on: ubuntu-latest timeout-minutes: 5 @@ -49,19 +49,19 @@ jobs: uses: actions/checkout@v4 - name: Collect rock platforms to build from rockcraft.yaml id: collect - run: collect-rock-bases --directory='${{ inputs.path-to-rock-directory }}' + run: collect-rock-platforms --directory='${{ inputs.path-to-rock-directory }}' outputs: - bases: ${{ steps.collect.outputs.bases }} + platforms: ${{ steps.collect.outputs.platforms }} artifact-prefix-with-inputs: ${{ inputs.artifact-prefix || steps.collect.outputs.default_prefix }} build: strategy: matrix: - base: ${{ fromJSON(needs.collect-bases.outputs.bases) }} - name: 'Build rock | ${{ matrix.base.id }}' + platform: ${{ fromJSON(needs.collect-platforms.outputs.platforms) }} + name: 'Build rock | ${{ matrix.platform.name }}' needs: - - collect-bases - runs-on: ${{ matrix.base.runner }} + - collect-platforms + runs-on: ${{ matrix.platform.runner }} timeout-minutes: 15 steps: - name: Get workflow version @@ -93,19 +93,19 @@ jobs: - name: Pack rock id: pack working-directory: ${{ inputs.path-to-rock-directory }} - run: sg lxd -c "rockcraft pack -v --platform='${{ matrix.base.id }}'" + run: sg lxd -c "rockcraft pack -v --platform='${{ matrix.platform.name }}'" - name: Upload rockcraft logs if: ${{ failure() && steps.pack.outcome == 'failure' }} uses: actions/upload-artifact@v4 with: - name: logs-rockcraft-build-${{ inputs.artifact-prefix }}-architecture-${{ matrix.base.id }} + name: logs-rockcraft-build-${{ inputs.artifact-prefix }}-platform-${{ matrix.platform.name }} path: ~/.local/state/rockcraft/log/ if-no-files-found: error - run: touch .empty - name: Upload rock package uses: actions/upload-artifact@v4 with: - name: ${{ needs.collect-bases.outputs.artifact-prefix-with-inputs }}-architecture-${{ matrix.base.id }} + name: ${{ needs.collect-platforms.outputs.artifact-prefix-with-inputs }}-platform-${{ matrix.platform.name }} # .empty file required to preserve directory structure # See https://github.com/actions/upload-artifact/issues/344#issuecomment-1379232156 path: | diff --git a/.github/workflows/build_snap.md b/.github/workflows/build_snap.md index 1ac88e00..181cf6a8 100644 --- a/.github/workflows/build_snap.md +++ b/.github/workflows/build_snap.md @@ -8,4 +8,35 @@ jobs: build: name: Build snap uses: canonical/data-platform-workflows/.github/workflows/build_snap.yaml@v0.0.0 -``` \ No newline at end of file +``` + +### Supported `platforms` and `architectures` syntax in snapcraft.yaml +See https://snapcraft.io/docs/architectures#how-to-create-a-snap-for-a-specific-architecture + +#### core24 +Only `platforms` is supported. `architectures` is not supported + +Only "shorthand notation" is supported + +Example snapcraft.yaml +```yaml +platforms: + amd64: + arm64: +``` + +`build-on` and `build-for` are not supported + +#### core22 +Only `architectures` is supported. `platforms` is not supported + +`architectures` must be a list of dictionaries. Each dictionary in the list must contain a `build-on` key + +Example snapcraft.yaml +```yaml +architectures: + - build-on: [amd64] + build-for: [amd64] + - build-on: [arm64] + build-for: [arm64] +``` diff --git a/.github/workflows/build_snap.yaml b/.github/workflows/build_snap.yaml index d7e46faa..f8ecee00 100644 --- a/.github/workflows/build_snap.yaml +++ b/.github/workflows/build_snap.yaml @@ -31,11 +31,11 @@ on: outputs: artifact-prefix: description: Snap packages are uploaded to GitHub artifacts beginning with this prefix - value: ${{ jobs.collect-bases.outputs.artifact-prefix-with-inputs }} + value: ${{ jobs.collect-platforms.outputs.artifact-prefix-with-inputs }} jobs: - collect-bases: - name: Collect architectures for snap | ${{ inputs.path-to-snap-project-directory }} + collect-platforms: + name: Collect platforms for snap | ${{ inputs.path-to-snap-project-directory }} runs-on: ubuntu-latest timeout-minutes: 5 steps: @@ -50,21 +50,21 @@ jobs: run: pipx install git+https://github.com/canonical/data-platform-workflows@'${{ steps.workflow-version.outputs.sha }}'#subdirectory=python/cli - name: Checkout uses: actions/checkout@v4 - - name: Collect snap architectures to build from snapcraft.yaml + - name: Collect snap platforms to build from snapcraft.yaml id: collect - run: collect-snap-bases --directory='${{ inputs.path-to-snap-project-directory }}' + run: collect-snap-platforms --directory='${{ inputs.path-to-snap-project-directory }}' outputs: - bases: ${{ steps.collect.outputs.bases }} + platforms: ${{ steps.collect.outputs.platforms }} artifact-prefix-with-inputs: ${{ inputs.artifact-prefix || steps.collect.outputs.default_prefix }} build: strategy: matrix: - base: ${{ fromJSON(needs.collect-bases.outputs.bases) }} - name: 'Build snap | ${{ matrix.base.id }}' + platform: ${{ fromJSON(needs.collect-platforms.outputs.platforms) }} + name: 'Build snap | ${{ matrix.platform.name }}' needs: - - collect-bases - runs-on: ${{ matrix.base.runner }} + - collect-platforms + runs-on: ${{ matrix.platform.runner }} timeout-minutes: 30 steps: - name: Get workflow version @@ -96,19 +96,19 @@ jobs: - name: Pack snap id: pack working-directory: ${{ inputs.path-to-snap-project-directory }} - run: sg lxd -c "snapcraft pack -v --build-for='${{ matrix.base.id }}'" + run: sg lxd -c "snapcraft pack -v --build-for='${{ matrix.platform.name }}'" - name: Upload snapcraft logs if: ${{ failure() && steps.pack.outcome == 'failure' }} uses: actions/upload-artifact@v4 with: - name: logs-snapcraft-build-${{ inputs.artifact-prefix }}-architecture-${{ matrix.base.id }} + name: logs-snapcraft-build-${{ inputs.artifact-prefix }}-platform-${{ matrix.platform.name }} path: ~/.local/state/snapcraft/log/ if-no-files-found: error - run: touch .empty - name: Upload snap package uses: actions/upload-artifact@v4 with: - name: ${{ needs.collect-bases.outputs.artifact-prefix-with-inputs }}-architecture-${{ matrix.base.id }} + name: ${{ needs.collect-platforms.outputs.artifact-prefix-with-inputs }}-platform-${{ matrix.platform.name }} # .empty file required to preserve directory structure # See https://github.com/actions/upload-artifact/issues/344#issuecomment-1379232156 path: | diff --git a/python/cli/data_platform_workflows_cli/craft_tools/collect_platforms.py b/python/cli/data_platform_workflows_cli/craft_tools/collect_platforms.py index 7bbff23d..18d8bbde 100644 --- a/python/cli/data_platform_workflows_cli/craft_tools/collect_platforms.py +++ b/python/cli/data_platform_workflows_cli/craft_tools/collect_platforms.py @@ -34,9 +34,8 @@ def collect(craft_: craft.Craft): if craft_ is craft.Craft.SNAP: craft_file = craft_file.parent / "snap" / craft_file.name yaml_data = yaml.safe_load(craft_file.read_text()) + platforms = [] if craft_ is craft.Craft.CHARM: - # todo: run ccst124 validate - platforms = [] for platform in yaml_data["platforms"]: # Example `platform`: "ubuntu@22.04:amd64" architecture = craft.Architecture(platform.split(":")[-1]) @@ -47,9 +46,37 @@ def collect(craft_: craft.Craft): "name_in_artifact": platform.replace(":", "-"), } ) - github_actions.output["platforms"] = json.dumps(platforms) + elif craft_ is craft.Craft.ROCK: + for platform in yaml_data["platforms"]: + # Example `platform`: "amd64" + architecture = craft.Architecture(platform) + platforms.append({"name": platform, "runner": RUNNERS[architecture]}) + elif craft_ is craft.Craft.SNAP: + if yaml_data["base"] == "core24": + platforms_ = yaml_data["platforms"] + if not isinstance(platforms_, dict): + raise TypeError("Expected type 'dict' for snapcraft.yaml 'platforms'") + for value in platforms_.values(): + if value is not None: + raise ValueError( + "Only shorthand notation supported in snapcraft.yaml 'platforms'. " + "'build-on' and 'build-for' not supported" + ) + for platform in platforms_: + # Example `platform`: "amd64" + architecture = craft.Architecture(platform) + platforms.append({"name": platform, "runner": RUNNERS[architecture]}) + elif yaml_data["base"] == "core22": + for entry in yaml_data["architectures"]: + # Example: "amd64" + platform = entry["build-on"] + architecture = craft.Architecture(platform) + platforms.append({"name": platform, "runner": RUNNERS[architecture]}) + else: + raise ValueError(f'Unsupported snapcraft.yaml base: {repr(yaml_data["base"])}') else: - raise ValueError("ST124 syntax not yet supported for snaps or rocks") + raise ValueError + github_actions.output["platforms"] = json.dumps(platforms) default_prefix = f'packed-{craft_.value}-{args.directory.replace("/", "-")}' if craft_ is craft.Craft.CHARM: default_prefix = f'packed-{craft_.value}-cache-{args.cache}-{args.directory.replace("/", "-")}' diff --git a/python/cli/pyproject.toml b/python/cli/pyproject.toml index 9029c18a..bf7dae2b 100644 --- a/python/cli/pyproject.toml +++ b/python/cli/pyproject.toml @@ -9,8 +9,8 @@ readme = "README.md" [tool.poetry.scripts] redact-secrets = "data_platform_workflows_cli.redact_secrets:main" -collect-snap-bases = "data_platform_workflows_cli.craft_tools.collect_platforms:snap" -collect-rock-bases = "data_platform_workflows_cli.craft_tools.collect_platforms:rock" +collect-snap-platforms = "data_platform_workflows_cli.craft_tools.collect_platforms:snap" +collect-rock-platforms = "data_platform_workflows_cli.craft_tools.collect_platforms:rock" collect-charm-platforms = "data_platform_workflows_cli.craft_tools.collect_platforms:charm" release-snap = "data_platform_workflows_cli.craft_tools.release:snap" release-rock = "data_platform_workflows_cli.craft_tools.release:rock"