Skip to content

Commit

Permalink
add flink operator mirror
Browse files Browse the repository at this point in the history
  • Loading branch information
AndersBennedsgaard committed Oct 30, 2023
1 parent 44acb12 commit bee6953
Show file tree
Hide file tree
Showing 3 changed files with 177 additions and 0 deletions.
133 changes: 133 additions & 0 deletions .github/scripts/sync.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
"""
Upload Flink operator Helm chart to OCI registry
Usage:
python sync.py
Environment variables:
GITHUB_TOKEN: GitHub token with read/write access to the OCI registry
Requirements:
requests (pip install requests)
helm (https://helm.sh/docs/intro/install/)
"""

import base64
import os
import re
import subprocess
import sys

import requests

ARCHIVE_URL = "https://archive.apache.org/dist/flink/"

GIT_REPO = "trifork/cheetah-charts"
IMAGE = "flink-kubernetes-operator"


def get_existing_versions() -> list[str]:
"""Get existing versions"""
token = os.getenv("GITHUB_TOKEN")
if token is None:
print("GITHUB_TOKEN is not set")
sys.exit(1)

b64_token_bytes = base64.b64encode(token.encode("ascii"))
headers = {"Authorization": f"Bearer {b64_token_bytes.decode('ascii')}"}
resp = requests.get(
f"https://ghcr.io/v2/{GIT_REPO}/{IMAGE}/tags/list",
timeout=10,
headers=headers,
)

result = resp.json()
if "tags" not in result:
return []

return result["tags"]


def download_chart(url: str, file: str):
"""Download Helm chart"""
resp = requests.get(url, timeout=10)
try:
resp.raise_for_status()
except requests.exceptions.HTTPError as err:
print(f"Error collecting tar ball from {url}: {err}")
sys.exit(1)

with open(file, "wb") as f:
f.write(resp.content)


def helm_login():
"""Login to the Helm registry"""
token = os.getenv("GITHUB_TOKEN")
if token is None:
print("GITHUB_TOKEN is not set")
sys.exit(1)

cmd = [
"helm",
"registry",
"login",
f"ghcr.io/{GIT_REPO}",
"--username",
"github-actions",
"--password-stdin",
]
try:
subprocess.run(cmd, input=token.encode("ascii"), check=True, timeout=10)
except subprocess.CalledProcessError as err:
print(f"Error logging into the Helm registry: {err}")
sys.exit(1)


def helm_push(chart: str):
"""Push the Helm chart"""
cmd = ["helm", "push", chart, f"oci://ghcr.io/{GIT_REPO}/{IMAGE}"]
try:
subprocess.run(cmd, check=True, timeout=10)
except subprocess.CalledProcessError as err:
print(f"Error pushing Helm chart: {err}")
sys.exit(1)


def main():
"""Main function"""
# Log into the Helm registry
helm_login()

# Get existing versions
existing_versions = get_existing_versions()
print(f"Found existing versions: {existing_versions}")

# Collect versions from the Apache archive
resp = requests.get(ARCHIVE_URL, timeout=10)
try:
resp.raise_for_status()
except requests.exceptions.HTTPError as err:
print(f"Error collecting Kubernetes operator versions: {err}")
sys.exit(1)

pattern = re.compile(
r"<a href=\"flink-kubernetes-operator-(?P<version>\d+\.\d+\.\d+)/\">"
)
for result in pattern.finditer(resp.text):
version = result.group("version")
print(f"Found version {version}")

if version in existing_versions:
print(f"Version {version} already exists")
continue

tar_url = f"{ARCHIVE_URL}flink-kubernetes-operator-{version}/flink-kubernetes-operator-{version}-helm.tgz"
chart = f"{IMAGE}-{version}.tgz"

download_chart(url=tar_url, file=chart)
helm_push(chart)


if __name__ == "__main__":
main()
37 changes: 37 additions & 0 deletions .github/workflows/flink-operator-sync.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
name: Flink operator Sync

on:
workflow_dispatch: {}
schedule:
- cron: "0 0 * * *" # every day at midnight

permissions:
packages: write

concurrency:
group: flink-operator-sync
cancel-in-progress: true

jobs:
sync:
name: Update Flink operator
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: "3.x"

- name: Install Helm
uses: azure/setup-helm@v1
with:
version: v3.13.1

- name: Install requests
run: pip install requests

- name: Run sync
run: python .github/scripts/sync.py
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
7 changes: 7 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,13 @@
[![Chart linting](https://github.com/trifork/cheetah-charts/actions/workflows/lint.yaml/badge.svg)](https://github.com/trifork/cheetah-charts/actions/workflows/lint.yaml)
[![Release to GHCR](https://github.com/trifork/cheetah-charts/actions/workflows/release-oci.yaml/badge.svg)](https://github.com/trifork/cheetah-charts/actions/workflows/release-oci.yaml)
[![Release to Github releases](https://github.com/trifork/cheetah-charts/actions/workflows/release.yaml/badge.svg)](https://github.com/trifork/cheetah-charts/actions/workflows/release.yaml)
[![Flink operator Sync](https://github.com/trifork/cheetah-charts/actions/workflows/flink-operator-sync.yaml/badge.svg)](https://github.com/trifork/cheetah-charts/actions/workflows/flink-operator-sync.yaml)

Repository containing the source code for Helm charts used in the Trifork Data-platform.

Additionally, this repository contains a GitHub action for syncing the [Flink operator](https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-main/).
The reason for this, is that Apache does not release new versions of the operator to the same Helm repository, making upgrades difficult.

## Usage

These helm charts are released to both GHCR (GitHub Container Registry) as OCI packages, and to GitHub releases as tar-ball assets.
Expand Down Expand Up @@ -40,8 +44,11 @@ Additionally, pull-requests will also create a pre-release to GitHub releases, w
After pull-requests has been merged to the main branch, charts that have changed version will be packaged and released.

## Development

### Prerequisites

For convenience, this repository uses `make` for linting and generating docs. Most dev-boxes already have `make`, and can easily be installed if you don't. On Windows, using `chocolatey`, simply run:

```bash
choco install make
```
Expand Down

0 comments on commit bee6953

Please sign in to comment.