From f2e83dee3d04d7344263d385cf222ef30c11e8a1 Mon Sep 17 00:00:00 2001 From: vardhaman22 Date: Wed, 20 Mar 2024 14:55:40 +0530 Subject: [PATCH] added automation for new version automated PR --- .github/workflows/add_new_versions.yml | 85 ++++++++++++++++++ workflow_scripts/check-for-new-versions.py | 87 ++++++++++++++++++ workflow_scripts/gen-new-version-files.py | 100 +++++++++++++++++++++ workflow_scripts/requirements.txt | 1 + 4 files changed, 273 insertions(+) create mode 100644 .github/workflows/add_new_versions.yml create mode 100644 workflow_scripts/check-for-new-versions.py create mode 100644 workflow_scripts/gen-new-version-files.py create mode 100644 workflow_scripts/requirements.txt diff --git a/.github/workflows/add_new_versions.yml b/.github/workflows/add_new_versions.yml new file mode 100644 index 0000000..bca712e --- /dev/null +++ b/.github/workflows/add_new_versions.yml @@ -0,0 +1,85 @@ +name: Add New Docker Versions + +on: + workflow_dispatch: + schedule: + - cron: "0 0 * * 3" + + +permissions: + contents: write + pull-requests: write + + +jobs: + generate_and_raise_pr: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.10' + cache: 'pip' + + - name: Pip + working-directory: ./workflow_scripts + run: pip install -r requirements.txt + + - name: Check if new versions available + id: check-versions + run: | + python workflow_scripts/check-for-new-versions.py + env: + EXCLUDED_VERSIONS: "v20.10.x,v23.0.x" + + - name: check if the PR exist + if: ${{ env.PR_TITLE != '' }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_TITLE: ${{env.PR_TITLE}} + run: | + EXISTING_PR=$(gh pr list --limit 1500 --json title,url | jq --arg title "${PR_TITLE}" -r '.[] | select(.title==$title) | .url') + if [ -n "${EXISTING_PR}" ]; then + echo "pr_exist=true" >> $GITHUB_ENV + echo "Pull request already exists: ${EXISTING_PR}" >> $GITHUB_STEP_SUMMARY + else + echo "pr_exist=false" >> $GITHUB_ENV + fi + + - name: generate files for new docker version + if: ${{ env.pr_exist == 'false' && env.PR_TITLE != '' }} + env: + NEW_VERSIONS: ${{ env.NEW_VERSIONS }} + run: | + python workflow_scripts/gen-new-version-files.py + + - name: Create branch, commit and push + if: ${{ env.pr_exist == 'false' && env.PR_TITLE != '' }} + id: branch + env: + NEW_VERSIONS: ${{ env.NEW_VERSIONS }} + run: | + BRANCH="gha-add-tag-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + echo "branch=${BRANCH}" >> $GITHUB_OUTPUT + git config user.name github-actions + git config user.email github-actions@github.com + git checkout -b "$BRANCH" + git add . + git commit -m "added docker ${NEW_VERSIONS}" + git push origin "$BRANCH" + + - name: Create Pull Request + if: ${{ env.pr_exist == 'false' && env.PR_TITLE != '' }} + id: cpr + env: + SOURCE_BRANCH: ${{ steps.branch.outputs.branch }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_TITLE: ${{env.PR_TITLE}} + PR_BODY: autogenerated PR to add docker ${{env.NEW_VERSIONS}} + run: | + PR_TITLE=$(echo "$PR_TITLE" | cut -c -256) + CREATED_PR=$(gh pr create --title "${PR_TITLE}" --body "${PR_BODY}" --label "status/auto-created" --base "${GITHUB_REF_NAME}" --head "${SOURCE_BRANCH}") + echo "Created pull request: ${CREATED_PR}" >> $GITHUB_STEP_SUMMARY diff --git a/workflow_scripts/check-for-new-versions.py b/workflow_scripts/check-for-new-versions.py new file mode 100644 index 0000000..6c14ed0 --- /dev/null +++ b/workflow_scripts/check-for-new-versions.py @@ -0,0 +1,87 @@ +import os +import subprocess +import requests + +# Constants +DIST_FOLDER = './dist' +EXCLUDED_VERSIONS = os.environ.get('EXCLUDED_VERSIONS', 'v20.10.x,v23.0.x') + +def get_excluded_version_patterns(excluded_ver_str): + excluded_ver_list = excluded_ver_str.split(',') + excluded_patterns = [] + for ver in excluded_ver_list: + ver_parts = ver.split('.') + if len(ver_parts) == 3: + if ver_parts[1] == 'x': + excluded_patterns.append(ver_parts[0]) + elif ver_parts[2] == 'x': + excluded_patterns.append(ver_parts[0] + '.'+ ver_parts[1]) + else: + excluded_patterns.append(ver) + return excluded_patterns + + +def is_excluded_version(excluded_patterns,version): + for pattern in excluded_patterns: + if version.startswith(pattern): + return True + return False + +def get_existing_versions(files_dir): + existing_versions = set() + for file in os.listdir(files_dir): + if file.endswith('.sh') and file.count('.') == 3: + existing_versions.add('v' + file[:-3]) + return existing_versions + +def fetch_ten_latest_github_releases(owner, repo): + url = f"https://api.github.com/repos/{owner}/{repo}/releases" + try: + response = requests.get(url) + response.raise_for_status() # Raise exception for bad status codes + releases = [release for release in response.json() if not release.get('prerelease')] + return sorted(releases, key=lambda x: x['created_at'], reverse=True)[:10] + except requests.exceptions.RequestException as e: + print(f"Failed to fetch releases: {e}") + return None + +def get_version_tuple(version): + if version.startswith('v'): + version = version[1:] + return tuple(map(int, version.split('.'))) + +def main(): + excluded_ver_patterns = get_excluded_version_patterns(EXCLUDED_VERSIONS) + existing_versions = get_existing_versions(DIST_FOLDER) + owner = "moby" + repo = "moby" + ten_latest_releases = fetch_ten_latest_github_releases(owner, repo) + ten_latest_versions = [release['tag_name'] for release in ten_latest_releases] + print("Ten latest versions: ",ten_latest_versions) + + new_versions = set(ten_latest_versions) - existing_versions + new_versions = list(filter(lambda ver: not is_excluded_version(excluded_ver_patterns,ver),new_versions)) + + sorted_new_versions = sorted(new_versions,key=get_version_tuple) + print('New versions: ',sorted_new_versions) + + versions_string = ",".join(sorted_new_versions) + PR_TITLE = "" + + if versions_string != "": + PR_TITLE = "[Auto] Add docker " + versions_string + print('PR Title: ', PR_TITLE) + + env_file = os.getenv('GITHUB_ENV') + + if env_file: + with open(env_file, "a") as envfile: + envfile.write("PR_TITLE="+PR_TITLE+"\n") + envfile.write("NEW_VERSIONS="+versions_string+"\n") + else: + exit(1) + + + +if __name__ == "__main__": + main() diff --git a/workflow_scripts/gen-new-version-files.py b/workflow_scripts/gen-new-version-files.py new file mode 100644 index 0000000..3861e68 --- /dev/null +++ b/workflow_scripts/gen-new-version-files.py @@ -0,0 +1,100 @@ +import os +import subprocess +import requests + +# Constants +DIST_FOLDER = './dist' +NEW_VERSIONS = os.environ.get("NEW_VERSIONS","") + +if NEW_VERSIONS == "": + print("no new versions available, NEW_VERSIONS env variable is empty") + exit(1) + +def get_max_version(ver1, ver2): + if ver1.startswith('v'): + ver1 = ver1[1:] + if ver2.startswith('v'): + ver2 = ver2[1:] + + ver1_tuple = tuple(map(int, ver1.split('.'))) + ver2_tuple = tuple(map(int, ver2.split('.'))) + if ver1 > ver2: + return ver1 + return ver2 + +def format_version(v): + if v.startswith('v'): + return v[1:] + return v + +def get_last_added_version(files_dir): + max_modification_time = 0.0 + last_added_version = "" + for file in os.listdir(files_dir): + if file.endswith('.sh') and file.count('.') == 3: + file_path = os.path.join(files_dir, file) + modification_time = os.path.getmtime(file_path) + file_version = 'v' + file[:-3] + if modification_time > max_modification_time: + max_modification_time = modification_time + last_added_version = file_version + elif modification_time == max_modification_time: + if last_added_version == "": + last_added_version = file_version + else: + last_added_version = get_max_version(last_added_version,file_version) + + return last_added_version + +def generate_diffs(prev_version, current_version): + print("executing add-new-version-script with PREVIOUS_ADD_DOCKER_VERSION: ",prev_version, + " ADD_DOCKER_VERSION",current_version) + add_new_version_script_path = "./scripts/add-new-version" + env_vars = {"PREVIOUS_ADD_DOCKER_VERSION": prev_version, "ADD_DOCKER_VERSION": current_version} + subprocess.run(["bash", add_new_version_script_path], check=True, env=env_vars) + +# it will return a dictonary with key set to major.minor of a version and the +# value will be the greatest version for that major minor combination +def get_version_dict(versions): + version_dict = {} + + for version in versions: + version_parts = version.split('.') + major_minor = version_parts[0] + '.' + version_parts[1] + + if major_minor in version_dict: + current_version = tuple(map(int, version_parts[2])) + max_version = tuple(map(int, version_dict[major_minor].split('.')[2])) + if current_version > max_version: + version_dict[major_minor] = version + else: + version_dict[major_minor] = version + + return version_dict + +def main(): + + last_added_version = get_last_added_version(DIST_FOLDER) + print("Last added version:",last_added_version) + + new_versions = NEW_VERSIONS.split(',') + formatted_new_versions = list(map(format_version,new_versions)) + print("Formatted new versions: ", formatted_new_versions) + + for version in formatted_new_versions: + generate_diffs(last_added_version, version) + + versions_string = ",".join(new_versions) + + print("running generate script") + subprocess.run(["bash", "./scripts/generate"], check=True) + + version_dict = get_version_dict(formatted_new_versions) + print("version dictionary for symlink: ",version_dict) + + for major_minor,version in version_dict.items(): + subprocess.call(['rm',f"{DIST_FOLDER}/{major_minor}.sh"]) + os.symlink(f"{version}.sh",f"{DIST_FOLDER}/{major_minor}.sh") + +if __name__ == "__main__": + main() diff --git a/workflow_scripts/requirements.txt b/workflow_scripts/requirements.txt new file mode 100644 index 0000000..9688b8e --- /dev/null +++ b/workflow_scripts/requirements.txt @@ -0,0 +1 @@ +Requests==2.31.0