From 64bcf0aa7894a6fb73bc48af488a9f63b3722ae2 Mon Sep 17 00:00:00 2001 From: Rohan Bansal Date: Tue, 16 Jan 2024 12:49:35 +0530 Subject: [PATCH] ci: update frappe setup jobs --- .github/helper/install.sh | 44 ++++++++ .github/helper/install_dependencies.sh | 14 +++ .github/helper/mariadb.json | 19 ++++ .github/helper/roulette.py | 146 +++++++++++++++++++++++++ .github/workflows/frappe.yaml | 29 ++--- 5 files changed, 235 insertions(+), 17 deletions(-) create mode 100644 .github/helper/install.sh create mode 100644 .github/helper/install_dependencies.sh create mode 100644 .github/helper/mariadb.json create mode 100644 .github/helper/roulette.py diff --git a/.github/helper/install.sh b/.github/helper/install.sh new file mode 100644 index 0000000..75f20ca --- /dev/null +++ b/.github/helper/install.sh @@ -0,0 +1,44 @@ +#!/bin/bash +set -e +cd ~ || exit + +echo "Setting Up Bench..." + +pip install frappe-bench +bench -v init frappe-bench --skip-redis-config-generation --skip-assets --python "$(which python)" --frappe-branch version-15 +cd ./frappe-bench || exit +bench get-app cloud_storage $GITHUB_WORKSPACE +bench -v setup requirements --dev +bench new-site --db-root-password root --admin-password admin test_site +bench --site test_site install-app cloud_storage + +echo "Setting Up Sites & Database..." + +mkdir ~/frappe-bench/sites/test_site +cp "${GITHUB_WORKSPACE}/.github/helper/db/mariadb.json" ~/frappe-bench/sites/test_site/site_config.json + +mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "SET GLOBAL character_set_server = 'utf8mb4'"; +mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'"; +mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE DATABASE test_frappe"; +mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE USER 'test_frappe'@'localhost' IDENTIFIED BY 'test_frappe'"; +mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "GRANT ALL PRIVILEGES ON \`test_frappe\`.* TO 'test_frappe'@'localhost'"; +mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "FLUSH PRIVILEGES"; + +echo "Setting Up Procfile..." + +sed -i 's/^watch:/# watch:/g' Procfile +sed -i 's/^schedule:/# schedule:/g' Procfile +sed -i 's/^socketio:/# socketio:/g' Procfile +sed -i 's/^redis_socketio:/# redis_socketio:/g' Procfile + +echo "Starting Bench..." + +bench start &> ~/frappe-bench/bench_start.log & + +CI=Yes bench build --app frappe & +build_pid=$! + +bench --site test_site reinstall --yes + +# wait till assets are built succesfully +wait $build_pid diff --git a/.github/helper/install_dependencies.sh b/.github/helper/install_dependencies.sh new file mode 100644 index 0000000..574144b --- /dev/null +++ b/.github/helper/install_dependencies.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -e + +echo "Setting Up System Dependencies..." + +sudo apt update +sudo apt remove mysql-server mysql-client +sudo apt install libcups2-dev redis-server mariadb-client-10.6 + +install_wkhtmltopdf() { + wget -q https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.focal_amd64.deb + sudo apt install ./wkhtmltox_0.12.6-1.focal_amd64.deb +} +install_wkhtmltopdf & diff --git a/.github/helper/mariadb.json b/.github/helper/mariadb.json new file mode 100644 index 0000000..0a6c989 --- /dev/null +++ b/.github/helper/mariadb.json @@ -0,0 +1,19 @@ +{ + "db_host": "127.0.0.1", + "db_port": 3306, + "db_name": "test_frappe", + "db_password": "test_frappe", + "allow_tests": true, + "db_type": "mariadb", + "auto_email_id": "test@example.com", + "mail_server": "localhost", + "mail_port": 2525, + "mail_login": "test@example.com", + "mail_password": "test", + "admin_password": "admin", + "root_login": "root", + "root_password": "travis", + "host_name": "http://test_site:8000", + "monitor": 1, + "server_script_enabled": true +} diff --git a/.github/helper/roulette.py b/.github/helper/roulette.py new file mode 100644 index 0000000..e3b212f --- /dev/null +++ b/.github/helper/roulette.py @@ -0,0 +1,146 @@ +import json +import os +import re +import shlex +import subprocess +import sys +import time +import urllib.request +from functools import lru_cache +from urllib.error import HTTPError + + +@lru_cache(maxsize=None) +def fetch_pr_data(pr_number, repo, endpoint=""): + api_url = f"https://api.github.com/repos/{repo}/pulls/{pr_number}" + + if endpoint: + api_url += f"/{endpoint}" + + res = req(api_url) + return json.loads(res.read().decode("utf8")) + + +def req(url): + "Simple resilient request call to handle rate limits." + headers = None + token = os.environ.get("GITHUB_TOKEN") + if token: + headers = {"authorization": f"Bearer {token}"} + + retries = 0 + while True: + try: + req = urllib.request.Request(url, headers=headers) + return urllib.request.urlopen(req) + except HTTPError as exc: + if exc.code == 403 and retries < 5: + retries += 1 + time.sleep(retries) + continue + raise + + +def get_files_list(pr_number, repo="frappe/frappe"): + return [change["filename"] for change in fetch_pr_data(pr_number, repo, "files")] + + +def get_output(command, shell=True): + print(command) + command = shlex.split(command) + return subprocess.check_output(command, shell=shell, encoding="utf8").strip() + + +def has_skip_ci_label(pr_number, repo="frappe/frappe"): + return has_label(pr_number, "Skip CI", repo) + + +def has_run_server_tests_label(pr_number, repo="frappe/frappe"): + return has_label(pr_number, "Run Server Tests", repo) + + +def has_run_ui_tests_label(pr_number, repo="frappe/frappe"): + return has_label(pr_number, "Run UI Tests", repo) + + +def has_label(pr_number, label, repo="frappe/frappe"): + return any( + [ + fetched_label["name"] + for fetched_label in fetch_pr_data(pr_number, repo)["labels"] + if fetched_label["name"] == label + ] + ) + + +def is_py(file): + return file.endswith("py") + + +def is_ci(file): + return ".github" in file + + +def is_frontend_code(file): + return file.lower().endswith( + (".css", ".scss", ".less", ".sass", ".styl", ".js", ".ts", ".vue", ".html") + ) + + +def is_docs(file): + regex = re.compile(r"\.(md|png|jpg|jpeg|csv|svg)$|^.github|LICENSE") + return bool(regex.search(file)) + + +if __name__ == "__main__": + files_list = sys.argv[1:] + build_type = os.environ.get("TYPE") + pr_number = os.environ.get("PR_NUMBER") + repo = os.environ.get("REPO_NAME") + + # this is a push build, run all builds + if not pr_number: + os.system('echo "build=strawberry" >> $GITHUB_OUTPUT') + sys.exit(0) + + files_list = files_list or get_files_list(pr_number=pr_number, repo=repo) + + if not files_list: + print("No files' changes detected. Build is shutting") + sys.exit(0) + + ci_files_changed = any(f for f in files_list if is_ci(f)) + only_docs_changed = len(list(filter(is_docs, files_list))) == len(files_list) + only_frontend_code_changed = len(list(filter(is_frontend_code, files_list))) == len(files_list) + updated_py_file_count = len(list(filter(is_py, files_list))) + only_py_changed = updated_py_file_count == len(files_list) + + if has_skip_ci_label(pr_number, repo): + if build_type == "ui" and has_run_ui_tests_label(pr_number, repo): + print("Running UI tests only.") + elif build_type == "server" and has_run_server_tests_label(pr_number, repo): + print("Running server tests only.") + else: + print("Found `Skip CI` label on pr, stopping build process.") + sys.exit(0) + + elif ci_files_changed: + print("CI related files were updated, running all build processes.") + + elif only_docs_changed: + print("Only docs were updated, stopping build process.") + sys.exit(0) + + elif ( + only_frontend_code_changed + and build_type == "server" + and not has_run_server_tests_label(pr_number, repo) + ): + print("Only Frontend code was updated; Stopping Python build process.") + sys.exit(0) + + elif build_type == "ui" and only_py_changed and not has_run_ui_tests_label(pr_number, repo): + print("Only Python code was updated, stopping Cypress build process.") + sys.exit(0) + + os.system('echo "build=strawberry" >> $GITHUB_OUTPUT') diff --git a/.github/workflows/frappe.yaml b/.github/workflows/frappe.yaml index 74bbbd3..c8e76bc 100644 --- a/.github/workflows/frappe.yaml +++ b/.github/workflows/frappe.yaml @@ -38,6 +38,14 @@ jobs: with: python-version: '3.10' + - name: Check for valid Python & Merge Conflicts + run: | + python -m compileall -q -f "${GITHUB_WORKSPACE}" + if grep -lr --exclude-dir=node_modules "^<<<<<<< " "${GITHUB_WORKSPACE}" + then echo "Found merge conflicts" + exit 1 + fi + - name: Setup Node uses: actions/setup-node@v4 with: @@ -48,14 +56,14 @@ jobs: uses: actions/cache@v3 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/pyproject.toml', '**/setup.py', '**/setup.cfg') }} + key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/pyproject.toml', '**/setup.py') }} restore-keys: | ${{ runner.os }}-pip- ${{ runner.os }}- - name: Get yarn cache directory path id: yarn-cache-dir-path - run: 'echo "::set-output name=dir::$(yarn cache dir)"' + run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT - uses: actions/cache@v3 id: yarn-cache @@ -65,23 +73,10 @@ jobs: restore-keys: | ${{ runner.os }}-yarn- - - name: Setup - run: | - pip install frappe-bench - bench init --skip-redis-config-generation --skip-assets --python "$(which python)" --frappe-branch version-15 ~/frappe-bench - mysql --host 127.0.0.1 --port 3306 -u root -proot -e "SET GLOBAL character_set_server = 'utf8mb4'" - mysql --host 127.0.0.1 --port 3306 -u root -proot -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'" - - name: Install - working-directory: /home/runner/frappe-bench run: | - bench get-app cloud_storage $GITHUB_WORKSPACE --resolve-deps - bench setup requirements --dev - bench new-site --db-root-password root --admin-password admin test_site - bench --site test_site install-app cloud_storage - bench build - env: - CI: 'Yes' + bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh + bash ${GITHUB_WORKSPACE}/.github/helper/install.sh - name: Run Tests working-directory: /home/runner/frappe-bench