diff --git a/.github/helper/install.sh b/.github/helper/install.sh index 3b39bc9..30d0d12 100644 --- a/.github/helper/install.sh +++ b/.github/helper/install.sh @@ -45,4 +45,4 @@ bench start &> bench_run_logs.txt & CI=Yes bench build --app frappe & bench --site test_site reinstall --yes -bench --site test_site install-app cloud_storage \ No newline at end of file +bench --site test_site install-app cloud_storage diff --git a/.github/helper/roulette.py b/.github/helper/roulette.py new file mode 100644 index 0000000..e3b212f --- /dev/null +++ b/.github/helper/roulette.py @@ -0,0 +1,146 @@ +import json +import os +import re +import shlex +import subprocess +import sys +import time +import urllib.request +from functools import lru_cache +from urllib.error import HTTPError + + +@lru_cache(maxsize=None) +def fetch_pr_data(pr_number, repo, endpoint=""): + api_url = f"https://api.github.com/repos/{repo}/pulls/{pr_number}" + + if endpoint: + api_url += f"/{endpoint}" + + res = req(api_url) + return json.loads(res.read().decode("utf8")) + + +def req(url): + "Simple resilient request call to handle rate limits." + headers = None + token = os.environ.get("GITHUB_TOKEN") + if token: + headers = {"authorization": f"Bearer {token}"} + + retries = 0 + while True: + try: + req = urllib.request.Request(url, headers=headers) + return urllib.request.urlopen(req) + except HTTPError as exc: + if exc.code == 403 and retries < 5: + retries += 1 + time.sleep(retries) + continue + raise + + +def get_files_list(pr_number, repo="frappe/frappe"): + return [change["filename"] for change in fetch_pr_data(pr_number, repo, "files")] + + +def get_output(command, shell=True): + print(command) + command = shlex.split(command) + return subprocess.check_output(command, shell=shell, encoding="utf8").strip() + + +def has_skip_ci_label(pr_number, repo="frappe/frappe"): + return has_label(pr_number, "Skip CI", repo) + + +def has_run_server_tests_label(pr_number, repo="frappe/frappe"): + return has_label(pr_number, "Run Server Tests", repo) + + +def has_run_ui_tests_label(pr_number, repo="frappe/frappe"): + return has_label(pr_number, "Run UI Tests", repo) + + +def has_label(pr_number, label, repo="frappe/frappe"): + return any( + [ + fetched_label["name"] + for fetched_label in fetch_pr_data(pr_number, repo)["labels"] + if fetched_label["name"] == label + ] + ) + + +def is_py(file): + return file.endswith("py") + + +def is_ci(file): + return ".github" in file + + +def is_frontend_code(file): + return file.lower().endswith( + (".css", ".scss", ".less", ".sass", ".styl", ".js", ".ts", ".vue", ".html") + ) + + +def is_docs(file): + regex = re.compile(r"\.(md|png|jpg|jpeg|csv|svg)$|^.github|LICENSE") + return bool(regex.search(file)) + + +if __name__ == "__main__": + files_list = sys.argv[1:] + build_type = os.environ.get("TYPE") + pr_number = os.environ.get("PR_NUMBER") + repo = os.environ.get("REPO_NAME") + + # this is a push build, run all builds + if not pr_number: + os.system('echo "build=strawberry" >> $GITHUB_OUTPUT') + sys.exit(0) + + files_list = files_list or get_files_list(pr_number=pr_number, repo=repo) + + if not files_list: + print("No files' changes detected. Build is shutting") + sys.exit(0) + + ci_files_changed = any(f for f in files_list if is_ci(f)) + only_docs_changed = len(list(filter(is_docs, files_list))) == len(files_list) + only_frontend_code_changed = len(list(filter(is_frontend_code, files_list))) == len(files_list) + updated_py_file_count = len(list(filter(is_py, files_list))) + only_py_changed = updated_py_file_count == len(files_list) + + if has_skip_ci_label(pr_number, repo): + if build_type == "ui" and has_run_ui_tests_label(pr_number, repo): + print("Running UI tests only.") + elif build_type == "server" and has_run_server_tests_label(pr_number, repo): + print("Running server tests only.") + else: + print("Found `Skip CI` label on pr, stopping build process.") + sys.exit(0) + + elif ci_files_changed: + print("CI related files were updated, running all build processes.") + + elif only_docs_changed: + print("Only docs were updated, stopping build process.") + sys.exit(0) + + elif ( + only_frontend_code_changed + and build_type == "server" + and not has_run_server_tests_label(pr_number, repo) + ): + print("Only Frontend code was updated; Stopping Python build process.") + sys.exit(0) + + elif build_type == "ui" and only_py_changed and not has_run_ui_tests_label(pr_number, repo): + print("Only Python code was updated, stopping Cypress build process.") + sys.exit(0) + + os.system('echo "build=strawberry" >> $GITHUB_OUTPUT') diff --git a/.github/helper/site_config.json b/.github/helper/site_config.json index b9d0173..18ba83c 100644 --- a/.github/helper/site_config.json +++ b/.github/helper/site_config.json @@ -1,19 +1,19 @@ { - "db_host": "127.0.0.1", - "db_port": 3306, - "db_name": "test_frappe", - "db_password": "test_frappe", - "allow_tests": true, - "db_type": "mariadb", - "auto_email_id": "test@example.com", - "mail_server": "localhost", - "mail_port": 2525, - "mail_login": "test@example.com", - "mail_password": "test", - "admin_password": "admin", - "root_login": "root", - "root_password": "root", - "host_name": "http://test_site:8000", - "monitor": 1, - "server_script_enabled": true -} \ No newline at end of file + "db_host": "127.0.0.1", + "db_port": 3306, + "db_name": "test_frappe", + "db_password": "test_frappe", + "allow_tests": true, + "db_type": "mariadb", + "auto_email_id": "test@example.com", + "mail_server": "localhost", + "mail_port": 2525, + "mail_login": "test@example.com", + "mail_password": "test", + "admin_password": "admin", + "root_login": "root", + "root_password": "root", + "host_name": "http://test_site:8000", + "monitor": 1, + "server_script_enabled": true +} diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index fc39f29..a02a968 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -29,7 +29,7 @@ jobs: with: python-version: '3.10' - - name: Install mypy and types + - name: Install mypy and types run: python -m pip install mypy types-python-dateutil types-pytz --no-input - name: Run mypy @@ -56,7 +56,7 @@ jobs: - name: Install Black (Frappe) run: pip install git+https://github.com/frappe/black.git - + - name: Run Black (Frappe) run: black --check . @@ -149,7 +149,7 @@ jobs: run: git clone --depth 1 https://gist.github.com/f1bf2c11f78331b2417189c385022c28.git validate_json - name: Validate JSON - run: python3 validate_json/validate_json.py ./inventory_tools/inventory_tools/ + run: python3 validate_json/validate_json.py ./cloud_storage/cloud_storage/ - name: Compile run: python3 -m compileall -q ./ diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 841a92d..b4d9f60 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -5,6 +5,7 @@ on: branches: - version-14 - version-15 + jobs: release: name: Release diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 87a76a0..1fbe0ae 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -38,11 +38,10 @@ repos: # Ignore any files that might contain jinja / bundles exclude: | (?x)^( - .*node_modules.*| - cloud_storage/public/dist/.*| - cloud_storage/public/js/lib/.*| - cloud_storage/templates/includes/.*| - cloud_storage/www/website_script.js + .*node_modules.*| + cloud_storage/public/dist/.*| + cloud_storage/public/js/lib/.*| + cloud_storage/templates/includes/.* )$ - repo: https://github.com/pre-commit/mirrors-eslint @@ -61,6 +60,22 @@ repos: cloud_storage/www/website_script.js )$ + - repo: https://github.com/pre-commit/mirrors-eslint + rev: v8.44.0 + hooks: + - id: eslint + types_or: [javascript] + args: ['--quiet'] + # Ignore any files that might contain jinja / bundles + exclude: | + (?x)^( + .*node_modules.*| + cloud_storage/public/dist/.*| + cloud_storage/public/js/lib/.*| + cloud_storage/templates/includes/.*| + cloud_storage/www/website_script.js + )$ + - repo: https://github.com/PyCQA/isort rev: 5.12.0 hooks: