ci: update frappe setup jobs

.github/helper/install.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+set -e
+cd ~ || exit
+
+echo "Setting Up Bench..."
+
+pip install frappe-bench
+bench -v init frappe-bench --skip-redis-config-generation --skip-assets --python "$(which python)" --frappe-branch version-15
+cd ./frappe-bench || exit
+bench get-app cloud_storage $GITHUB_WORKSPACE
+bench -v setup requirements --dev
+bench new-site --db-root-password root --admin-password admin test_site
+bench --site test_site install-app cloud_storage
+
+echo "Setting Up Sites & Database..."
+
+mkdir ~/frappe-bench/sites/test_site
+cp "${GITHUB_WORKSPACE}/.github/helper/db/mariadb.json" ~/frappe-bench/sites/test_site/site_config.json
+
+mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "SET GLOBAL character_set_server = 'utf8mb4'";
+mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'";
+mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE DATABASE test_frappe";
+mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE USER 'test_frappe'@'localhost' IDENTIFIED BY 'test_frappe'";
+mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "GRANT ALL PRIVILEGES ON \`test_frappe\`.* TO 'test_frappe'@'localhost'";
+mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "FLUSH PRIVILEGES";
+
+echo "Setting Up Procfile..."
+
+sed -i 's/^watch:/# watch:/g' Procfile
+sed -i 's/^schedule:/# schedule:/g' Procfile
+sed -i 's/^socketio:/# socketio:/g' Procfile
+sed -i 's/^redis_socketio:/# redis_socketio:/g' Procfile
+
+echo "Starting Bench..."
+
+bench start &> ~/frappe-bench/bench_start.log &
+
+CI=Yes bench build --app frappe &
+build_pid=$!
+
+bench --site test_site reinstall --yes
+
+# wait till assets are built succesfully
+wait $build_pid

.github/helper/install_dependencies.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+set -e
+
+echo "Setting Up System Dependencies..."
+
+sudo apt update
+sudo apt remove mysql-server mysql-client
+sudo apt install libcups2-dev redis-server mariadb-client-10.6
+
+install_wkhtmltopdf() {
+  wget -q https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.focal_amd64.deb
+  sudo apt install ./wkhtmltox_0.12.6-1.focal_amd64.deb
+}
+install_wkhtmltopdf &

.github/helper/mariadb.json

@@ -0,0 +1,19 @@
+{
+    "db_host": "127.0.0.1",
+    "db_port": 3306,
+    "db_name": "test_frappe",
+    "db_password": "test_frappe",
+    "allow_tests": true,
+    "db_type": "mariadb",
+    "auto_email_id": "[email protected]",
+    "mail_server": "localhost",
+    "mail_port": 2525,
+    "mail_login": "[email protected]",
+    "mail_password": "test",
+    "admin_password": "admin",
+    "root_login": "root",
+    "root_password": "travis",
+    "host_name": "http://test_site:8000",
+    "monitor": 1,
+    "server_script_enabled": true
+}

.github/helper/roulette.py

@@ -0,0 +1,146 @@
+import json
+import os
+import re
+import shlex
+import subprocess
+import sys
+import time
+import urllib.request
+from functools import lru_cache
+from urllib.error import HTTPError
+
+
+@lru_cache(maxsize=None)
+def fetch_pr_data(pr_number, repo, endpoint=""):
+	api_url = f"https://api.github.com/repos/{repo}/pulls/{pr_number}"
+
+	if endpoint:
+		api_url += f"/{endpoint}"
+
+	res = req(api_url)
+	return json.loads(res.read().decode("utf8"))
+
+
+def req(url):
+	"Simple resilient request call to handle rate limits."
+	headers = None
+	token = os.environ.get("GITHUB_TOKEN")
+	if token:
+		headers = {"authorization": f"Bearer {token}"}
+
+	retries = 0
+	while True:
+		try:
+			req = urllib.request.Request(url, headers=headers)
+			return urllib.request.urlopen(req)
+		except HTTPError as exc:
+			if exc.code == 403 and retries < 5:
+				retries += 1
+				time.sleep(retries)
+				continue
+			raise
+
+
+def get_files_list(pr_number, repo="frappe/frappe"):
+	return [change["filename"] for change in fetch_pr_data(pr_number, repo, "files")]
+
+
+def get_output(command, shell=True):
+	print(command)
+	command = shlex.split(command)
+	return subprocess.check_output(command, shell=shell, encoding="utf8").strip()
+
+
+def has_skip_ci_label(pr_number, repo="frappe/frappe"):
+	return has_label(pr_number, "Skip CI", repo)
+
+
+def has_run_server_tests_label(pr_number, repo="frappe/frappe"):
+	return has_label(pr_number, "Run Server Tests", repo)
+
+
+def has_run_ui_tests_label(pr_number, repo="frappe/frappe"):
+	return has_label(pr_number, "Run UI Tests", repo)
+
+
+def has_label(pr_number, label, repo="frappe/frappe"):
+	return any(
+		[
+			fetched_label["name"]
+			for fetched_label in fetch_pr_data(pr_number, repo)["labels"]
+			if fetched_label["name"] == label
+		]
+	)
+
+
+def is_py(file):
+	return file.endswith("py")
+
+
+def is_ci(file):
+	return ".github" in file
+
+
+def is_frontend_code(file):
+	return file.lower().endswith(
+		(".css", ".scss", ".less", ".sass", ".styl", ".js", ".ts", ".vue", ".html")
+	)
+
+
+def is_docs(file):
+	regex = re.compile(r"\.(md|png|jpg|jpeg|csv|svg)$|^.github|LICENSE")
+	return bool(regex.search(file))
+
+
+if __name__ == "__main__":
+	files_list = sys.argv[1:]
+	build_type = os.environ.get("TYPE")
+	pr_number = os.environ.get("PR_NUMBER")
+	repo = os.environ.get("REPO_NAME")
+
+	# this is a push build, run all builds
+	if not pr_number:
+		os.system('echo "build=strawberry" >> $GITHUB_OUTPUT')
+		sys.exit(0)
+
+	files_list = files_list or get_files_list(pr_number=pr_number, repo=repo)
+
+	if not files_list:
+		print("No files' changes detected. Build is shutting")
+		sys.exit(0)
+
+	ci_files_changed = any(f for f in files_list if is_ci(f))
+	only_docs_changed = len(list(filter(is_docs, files_list))) == len(files_list)
+	only_frontend_code_changed = len(list(filter(is_frontend_code, files_list))) == len(files_list)
+	updated_py_file_count = len(list(filter(is_py, files_list)))
+	only_py_changed = updated_py_file_count == len(files_list)
+
+	if has_skip_ci_label(pr_number, repo):
+		if build_type == "ui" and has_run_ui_tests_label(pr_number, repo):
+			print("Running UI tests only.")
+		elif build_type == "server" and has_run_server_tests_label(pr_number, repo):
+			print("Running server tests only.")
+		else:
+			print("Found `Skip CI` label on pr, stopping build process.")
+			sys.exit(0)
+
+	elif ci_files_changed:
+		print("CI related files were updated, running all build processes.")
+
+	elif only_docs_changed:
+		print("Only docs were updated, stopping build process.")
+		sys.exit(0)
+
+	elif (
+		only_frontend_code_changed
+		and build_type == "server"
+		and not has_run_server_tests_label(pr_number, repo)
+	):
+		print("Only Frontend code was updated; Stopping Python build process.")
+		sys.exit(0)
+
+	elif build_type == "ui" and only_py_changed and not has_run_ui_tests_label(pr_number, repo):
+		print("Only Python code was updated, stopping Cypress build process.")
+		sys.exit(0)
+
+	os.system('echo "build=strawberry" >> $GITHUB_OUTPUT')

.github/workflows/frappe.yaml

@@ -38,6 +38,14 @@ jobs:
         with:
           python-version: '3.10'
 
+      - name: Check for valid Python & Merge Conflicts
+        run: |
+          python -m compileall -q -f "${GITHUB_WORKSPACE}"
+          if grep -lr --exclude-dir=node_modules "^<<<<<<< " "${GITHUB_WORKSPACE}"
+              then echo "Found merge conflicts"
+              exit 1
+          fi
+
       - name: Setup Node
         uses: actions/setup-node@v4
         with:
@@ -48,14 +56,14 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/pyproject.toml', '**/setup.py', '**/setup.cfg') }}
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/pyproject.toml', '**/setup.py') }}
           restore-keys: |
             ${{ runner.os }}-pip-
             ${{ runner.os }}-
 
       - name: Get yarn cache directory path
         id: yarn-cache-dir-path
-        run: 'echo "::set-output name=dir::$(yarn cache dir)"'
+        run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
 
       - uses: actions/cache@v3
         id: yarn-cache
@@ -65,23 +73,10 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-yarn-
 
-      - name: Setup
-        run: |
-          pip install frappe-bench
-          bench init --skip-redis-config-generation --skip-assets --python "$(which python)" --frappe-branch version-15 ~/frappe-bench
-          mysql --host 127.0.0.1 --port 3306 -u root -proot -e "SET GLOBAL character_set_server = 'utf8mb4'"
-          mysql --host 127.0.0.1 --port 3306 -u root -proot -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'"
-
       - name: Install
-        working-directory: /home/runner/frappe-bench
         run: |
-          bench get-app cloud_storage $GITHUB_WORKSPACE --resolve-deps
-          bench setup requirements --dev
-          bench new-site --db-root-password root --admin-password admin test_site
-          bench --site test_site install-app cloud_storage
-          bench build
-        env:
-          CI: 'Yes'
+          bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh
+          bash ${GITHUB_WORKSPACE}/.github/helper/install.sh
 
       - name: Run Tests
         working-directory: /home/runner/frappe-bench