test: fix delete file test (#63)

* ci: add frappe black to CI (#57)

* ci: add frappe black to CI

* chore: black

* ci: fix v15 tests

* ci: run redis server

---------

Co-authored-by: Tyler Matteson <[email protected]>
Co-authored-by: Heather Kusmierz <[email protected]>
Co-authored-by: Rohan Bansal <[email protected]>

.backportrc.json

@@ -1,17 +1,11 @@
-{
-	"repoOwner": "agritheory",
-	"repoName": "cloud_storage",
-	"targetBranchChoices": [
-		"version-14",
-		"version-15"
-	],
-	"targetBranches": [
-		"version-14",
-		"version-15"
-	],
-	"autoMerge": true,
-	"autoMergeMethod": "squash",
-	"branchLabelMapping": {
-		"^auto-backport-to-(.+)$": "$1"
-	}
-}
+{
+	"repoOwner": "agritheory",
+	"repoName": "cloud_storage",
+	"targetBranchChoices": ["version-14", "version-15"],
+	"targetBranches": ["version-14", "version-15"],
+	"autoMerge": true,
+	"autoMergeMethod": "squash",
+	"branchLabelMapping": {
+		"^auto-backport-to-(.+)$": "$1"
+	}
+}

.github/helper/install.sh

@@ -10,7 +10,7 @@ sudo apt install libcups2-dev redis-server mariadb-client-10.6
 
 pip install frappe-bench
 
-bench init --skip-redis-config-generation --skip-assets --python "$(which python)" --frappe-branch version-15 frappe-bench
+bench init --skip-assets --python "$(which python)" --frappe-branch version-15 frappe-bench
 
 mkdir ~/frappe-bench/sites/test_site
 cp -r "${GITHUB_WORKSPACE}/.github/helper/site_config.json" ~/frappe-bench/sites/test_site/

.github/validate_customizations.py

@@ -0,0 +1,164 @@
+import json
+import pathlib
+import sys
+
+
+def scrub(txt: str) -> str:
+	return txt.replace(" ", "_").replace("-", "_").lower()
+
+
+def unscrub(txt: str) -> str:
+	return txt.replace("_", " ").replace("-", " ").title()
+
+
+def get_customized_doctypes():
+	apps_dir = pathlib.Path(__file__).resolve().parent.parent.parent
+	apps_order = pathlib.Path(__file__).resolve().parent.parent.parent.parent / "sites" / "apps.txt"
+	apps_order = apps_order.read_text().split("\n")
+	customized_doctypes = {}
+	for _app_dir in apps_order:
+		app_dir = (apps_dir / _app_dir).resolve()
+		if not app_dir.is_dir():
+			continue
+		modules = (app_dir / _app_dir / "modules.txt").read_text().split("\n")
+		for module in modules:
+			if not (app_dir / _app_dir / scrub(module) / "custom").exists():
+				continue
+			for custom_file in list((app_dir / _app_dir / scrub(module) / "custom").glob("**/*.json")):
+				if custom_file.stem in customized_doctypes:
+					customized_doctypes[custom_file.stem].append(custom_file.resolve())
+				else:
+					customized_doctypes[custom_file.stem] = [custom_file.resolve()]
+
+	return dict(sorted(customized_doctypes.items()))
+
+
+def validate_module(customized_doctypes, set_module=False):
+	exceptions = []
+	app_dir = pathlib.Path(__file__).resolve().parent.parent
+	this_app = app_dir.stem
+	for doctype, customize_files in customized_doctypes.items():
+		for customize_file in customize_files:
+			if not this_app in str(customize_file):
+				continue
+			module = customize_file.parent.parent.stem
+			file_contents = json.loads(customize_file.read_text())
+			if file_contents.get("custom_fields"):
+				for custom_field in file_contents.get("custom_fields"):
+					if set_module:
+						custom_field["module"] = unscrub(module)
+						continue
+					if not custom_field.get("module"):
+						exceptions.append(
+							f"Custom Field for {custom_field.get('dt')} in {this_app} '{custom_field.get('fieldname')}' does not have a module key"
+						)
+						continue
+					elif custom_field.get("module") != unscrub(module):
+						exceptions.append(
+							f"Custom Field for {custom_field.get('dt')} in {this_app} '{custom_field.get('fieldname')}' has module key ({custom_field.get('module')}) associated with another app"
+						)
+						continue
+			if file_contents.get("property_setters"):
+				for ps in file_contents.get("property_setters"):
+					if set_module:
+						ps["module"] = unscrub(module)
+						continue
+					if not ps.get("module"):
+						exceptions.append(
+							f"Property Setter for {ps.get('doc_type')} in {this_app} '{ps.get('property')}' on {ps.get('field_name')} does not have a module key"
+						)
+						continue
+					elif ps.get("module") != unscrub(module):
+						exceptions.append(
+							f"Property Setter for {ps.get('doc_type')} in {this_app} '{ps.get('property')}' on {ps.get('field_name')} has module key ({ps.get('module')}) associated with another app"
+						)
+						continue
+			if set_module:
+				with customize_file.open("w", encoding="UTF-8") as target:
+					json.dump(file_contents, target, sort_keys=True, indent=2)
+
+	return exceptions
+
+
+def validate_no_custom_perms(customized_doctypes):
+	exceptions = []
+	this_app = pathlib.Path(__file__).resolve().parent.parent.stem
+	for doctype, customize_files in customized_doctypes.items():
+		for customize_file in customize_files:
+			if not this_app in str(customize_file):
+				continue
+			file_contents = json.loads(customize_file.read_text())
+			if file_contents.get("custom_perms"):
+				exceptions.append(f"Customization for {doctype} in {this_app} contains custom permissions")
+	return exceptions
+
+
+def validate_duplicate_customizations(customized_doctypes):
+	exceptions = []
+	common_fields = {}
+	common_property_setters = {}
+	app_dir = pathlib.Path(__file__).resolve().parent.parent
+	this_app = app_dir.stem
+	for doctype, customize_files in customized_doctypes.items():
+		if len(customize_files) == 1:
+			continue
+		common_fields[doctype] = {}
+		common_property_setters[doctype] = {}
+		for customize_file in customize_files:
+			module = customize_file.parent.parent.stem
+			app = customize_file.parent.parent.parent.parent.stem
+			file_contents = json.loads(customize_file.read_text())
+			if file_contents.get("custom_fields"):
+				fields = [cf.get("fieldname") for cf in file_contents.get("custom_fields")]
+				common_fields[doctype][module] = fields
+			if file_contents.get("property_setters"):
+				ps = [ps.get("name") for ps in file_contents.get("property_setters")]
+				common_property_setters[doctype][module] = ps
+
+	for doctype, module_and_fields in common_fields.items():
+		if this_app not in module_and_fields.keys():
+			continue
+		this_modules_fields = module_and_fields.pop(this_app)
+		for module, fields in module_and_fields.items():
+			for field in fields:
+				if field in this_modules_fields:
+					exceptions.append(
+						f"Custom Field for {unscrub(doctype)} in {this_app} '{field}' also appears in customizations for {module}"
+					)
+
+	for doctype, module_and_ps in common_property_setters.items():
+		if this_app not in module_and_ps.keys():
+			continue
+		this_modules_ps = module_and_ps.pop(this_app)
+		for module, ps in module_and_ps.items():
+			for p in ps:
+				if p in this_modules_ps:
+					exceptions.append(
+						f"Property Setter for {unscrub(doctype)} in {this_app} on '{p}' also appears in customizations for {module}"
+					)
+
+	return exceptions
+
+
+def validate_customizations(set_module):
+	customized_doctypes = get_customized_doctypes()
+	exceptions = validate_no_custom_perms(customized_doctypes)
+	exceptions += validate_module(customized_doctypes, set_module)
+	exceptions += validate_duplicate_customizations(customized_doctypes)
+
+	return exceptions
+
+
+if __name__ == "__main__":
+	exceptions = []
+	set_module = False
+	for arg in sys.argv:
+		if arg == "--set-module":
+			set_module = True
+		exceptions.append(validate_customizations(set_module))
+
+	if exceptions:
+		for exception in exceptions:
+			[print(e) for e in exception]  # TODO: colorize
+
+	sys.exit(1) if all(exceptions) else sys.exit(0)

.github/workflows/lint.yaml

@@ -3,8 +3,12 @@ name: Linters
 on:
   push:
     branches:
+      - version-14
       - version-15
   pull_request:
+    branches:
+      - version-14
+      - version-15
 
 env:
   GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -25,8 +29,8 @@ jobs:
         with:
           python-version: '3.10'
 
-      - name: Install mypy
-        run: pip install mypy
+      - name: Install mypy and types
+        run: python -m pip install mypy types-python-dateutil types-pytz --no-input
 
       - name: Run mypy
         uses: sasanquaneuf/mypy-github-action@releases/v1
@@ -35,6 +39,27 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
+  black:
+    needs: [ py_json_merge ]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.head_ref }}
+          fetch-depth: 2
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Install Black (Frappe)
+        run: pip install git+https://github.com/frappe/black.git
+
+      - name: Run Black (Frappe)
+        run: black --check .
+
   prettier:
     needs: [ py_json_merge ]
     runs-on: ubuntu-latest
@@ -46,9 +71,17 @@ jobs:
           fetch-depth: 2
 
       - name: Prettify code
-        uses: creyD/[email protected]
+        uses: rutajdash/[email protected]
         with:
-          commit_message: "style: prettify code"
+          config_path: ./.prettierrc.js
+          ignore_path: ./.prettierignore
+
+      - name: Prettier Output
+        if: ${{ failure() }}
+        shell: bash
+        run: |
+          echo "The following files are not formatted:"
+          echo "${{steps.prettier-run.outputs.prettier_output}}" >> $GITHUB_OUTPUT
 
   json_diff:
     needs: [ py_json_merge ]
@@ -62,7 +95,7 @@ jobs:
 
       - name: Find JSON changes
         id: changed-json
-        uses: tj-actions/changed-files@v37
+        uses: tj-actions/changed-files@v43
         with:
           files: |
             **/*.json
@@ -106,14 +139,6 @@ jobs:
             echo "D,${file}" >> base/mrd.txt
           done
 
-      - name: Setup requirements and script
-        run: |
-          pip install rich
-          pip install json_source_map
-          git clone --depth 1 https://gist.github.com/3eea518743067f1b971114f1a2016f69 fsjd
-
-      - name: Diff table
-        run: python3 fsjd/frappe_schema_json_diff.py base/mrd.txt head/acmr.txt 1
 
   py_json_merge:
     runs-on: ubuntu-latest
@@ -124,7 +149,7 @@ jobs:
         run: git clone --depth 1 https://gist.github.com/f1bf2c11f78331b2417189c385022c28.git validate_json
 
       - name: Validate JSON
-        run: python3 validate_json/validate_json.py ./
+        run: python3 validate_json/validate_json.py ./cloud_storage/cloud_storage/
 
       - name: Compile
         run: python3 -m compileall -q ./

.github/workflows/frappe.yaml → .github/workflows/pytest.yaml

@@ -50,15 +50,16 @@ jobs:
         with:
           node-version: 18
           check-latest: true
+          cache: 'yarn'
 
       - name: Add to Hosts
         run: echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts
 
       - name: Cache pip
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/pyproject.toml', '**/setup.py') }}
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/pyproject.toml') }}
           restore-keys: |
             ${{ runner.os }}-pip-
             ${{ runner.os }}-
@@ -67,7 +68,7 @@ jobs:
         id: yarn-cache-dir-path
         run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
 
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         id: yarn-cache
         with:
           path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
@@ -82,9 +83,7 @@ jobs:
       - name: Run Tests
         working-directory: /home/runner/frappe-bench
         run: |
-          bench --site test_site set-config allow_tests true
-          bench --site test_site run-tests --app cloud_storage
           source env/bin/activate
-          pytest ./apps/cloud_storage/cloud_storage/tests/test_file_association.py --disable-warnings -s
+          pytest ./apps/cloud_storage/cloud_storage/tests/ -s
         env:
           TYPE: server

.github/workflows/release.yaml

@@ -3,6 +3,7 @@ name: Release
 on:
   push:
     branches:
+      - version-14
       - version-15
 
 jobs: