diff --git a/.backportrc.json b/.backportrc.json
index 6feb957..e48e270 100644
--- a/.backportrc.json
+++ b/.backportrc.json
@@ -1,17 +1,11 @@
-{
- "repoOwner": "agritheory",
- "repoName": "cloud_storage",
- "targetBranchChoices": [
- "version-14",
- "version-15"
- ],
- "targetBranches": [
- "version-14",
- "version-15"
- ],
- "autoMerge": true,
- "autoMergeMethod": "squash",
- "branchLabelMapping": {
- "^auto-backport-to-(.+)$": "$1"
- }
-}
\ No newline at end of file
+{
+ "repoOwner": "agritheory",
+ "repoName": "cloud_storage",
+ "targetBranchChoices": ["version-14", "version-15"],
+ "targetBranches": ["version-14", "version-15"],
+ "autoMerge": true,
+ "autoMergeMethod": "squash",
+ "branchLabelMapping": {
+ "^auto-backport-to-(.+)$": "$1"
+ }
+}
diff --git a/.github/helper/install.sh b/.github/helper/install.sh
index 30d0d12..8990865 100644
--- a/.github/helper/install.sh
+++ b/.github/helper/install.sh
@@ -10,7 +10,7 @@ sudo apt install libcups2-dev redis-server mariadb-client-10.6
pip install frappe-bench
-bench init --skip-redis-config-generation --skip-assets --python "$(which python)" --frappe-branch version-15 frappe-bench
+bench init --skip-assets --python "$(which python)" --frappe-branch version-15 frappe-bench
mkdir ~/frappe-bench/sites/test_site
cp -r "${GITHUB_WORKSPACE}/.github/helper/site_config.json" ~/frappe-bench/sites/test_site/
diff --git a/.github/validate_customizations.py b/.github/validate_customizations.py
new file mode 100644
index 0000000..f942698
--- /dev/null
+++ b/.github/validate_customizations.py
@@ -0,0 +1,164 @@
+import json
+import pathlib
+import sys
+
+
+def scrub(txt: str) -> str:
+ return txt.replace(" ", "_").replace("-", "_").lower()
+
+
+def unscrub(txt: str) -> str:
+ return txt.replace("_", " ").replace("-", " ").title()
+
+
+def get_customized_doctypes():
+ apps_dir = pathlib.Path(__file__).resolve().parent.parent.parent
+ apps_order = pathlib.Path(__file__).resolve().parent.parent.parent.parent / "sites" / "apps.txt"
+ apps_order = apps_order.read_text().split("\n")
+ customized_doctypes = {}
+ for _app_dir in apps_order:
+ app_dir = (apps_dir / _app_dir).resolve()
+ if not app_dir.is_dir():
+ continue
+ modules = (app_dir / _app_dir / "modules.txt").read_text().split("\n")
+ for module in modules:
+ if not (app_dir / _app_dir / scrub(module) / "custom").exists():
+ continue
+ for custom_file in list((app_dir / _app_dir / scrub(module) / "custom").glob("**/*.json")):
+ if custom_file.stem in customized_doctypes:
+ customized_doctypes[custom_file.stem].append(custom_file.resolve())
+ else:
+ customized_doctypes[custom_file.stem] = [custom_file.resolve()]
+
+ return dict(sorted(customized_doctypes.items()))
+
+
+def validate_module(customized_doctypes, set_module=False):
+ exceptions = []
+ app_dir = pathlib.Path(__file__).resolve().parent.parent
+ this_app = app_dir.stem
+ for doctype, customize_files in customized_doctypes.items():
+ for customize_file in customize_files:
+ if not this_app in str(customize_file):
+ continue
+ module = customize_file.parent.parent.stem
+ file_contents = json.loads(customize_file.read_text())
+ if file_contents.get("custom_fields"):
+ for custom_field in file_contents.get("custom_fields"):
+ if set_module:
+ custom_field["module"] = unscrub(module)
+ continue
+ if not custom_field.get("module"):
+ exceptions.append(
+ f"Custom Field for {custom_field.get('dt')} in {this_app} '{custom_field.get('fieldname')}' does not have a module key"
+ )
+ continue
+ elif custom_field.get("module") != unscrub(module):
+ exceptions.append(
+ f"Custom Field for {custom_field.get('dt')} in {this_app} '{custom_field.get('fieldname')}' has module key ({custom_field.get('module')}) associated with another app"
+ )
+ continue
+ if file_contents.get("property_setters"):
+ for ps in file_contents.get("property_setters"):
+ if set_module:
+ ps["module"] = unscrub(module)
+ continue
+ if not ps.get("module"):
+ exceptions.append(
+ f"Property Setter for {ps.get('doc_type')} in {this_app} '{ps.get('property')}' on {ps.get('field_name')} does not have a module key"
+ )
+ continue
+ elif ps.get("module") != unscrub(module):
+ exceptions.append(
+ f"Property Setter for {ps.get('doc_type')} in {this_app} '{ps.get('property')}' on {ps.get('field_name')} has module key ({ps.get('module')}) associated with another app"
+ )
+ continue
+ if set_module:
+ with customize_file.open("w", encoding="UTF-8") as target:
+ json.dump(file_contents, target, sort_keys=True, indent=2)
+
+ return exceptions
+
+
+def validate_no_custom_perms(customized_doctypes):
+ exceptions = []
+ this_app = pathlib.Path(__file__).resolve().parent.parent.stem
+ for doctype, customize_files in customized_doctypes.items():
+ for customize_file in customize_files:
+ if not this_app in str(customize_file):
+ continue
+ file_contents = json.loads(customize_file.read_text())
+ if file_contents.get("custom_perms"):
+ exceptions.append(f"Customization for {doctype} in {this_app} contains custom permissions")
+ return exceptions
+
+
+def validate_duplicate_customizations(customized_doctypes):
+ exceptions = []
+ common_fields = {}
+ common_property_setters = {}
+ app_dir = pathlib.Path(__file__).resolve().parent.parent
+ this_app = app_dir.stem
+ for doctype, customize_files in customized_doctypes.items():
+ if len(customize_files) == 1:
+ continue
+ common_fields[doctype] = {}
+ common_property_setters[doctype] = {}
+ for customize_file in customize_files:
+ module = customize_file.parent.parent.stem
+ app = customize_file.parent.parent.parent.parent.stem
+ file_contents = json.loads(customize_file.read_text())
+ if file_contents.get("custom_fields"):
+ fields = [cf.get("fieldname") for cf in file_contents.get("custom_fields")]
+ common_fields[doctype][module] = fields
+ if file_contents.get("property_setters"):
+ ps = [ps.get("name") for ps in file_contents.get("property_setters")]
+ common_property_setters[doctype][module] = ps
+
+ for doctype, module_and_fields in common_fields.items():
+ if this_app not in module_and_fields.keys():
+ continue
+ this_modules_fields = module_and_fields.pop(this_app)
+ for module, fields in module_and_fields.items():
+ for field in fields:
+ if field in this_modules_fields:
+ exceptions.append(
+ f"Custom Field for {unscrub(doctype)} in {this_app} '{field}' also appears in customizations for {module}"
+ )
+
+ for doctype, module_and_ps in common_property_setters.items():
+ if this_app not in module_and_ps.keys():
+ continue
+ this_modules_ps = module_and_ps.pop(this_app)
+ for module, ps in module_and_ps.items():
+ for p in ps:
+ if p in this_modules_ps:
+ exceptions.append(
+ f"Property Setter for {unscrub(doctype)} in {this_app} on '{p}' also appears in customizations for {module}"
+ )
+
+ return exceptions
+
+
+def validate_customizations(set_module):
+ customized_doctypes = get_customized_doctypes()
+ exceptions = validate_no_custom_perms(customized_doctypes)
+ exceptions += validate_module(customized_doctypes, set_module)
+ exceptions += validate_duplicate_customizations(customized_doctypes)
+
+ return exceptions
+
+
+if __name__ == "__main__":
+ exceptions = []
+ set_module = False
+ for arg in sys.argv:
+ if arg == "--set-module":
+ set_module = True
+ exceptions.append(validate_customizations(set_module))
+
+ if exceptions:
+ for exception in exceptions:
+ [print(e) for e in exception] # TODO: colorize
+
+ sys.exit(1) if all(exceptions) else sys.exit(0)
diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml
index c3d22bf..a02a968 100644
--- a/.github/workflows/lint.yaml
+++ b/.github/workflows/lint.yaml
@@ -3,8 +3,12 @@ name: Linters
on:
push:
branches:
+ - version-14
- version-15
pull_request:
+ branches:
+ - version-14
+ - version-15
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -25,8 +29,8 @@ jobs:
with:
python-version: '3.10'
- - name: Install mypy
- run: pip install mypy
+ - name: Install mypy and types
+ run: python -m pip install mypy types-python-dateutil types-pytz --no-input
- name: Run mypy
uses: sasanquaneuf/mypy-github-action@releases/v1
@@ -35,6 +39,27 @@ jobs:
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ black:
+ needs: [ py_json_merge ]
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v4
+ with:
+ ref: ${{ github.head_ref }}
+ fetch-depth: 2
+
+ - name: Setup Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: '3.10'
+
+ - name: Install Black (Frappe)
+ run: pip install git+https://github.com/frappe/black.git
+
+ - name: Run Black (Frappe)
+ run: black --check .
+
prettier:
needs: [ py_json_merge ]
runs-on: ubuntu-latest
@@ -46,9 +71,17 @@ jobs:
fetch-depth: 2
- name: Prettify code
- uses: creyD/prettier_action@v4.3
+ uses: rutajdash/prettier-cli-action@v1.0.0
with:
- commit_message: "style: prettify code"
+ config_path: ./.prettierrc.js
+ ignore_path: ./.prettierignore
+
+ - name: Prettier Output
+ if: ${{ failure() }}
+ shell: bash
+ run: |
+ echo "The following files are not formatted:"
+ echo "${{steps.prettier-run.outputs.prettier_output}}" >> $GITHUB_OUTPUT
json_diff:
needs: [ py_json_merge ]
@@ -62,7 +95,7 @@ jobs:
- name: Find JSON changes
id: changed-json
- uses: tj-actions/changed-files@v37
+ uses: tj-actions/changed-files@v43
with:
files: |
**/*.json
@@ -106,14 +139,6 @@ jobs:
echo "D,${file}" >> base/mrd.txt
done
- - name: Setup requirements and script
- run: |
- pip install rich
- pip install json_source_map
- git clone --depth 1 https://gist.github.com/3eea518743067f1b971114f1a2016f69 fsjd
-
- - name: Diff table
- run: python3 fsjd/frappe_schema_json_diff.py base/mrd.txt head/acmr.txt 1
py_json_merge:
runs-on: ubuntu-latest
@@ -124,7 +149,7 @@ jobs:
run: git clone --depth 1 https://gist.github.com/f1bf2c11f78331b2417189c385022c28.git validate_json
- name: Validate JSON
- run: python3 validate_json/validate_json.py ./
+ run: python3 validate_json/validate_json.py ./cloud_storage/cloud_storage/
- name: Compile
run: python3 -m compileall -q ./
diff --git a/.github/workflows/frappe.yaml b/.github/workflows/pytest.yaml
similarity index 83%
rename from .github/workflows/frappe.yaml
rename to .github/workflows/pytest.yaml
index fbb7997..683b29d 100644
--- a/.github/workflows/frappe.yaml
+++ b/.github/workflows/pytest.yaml
@@ -50,15 +50,16 @@ jobs:
with:
node-version: 18
check-latest: true
+ cache: 'yarn'
- name: Add to Hosts
run: echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts
- name: Cache pip
- uses: actions/cache@v3
+ uses: actions/cache@v4
with:
path: ~/.cache/pip
- key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/pyproject.toml', '**/setup.py') }}
+ key: ${{ runner.os }}-pip-${{ hashFiles('**/pyproject.toml') }}
restore-keys: |
${{ runner.os }}-pip-
${{ runner.os }}-
@@ -67,7 +68,7 @@ jobs:
id: yarn-cache-dir-path
run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
- - uses: actions/cache@v3
+ - uses: actions/cache@v4
id: yarn-cache
with:
path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
@@ -82,9 +83,7 @@ jobs:
- name: Run Tests
working-directory: /home/runner/frappe-bench
run: |
- bench --site test_site set-config allow_tests true
- bench --site test_site run-tests --app cloud_storage
source env/bin/activate
- pytest ./apps/cloud_storage/cloud_storage/tests/test_file_association.py --disable-warnings -s
+ pytest ./apps/cloud_storage/cloud_storage/tests/ -s
env:
TYPE: server
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index b9c3392..b4d9f60 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -3,6 +3,7 @@ name: Release
on:
push:
branches:
+ - version-14
- version-15
jobs:
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 442e8e4..1fbe0ae 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -7,8 +7,8 @@ repos:
rev: v4.3.0
hooks:
- id: trailing-whitespace
- files: "cloud_storage.*"
- exclude: ".*json$|.*txt$|.*csv|.*md|.*svg"
+ files: 'cloud_storage.*'
+ exclude: '.*json$|.*txt$|.*csv|.*md|.*svg'
- id: check-yaml
- id: no-commit-to-branch
args: ['--branch', 'develop']
@@ -37,14 +37,28 @@ repos:
types_or: [javascript, vue, scss]
# Ignore any files that might contain jinja / bundles
exclude: |
- (?x)^(
- .*node_modules.*|
- cloud_storage/public/dist/.*|
- cloud_storage/public/js/lib/.*|
- cloud_storage/templates/includes/.*|
- cloud_storage/www/website_script.js
- )$
+ (?x)^(
+ .*node_modules.*|
+ cloud_storage/public/dist/.*|
+ cloud_storage/public/js/lib/.*|
+ cloud_storage/templates/includes/.*
+ )$
+ - repo: https://github.com/pre-commit/mirrors-eslint
+ rev: v8.44.0
+ hooks:
+ - id: eslint
+ types_or: [javascript]
+ args: ['--quiet']
+ # Ignore any files that might contain jinja / bundles
+ exclude: |
+ (?x)^(
+ .*node_modules.*|
+ cloud_storage/public/dist/.*|
+ cloud_storage/public/js/lib/.*|
+ cloud_storage/templates/includes/.*|
+ cloud_storage/www/website_script.js
+ )$
- repo: https://github.com/pre-commit/mirrors-eslint
rev: v8.44.0
@@ -71,9 +85,18 @@ repos:
rev: 6.0.0
hooks:
- id: flake8
- additional_dependencies: ['flake8-bugbear',]
+ additional_dependencies: ['flake8-bugbear']
+
+ - repo: local
+ hooks:
+ - id: validate_customizations
+ always_run: true
+ name: .github/validate_customizations.py
+ entry: python .github/validate_customizations.py
+ language: system
+ types: [python]
ci:
- autoupdate_schedule: weekly
- skip: []
- submodules: false
+ autoupdate_schedule: weekly
+ skip: []
+ submodules: false
diff --git a/.prettierrc.js b/.prettierrc.cjs
similarity index 100%
rename from .prettierrc.js
rename to .prettierrc.cjs
diff --git a/cloud_storage/cloud_storage/custom/file.json b/cloud_storage/cloud_storage/custom/file.json
index 11617aa..70a7500 100644
--- a/cloud_storage/cloud_storage/custom/file.json
+++ b/cloud_storage/cloud_storage/custom/file.json
@@ -1,468 +1,342 @@
{
- "custom_fields": [
- {
- "_assign": null,
- "_comments": null,
- "_liked_by": null,
- "_user_tags": null,
- "allow_in_quick_entry": 0,
- "allow_on_submit": 0,
- "bold": 0,
- "collapsible": 0,
- "collapsible_depends_on": null,
- "columns": 0,
- "creation": "2023-02-03 12:03:51.204553",
- "default": null,
- "depends_on": null,
- "description": null,
- "docstatus": 0,
- "dt": "File",
- "fetch_from": null,
- "fetch_if_empty": 0,
- "fieldname": "s3_key",
- "fieldtype": "Data",
- "hidden": 0,
- "hide_border": 0,
- "hide_days": 0,
- "hide_seconds": 0,
- "idx": 23,
- "ignore_user_permissions": 0,
- "ignore_xss_filter": 0,
- "in_global_search": 0,
- "in_list_view": 0,
- "in_preview": 0,
- "in_standard_filter": 0,
- "insert_after": "uploaded_to_google_drive",
- "is_system_generated": 0,
- "is_virtual": 0,
- "label": "S3 Key",
- "length": 0,
- "mandatory_depends_on": null,
- "modified": "2023-07-06 02:04:01.131489",
- "modified_by": "Administrator",
- "module": null,
- "name": "File-s3_key",
- "no_copy": 0,
- "non_negative": 0,
- "options": null,
- "owner": "Administrator",
- "permlevel": 0,
- "precision": "",
- "print_hide": 0,
- "print_hide_if_no_value": 0,
- "print_width": null,
- "read_only": 1,
- "read_only_depends_on": null,
- "report_hide": 0,
- "reqd": 0,
- "search_index": 0,
- "sort_options": 0,
- "translatable": 0,
- "unique": 0,
- "width": null
- },
- {
- "_assign": null,
- "_comments": null,
- "_liked_by": null,
- "_user_tags": null,
- "allow_in_quick_entry": 0,
- "allow_on_submit": 0,
- "bold": 0,
- "collapsible": 0,
- "collapsible_depends_on": null,
- "columns": 0,
- "creation": "2023-02-15 14:00:22.073075",
- "default": null,
- "depends_on": null,
- "description": null,
- "docstatus": 0,
- "dt": "File",
- "fetch_from": null,
- "fetch_if_empty": 0,
- "fieldname": "sharing_link",
- "fieldtype": "Data",
- "hidden": 0,
- "hide_border": 0,
- "hide_days": 0,
- "hide_seconds": 0,
- "idx": 24,
- "ignore_user_permissions": 0,
- "ignore_xss_filter": 0,
- "in_global_search": 0,
- "in_list_view": 0,
- "in_preview": 0,
- "in_standard_filter": 0,
- "insert_after": "s3_key",
- "is_system_generated": 0,
- "is_virtual": 0,
- "label": "Sharing Link",
- "length": 0,
- "mandatory_depends_on": null,
- "modified": "2023-07-06 02:04:01.272045",
- "modified_by": "Administrator",
- "module": null,
- "name": "File-sharing_link",
- "no_copy": 0,
- "non_negative": 0,
- "options": null,
- "owner": "Administrator",
- "permlevel": 0,
- "precision": "",
- "print_hide": 0,
- "print_hide_if_no_value": 0,
- "print_width": null,
- "read_only": 1,
- "read_only_depends_on": null,
- "report_hide": 0,
- "reqd": 0,
- "search_index": 0,
- "sort_options": 0,
- "translatable": 1,
- "unique": 0,
- "width": null
- },
- {
- "_assign": null,
- "_comments": null,
- "_liked_by": null,
- "_user_tags": null,
- "allow_in_quick_entry": 0,
- "allow_on_submit": 0,
- "bold": 0,
- "collapsible": 0,
- "collapsible_depends_on": null,
- "columns": 0,
- "creation": "2023-03-20 12:48:48.264679",
- "default": null,
- "depends_on": null,
- "description": null,
- "docstatus": 0,
- "dt": "File",
- "fetch_from": null,
- "fetch_if_empty": 0,
- "fieldname": "section_break_1lqhx",
- "fieldtype": "Section Break",
- "hidden": 0,
- "hide_border": 0,
- "hide_days": 0,
- "hide_seconds": 0,
- "idx": 25,
- "ignore_user_permissions": 0,
- "ignore_xss_filter": 0,
- "in_global_search": 0,
- "in_list_view": 0,
- "in_preview": 0,
- "in_standard_filter": 0,
- "insert_after": "sharing_link",
- "is_system_generated": 0,
- "is_virtual": 0,
- "label": null,
- "length": 0,
- "mandatory_depends_on": null,
- "modified": "2023-07-06 02:04:01.389708",
- "modified_by": "Administrator",
- "module": null,
- "name": "File-section_break_1lqhx",
- "no_copy": 0,
- "non_negative": 0,
- "options": null,
- "owner": "Administrator",
- "permlevel": 0,
- "precision": "",
- "print_hide": 0,
- "print_hide_if_no_value": 0,
- "print_width": null,
- "read_only": 0,
- "read_only_depends_on": null,
- "report_hide": 0,
- "reqd": 0,
- "search_index": 0,
- "sort_options": 0,
- "translatable": 0,
- "unique": 0,
- "width": null
- },
- {
- "_assign": null,
- "_comments": null,
- "_liked_by": null,
- "_user_tags": null,
- "allow_in_quick_entry": 0,
- "allow_on_submit": 0,
- "bold": 0,
- "collapsible": 0,
- "collapsible_depends_on": null,
- "columns": 0,
- "creation": "2023-03-20 12:48:48.360631",
- "default": null,
- "depends_on": null,
- "description": null,
- "docstatus": 0,
- "dt": "File",
- "fetch_from": null,
- "fetch_if_empty": 0,
- "fieldname": "file_association",
- "fieldtype": "Table",
- "hidden": 0,
- "hide_border": 0,
- "hide_days": 0,
- "hide_seconds": 0,
- "idx": 26,
- "ignore_user_permissions": 0,
- "ignore_xss_filter": 0,
- "in_global_search": 0,
- "in_list_view": 0,
- "in_preview": 0,
- "in_standard_filter": 0,
- "insert_after": "section_break_1lqhx",
- "is_system_generated": 0,
- "is_virtual": 0,
- "label": "",
- "length": 0,
- "mandatory_depends_on": null,
- "modified": "2023-07-06 02:04:01.499473",
- "modified_by": "Administrator",
- "module": null,
- "name": "File-file_association",
- "no_copy": 0,
- "non_negative": 0,
- "options": "File Association",
- "owner": "Administrator",
- "permlevel": 0,
- "precision": "",
- "print_hide": 0,
- "print_hide_if_no_value": 0,
- "print_width": null,
- "read_only": 0,
- "read_only_depends_on": null,
- "report_hide": 0,
- "reqd": 0,
- "search_index": 0,
- "sort_options": 0,
- "translatable": 0,
- "unique": 0,
- "width": null
- },
- {
- "_assign": null,
- "_comments": null,
- "_liked_by": null,
- "_user_tags": null,
- "allow_in_quick_entry": 0,
- "allow_on_submit": 0,
- "bold": 0,
- "collapsible": 0,
- "collapsible_depends_on": null,
- "columns": 0,
- "creation": "2023-07-06 05:39:34.115162",
- "default": null,
- "depends_on": null,
- "description": null,
- "docstatus": 0,
- "dt": "File",
- "fetch_from": null,
- "fetch_if_empty": 0,
- "fieldname": "section_break_1xyhd",
- "fieldtype": "Section Break",
- "hidden": 0,
- "hide_border": 0,
- "hide_days": 0,
- "hide_seconds": 0,
- "idx": 27,
- "ignore_user_permissions": 0,
- "ignore_xss_filter": 0,
- "in_global_search": 0,
- "in_list_view": 0,
- "in_preview": 0,
- "in_standard_filter": 0,
- "insert_after": "file_association",
- "is_system_generated": 0,
- "is_virtual": 0,
- "label": null,
- "length": 0,
- "mandatory_depends_on": null,
- "modified": "2023-07-06 05:39:34.115162",
- "modified_by": "Administrator",
- "module": null,
- "name": "File-section_break_1xyhd",
- "no_copy": 0,
- "non_negative": 0,
- "options": null,
- "owner": "Administrator",
- "permlevel": 0,
- "precision": "",
- "print_hide": 0,
- "print_hide_if_no_value": 0,
- "print_width": null,
- "read_only": 0,
- "read_only_depends_on": null,
- "report_hide": 0,
- "reqd": 0,
- "search_index": 0,
- "sort_options": 0,
- "translatable": 0,
- "unique": 0,
- "width": null
- },
- {
- "_assign": null,
- "_comments": null,
- "_liked_by": null,
- "_user_tags": null,
- "allow_in_quick_entry": 0,
- "allow_on_submit": 0,
- "bold": 0,
- "collapsible": 0,
- "collapsible_depends_on": null,
- "columns": 0,
- "creation": "2023-07-06 05:39:57.024056",
- "default": null,
- "depends_on": null,
- "description": null,
- "docstatus": 0,
- "dt": "File",
- "fetch_from": null,
- "fetch_if_empty": 0,
- "fieldname": "versions",
- "fieldtype": "Table",
- "hidden": 0,
- "hide_border": 0,
- "hide_days": 0,
- "hide_seconds": 0,
- "idx": 28,
- "ignore_user_permissions": 0,
- "ignore_xss_filter": 0,
- "in_global_search": 0,
- "in_list_view": 0,
- "in_preview": 0,
- "in_standard_filter": 0,
- "insert_after": "section_break_1xyhd",
- "is_system_generated": 0,
- "is_virtual": 0,
- "label": "",
- "length": 0,
- "mandatory_depends_on": null,
- "modified": "2023-07-06 05:39:57.024056",
- "modified_by": "Administrator",
- "module": null,
- "name": "File-versions",
- "no_copy": 0,
- "non_negative": 0,
- "options": "File Version",
- "owner": "Administrator",
- "permlevel": 0,
- "precision": "",
- "print_hide": 0,
- "print_hide_if_no_value": 0,
- "print_width": null,
- "read_only": 0,
- "read_only_depends_on": null,
- "report_hide": 0,
- "reqd": 0,
- "search_index": 0,
- "sort_options": 0,
- "translatable": 0,
- "unique": 0,
- "width": null
- }
- ],
- "custom_perms": [],
- "doctype": "File",
- "links": [],
- "property_setters": [
- {
- "_assign": null,
- "_comments": null,
- "_liked_by": null,
- "_user_tags": null,
- "creation": "2023-07-06 02:04:01.645547",
- "default_value": null,
- "doc_type": "File",
- "docstatus": 0,
- "doctype_or_field": "DocField",
- "field_name": "preview",
- "idx": 0,
- "is_system_generated": 0,
- "modified": "2023-07-06 02:04:01.645547",
- "modified_by": "Administrator",
- "module": null,
- "name": "File-preview-collapsible_depends_on",
- "owner": "Administrator",
- "property": "collapsible_depends_on",
- "property_type": "Data",
- "row_name": null,
- "value": "eval:true"
- },
- {
- "_assign": null,
- "_comments": null,
- "_liked_by": null,
- "_user_tags": null,
- "creation": "2023-07-06 02:04:01.636840",
- "default_value": null,
- "doc_type": "File",
- "docstatus": 0,
- "doctype_or_field": "DocField",
- "field_name": "preview",
- "idx": 0,
- "is_system_generated": 0,
- "modified": "2023-07-06 02:04:01.636840",
- "modified_by": "Administrator",
- "module": null,
- "name": "File-preview-collapsible",
- "owner": "Administrator",
- "property": "collapsible",
- "property_type": "Check",
- "row_name": null,
- "value": "1"
- },
- {
- "_assign": null,
- "_comments": null,
- "_liked_by": null,
- "_user_tags": null,
- "creation": "2023-07-06 02:04:01.627758",
- "default_value": null,
- "doc_type": "File",
- "docstatus": 0,
- "doctype_or_field": "DocField",
- "field_name": "section_break_8",
- "idx": 0,
- "is_system_generated": 0,
- "modified": "2023-07-06 02:04:01.627758",
- "modified_by": "Administrator",
- "module": null,
- "name": "File-section_break_8-permlevel",
- "owner": "Administrator",
- "property": "permlevel",
- "property_type": "Int",
- "row_name": null,
- "value": "1"
- },
- {
- "_assign": null,
- "_comments": null,
- "_liked_by": null,
- "_user_tags": null,
- "creation": "2023-07-06 02:04:01.616923",
- "default_value": null,
- "doc_type": "File",
- "docstatus": 0,
- "doctype_or_field": "DocField",
- "field_name": "section_break_8",
- "idx": 0,
- "is_system_generated": 0,
- "modified": "2023-07-06 02:04:01.616923",
- "modified_by": "Administrator",
- "module": null,
- "name": "File-section_break_8-label",
- "owner": "Administrator",
- "property": "label",
- "property_type": "Data",
- "row_name": null,
- "value": "Storage Details"
- }
- ],
- "sync_on_migrate": 1
-}
\ No newline at end of file
+ "custom_fields": [
+ {
+ "_assign": null,
+ "_comments": null,
+ "_liked_by": null,
+ "_user_tags": null,
+ "allow_in_quick_entry": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "collapsible_depends_on": null,
+ "columns": 0,
+ "creation": "2023-02-03 12:03:51.204553",
+ "default": null,
+ "depends_on": null,
+ "description": null,
+ "docstatus": 0,
+ "dt": "File",
+ "fetch_from": null,
+ "fetch_if_empty": 0,
+ "fieldname": "s3_key",
+ "fieldtype": "Data",
+ "hidden": 0,
+ "hide_border": 0,
+ "hide_days": 0,
+ "hide_seconds": 0,
+ "idx": 23,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 0,
+ "in_preview": 0,
+ "in_standard_filter": 0,
+ "insert_after": "uploaded_to_google_drive",
+ "is_system_generated": 0,
+ "is_virtual": 0,
+ "label": "S3 Key",
+ "length": 0,
+ "mandatory_depends_on": null,
+ "modified": "2023-04-28 00:05:56.057223",
+ "modified_by": "Administrator",
+ "module": "Cloud Storage",
+ "name": "File-s3_key",
+ "no_copy": 0,
+ "non_negative": 0,
+ "options": null,
+ "owner": "Administrator",
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "print_width": null,
+ "read_only": 1,
+ "read_only_depends_on": null,
+ "report_hide": 0,
+ "reqd": 0,
+ "search_index": 0,
+ "translatable": 0,
+ "unique": 0,
+ "width": null
+ },
+ {
+ "_assign": null,
+ "_comments": null,
+ "_liked_by": null,
+ "_user_tags": null,
+ "allow_in_quick_entry": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "collapsible_depends_on": null,
+ "columns": 0,
+ "creation": "2023-02-15 14:00:22.073075",
+ "default": null,
+ "depends_on": null,
+ "description": null,
+ "docstatus": 0,
+ "dt": "File",
+ "fetch_from": null,
+ "fetch_if_empty": 0,
+ "fieldname": "sharing_link",
+ "fieldtype": "Data",
+ "hidden": 0,
+ "hide_border": 0,
+ "hide_days": 0,
+ "hide_seconds": 0,
+ "idx": 24,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 0,
+ "in_preview": 0,
+ "in_standard_filter": 0,
+ "insert_after": "s3_key",
+ "is_system_generated": 0,
+ "is_virtual": 0,
+ "label": "Sharing Link",
+ "length": 0,
+ "mandatory_depends_on": null,
+ "modified": "2023-04-28 00:05:56.148197",
+ "modified_by": "Administrator",
+ "module": "Cloud Storage",
+ "name": "File-sharing_link",
+ "no_copy": 0,
+ "non_negative": 0,
+ "options": null,
+ "owner": "Administrator",
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "print_width": null,
+ "read_only": 1,
+ "read_only_depends_on": null,
+ "report_hide": 0,
+ "reqd": 0,
+ "search_index": 0,
+ "translatable": 1,
+ "unique": 0,
+ "width": null
+ },
+ {
+ "_assign": null,
+ "_comments": null,
+ "_liked_by": null,
+ "_user_tags": null,
+ "allow_in_quick_entry": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "collapsible_depends_on": null,
+ "columns": 0,
+ "creation": "2023-03-20 12:48:48.264679",
+ "default": null,
+ "depends_on": null,
+ "description": null,
+ "docstatus": 0,
+ "dt": "File",
+ "fetch_from": null,
+ "fetch_if_empty": 0,
+ "fieldname": "section_break_1lqhx",
+ "fieldtype": "Section Break",
+ "hidden": 0,
+ "hide_border": 0,
+ "hide_days": 0,
+ "hide_seconds": 0,
+ "idx": 25,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 0,
+ "in_preview": 0,
+ "in_standard_filter": 0,
+ "insert_after": "sharing_link",
+ "is_system_generated": 0,
+ "is_virtual": 0,
+ "label": null,
+ "length": 0,
+ "mandatory_depends_on": null,
+ "modified": "2023-04-28 00:05:56.255829",
+ "modified_by": "Administrator",
+ "module": "Cloud Storage",
+ "name": "File-section_break_1lqhx",
+ "no_copy": 0,
+ "non_negative": 0,
+ "options": null,
+ "owner": "Administrator",
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "print_width": null,
+ "read_only": 0,
+ "read_only_depends_on": null,
+ "report_hide": 0,
+ "reqd": 0,
+ "search_index": 0,
+ "translatable": 0,
+ "unique": 0,
+ "width": null
+ },
+ {
+ "_assign": null,
+ "_comments": null,
+ "_liked_by": null,
+ "_user_tags": null,
+ "allow_in_quick_entry": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "collapsible_depends_on": null,
+ "columns": 0,
+ "creation": "2023-03-20 12:48:48.360631",
+ "default": null,
+ "depends_on": null,
+ "description": null,
+ "docstatus": 0,
+ "dt": "File",
+ "fetch_from": null,
+ "fetch_if_empty": 0,
+ "fieldname": "file_association",
+ "fieldtype": "Table",
+ "hidden": 0,
+ "hide_border": 0,
+ "hide_days": 0,
+ "hide_seconds": 0,
+ "idx": 26,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 0,
+ "in_preview": 0,
+ "in_standard_filter": 0,
+ "insert_after": "section_break_1lqhx",
+ "is_system_generated": 0,
+ "is_virtual": 0,
+ "label": "",
+ "length": 0,
+ "mandatory_depends_on": null,
+ "modified": "2023-04-28 00:05:56.364321",
+ "modified_by": "Administrator",
+ "module": "Cloud Storage",
+ "name": "File-file_association",
+ "no_copy": 0,
+ "non_negative": 0,
+ "options": "File Association",
+ "owner": "Administrator",
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "print_width": null,
+ "read_only": 0,
+ "read_only_depends_on": null,
+ "report_hide": 0,
+ "reqd": 0,
+ "search_index": 0,
+ "translatable": 0,
+ "unique": 0,
+ "width": null
+ }
+ ],
+ "custom_perms": [],
+ "doctype": "File",
+ "links": [],
+ "property_setters": [
+ {
+ "_assign": null,
+ "_comments": null,
+ "_liked_by": null,
+ "_user_tags": null,
+ "creation": "2023-04-28 00:05:56.507961",
+ "default_value": null,
+ "doc_type": "File",
+ "docstatus": 0,
+ "doctype_or_field": "DocField",
+ "field_name": "section_break_8",
+ "idx": 0,
+ "is_system_generated": 0,
+ "modified": "2023-04-28 00:05:56.507961",
+ "modified_by": "Administrator",
+ "module": "Cloud Storage",
+ "name": "File-section_break_8-label",
+ "owner": "Administrator",
+ "property": "label",
+ "property_type": "Data",
+ "row_name": null,
+ "value": "Storage Details"
+ },
+ {
+ "_assign": null,
+ "_comments": null,
+ "_liked_by": null,
+ "_user_tags": null,
+ "creation": "2023-04-28 00:05:56.499487",
+ "default_value": null,
+ "doc_type": "File",
+ "docstatus": 0,
+ "doctype_or_field": "DocField",
+ "field_name": "section_break_8",
+ "idx": 0,
+ "is_system_generated": 0,
+ "modified": "2023-04-28 00:05:56.499487",
+ "modified_by": "Administrator",
+ "module": "Cloud Storage",
+ "name": "File-section_break_8-permlevel",
+ "owner": "Administrator",
+ "property": "permlevel",
+ "property_type": "Int",
+ "row_name": null,
+ "value": "1"
+ },
+ {
+ "_assign": null,
+ "_comments": null,
+ "_liked_by": null,
+ "_user_tags": null,
+ "creation": "2023-04-28 00:05:56.491129",
+ "default_value": null,
+ "doc_type": "File",
+ "docstatus": 0,
+ "doctype_or_field": "DocField",
+ "field_name": "preview",
+ "idx": 0,
+ "is_system_generated": 0,
+ "modified": "2023-04-28 00:05:56.491129",
+ "modified_by": "Administrator",
+ "module": "Cloud Storage",
+ "name": "File-preview-collapsible",
+ "owner": "Administrator",
+ "property": "collapsible",
+ "property_type": "Check",
+ "row_name": null,
+ "value": "1"
+ },
+ {
+ "_assign": null,
+ "_comments": null,
+ "_liked_by": null,
+ "_user_tags": null,
+ "creation": "2023-04-28 00:05:56.481095",
+ "default_value": null,
+ "doc_type": "File",
+ "docstatus": 0,
+ "doctype_or_field": "DocField",
+ "field_name": "preview",
+ "idx": 0,
+ "is_system_generated": 0,
+ "modified": "2023-04-28 00:05:56.481095",
+ "modified_by": "Administrator",
+ "module": "Cloud Storage",
+ "name": "File-preview-collapsible_depends_on",
+ "owner": "Administrator",
+ "property": "collapsible_depends_on",
+ "property_type": "Data",
+ "row_name": null,
+ "value": "eval:true"
+ }
+ ],
+ "sync_on_migrate": 1
+}
diff --git a/cloud_storage/cloud_storage/doctype/file_association/file_association.json b/cloud_storage/cloud_storage/doctype/file_association/file_association.json
index e1fc88c..bac4e0e 100644
--- a/cloud_storage/cloud_storage/doctype/file_association/file_association.json
+++ b/cloud_storage/cloud_storage/doctype/file_association/file_association.json
@@ -1,59 +1,54 @@
{
- "actions": [],
- "creation": "2023-03-20 12:48:05.086487",
- "default_view": "List",
- "doctype": "DocType",
- "editable_grid": 1,
- "engine": "InnoDB",
- "field_order": [
- "link_doctype",
- "link_name",
- "user",
- "timestamp"
- ],
- "fields": [
- {
- "fieldname": "link_doctype",
- "fieldtype": "Link",
- "in_list_view": 1,
- "label": "Document Type",
- "options": "DocType",
- "reqd": 1
- },
- {
- "fieldname": "link_name",
- "fieldtype": "Dynamic Link",
- "in_list_view": 1,
- "label": "Document Name",
- "options": "link_doctype",
- "reqd": 1
- },
- {
- "fieldname": "timestamp",
- "fieldtype": "Datetime",
- "in_list_view": 1,
- "label": "Timestamp",
- "read_only": 1
- },
- {
- "fieldname": "user",
- "fieldtype": "Link",
- "in_list_view": 1,
- "label": "User",
- "options": "User"
- }
- ],
- "istable": 1,
- "links": [],
- "modified": "2023-09-27 23:42:57.356690",
- "modified_by": "Administrator",
- "module": "Cloud Storage",
- "name": "File Association",
- "owner": "Administrator",
- "permissions": [],
- "quick_entry": 1,
- "sort_field": "modified",
- "sort_order": "DESC",
- "states": [],
- "track_changes": 1
-}
\ No newline at end of file
+ "actions": [],
+ "creation": "2023-03-20 12:48:05.086487",
+ "default_view": "List",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": ["link_doctype", "link_name", "user", "timestamp"],
+ "fields": [
+ {
+ "fieldname": "link_doctype",
+ "fieldtype": "Link",
+ "in_list_view": 1,
+ "label": "Document Type",
+ "options": "DocType",
+ "reqd": 1
+ },
+ {
+ "fieldname": "link_name",
+ "fieldtype": "Dynamic Link",
+ "in_list_view": 1,
+ "label": "Document Name",
+ "options": "link_doctype",
+ "reqd": 1
+ },
+ {
+ "fieldname": "timestamp",
+ "fieldtype": "Datetime",
+ "in_list_view": 1,
+ "label": "Timestamp",
+ "read_only": 1
+ },
+ {
+ "fieldname": "user",
+ "fieldtype": "Link",
+ "in_list_view": 1,
+ "label": "User",
+ "options": "User"
+ }
+ ],
+ "istable": 1,
+ "links": [],
+ "modified": "2023-09-27 23:42:57.356690",
+ "modified_by": "Administrator",
+ "module": "Cloud Storage",
+ "name": "File Association",
+ "owner": "Administrator",
+ "permissions": [],
+ "quick_entry": 1,
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "states": [],
+ "track_changes": 1
+}
diff --git a/cloud_storage/cloud_storage/doctype/file_version/file_version.json b/cloud_storage/cloud_storage/doctype/file_version/file_version.json
index 0b57e6d..fdc6092 100644
--- a/cloud_storage/cloud_storage/doctype/file_version/file_version.json
+++ b/cloud_storage/cloud_storage/doctype/file_version/file_version.json
@@ -1,46 +1,42 @@
{
- "actions": [],
- "creation": "2023-07-06 05:38:35.891393",
- "doctype": "DocType",
- "editable_grid": 1,
- "engine": "InnoDB",
- "field_order": [
- "version",
- "user",
- "timestamp"
- ],
- "fields": [
- {
- "fieldname": "version",
- "fieldtype": "Data",
- "in_list_view": 1,
- "label": "Version",
- "read_only": 1
- },
- {
- "fieldname": "timestamp",
- "fieldtype": "Datetime",
- "in_list_view": 1,
- "label": "Timestamp",
- "read_only": 1
- },
- {
- "fieldname": "user",
- "fieldtype": "Link",
- "in_list_view": 1,
- "label": "User",
- "options": "User"
- }
- ],
- "istable": 1,
- "links": [],
- "modified": "2023-09-27 23:45:54.628246",
- "modified_by": "Administrator",
- "module": "Cloud Storage",
- "name": "File Version",
- "owner": "Administrator",
- "permissions": [],
- "sort_field": "modified",
- "sort_order": "DESC",
- "states": []
-}
\ No newline at end of file
+ "actions": [],
+ "creation": "2023-07-06 05:38:35.891393",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": ["version", "user", "timestamp"],
+ "fields": [
+ {
+ "fieldname": "version",
+ "fieldtype": "Data",
+ "in_list_view": 1,
+ "label": "Version",
+ "read_only": 1
+ },
+ {
+ "fieldname": "timestamp",
+ "fieldtype": "Datetime",
+ "in_list_view": 1,
+ "label": "Timestamp",
+ "read_only": 1
+ },
+ {
+ "fieldname": "user",
+ "fieldtype": "Link",
+ "in_list_view": 1,
+ "label": "User",
+ "options": "User"
+ }
+ ],
+ "istable": 1,
+ "links": [],
+ "modified": "2023-09-27 23:45:54.628246",
+ "modified_by": "Administrator",
+ "module": "Cloud Storage",
+ "name": "File Version",
+ "owner": "Administrator",
+ "permissions": [],
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "states": []
+}
diff --git a/cloud_storage/cloud_storage/overrides/file.py b/cloud_storage/cloud_storage/overrides/file.py
index c51d616..039b645 100644
--- a/cloud_storage/cloud_storage/overrides/file.py
+++ b/cloud_storage/cloud_storage/overrides/file.py
@@ -27,9 +27,48 @@
class CustomFile(File):
+ @File.is_remote_file.getter
+ def is_remote_file(self) -> bool:
+ if self.file_url: # type: ignore
+ return self.file_url.startswith(URL_PREFIXES) # type: ignore
+ return not self.content
+
def has_permission(self, ptype: Optional[str] = None, user: Optional[str] = None) -> bool:
return has_permission(self, ptype, user)
+ def validate(self) -> None:
+ self.associate_files()
+ if self.flags.cloud_storage or self.flags.ignore_file_validate:
+ return
+ if not self.is_remote_file:
+ super().validate()
+ else:
+ self.validate_file_url()
+
+ def after_insert(self) -> File:
+ if self.attached_to_doctype and self.attached_to_name and not self.file_association: # type: ignore
+ if not self.content_hash and "/api/method/retrieve" in self.file_url: # type: ignore
+ associated_doc = frappe.get_value("File", {"file_url": self.file_url}, "name") # type: ignore
+ else:
+ associated_doc = frappe.get_value(
+ "File",
+ {"content_hash": self.content_hash, "name": ["!=", self.name], "is_folder": False}, # type: ignore
+ )
+ if associated_doc:
+ self.db_set(
+ "file_url", ""
+ ) # this is done to prevent deletion of the remote file with the delete_file hook
+ rename_doc(
+ self.doctype,
+ self.name,
+ associated_doc,
+ merge=True,
+ force=True,
+ show_alert=False,
+ ignore_permissions=True,
+ # validate=False,
+ )
+
def on_trash(self) -> None:
user_roles = frappe.get_roles(frappe.session.user)
if (
@@ -64,7 +103,7 @@ def associate_files(
path = get_file_path(self, client.folder)
self.file_url = FILE_URL.format(path=path)
if not self.content_hash and "/api/method/retrieve" in self.file_url: # type: ignore
- associated_doc = frappe.get_value("File", {"file_url": self.file_url}, "name")
+ associated_doc = frappe.get_value("File", {"file_url": self.file_url}, "name") # type: ignore
else:
associated_doc = frappe.get_value(
"File",
@@ -76,7 +115,7 @@ def associate_files(
existing_file.attached_to_name = attached_to_name
self.content_hash = existing_file.content_hash
# if a File exists already where this association should be, we continue validating that File at this time
- # the original File will then be removed in the after insert hook
+ # the original File will then be removed in the after insert hook (also avoids recursion issues)
self = existing_file
existing_attachment = list(
@@ -98,39 +137,6 @@ def associate_files(
if associated_doc and associated_doc != self.name:
self.save()
- def validate(self) -> None:
- self.associate_files()
- if self.flags.cloud_storage or self.flags.ignore_file_validate:
- return
- if not self.is_remote_file:
- super().validate()
- else:
- self.validate_file_url()
-
- def after_insert(self) -> File:
- if self.attached_to_doctype and self.attached_to_name and not self.file_association: # type: ignore
- if not self.content_hash and "/api/method/retrieve" in self.file_url:
- associated_doc = frappe.get_value("File", {"file_url": self.file_url}, "name")
- else:
- associated_doc = frappe.get_value(
- "File",
- {"content_hash": self.content_hash, "name": ["!=", self.name], "is_folder": False},
- "name",
- )
- if associated_doc:
- self.db_set(
- "file_url", ""
- ) # this is done to prevent deletion of the remote file with the delete_file hook
- rename_doc(
- self.doctype,
- self.name,
- associated_doc,
- merge=True,
- force=True,
- show_alert=False,
- ignore_permissions=True,
- )
-
def add_file_version(self, version_id):
self.append(
"versions",
@@ -161,12 +167,6 @@ def remove_file_association(self, dt: str, dn: str) -> None:
association.idx = idx
self.save()
- @property
- def is_remote_file(self) -> bool:
- if self.file_url:
- return self.file_url.startswith(URL_PREFIXES)
- return not self.content
-
def get_content(self) -> bytes:
if self.is_folder:
frappe.throw(_("Cannot get file contents of a Folder"))
@@ -394,8 +394,6 @@ def upload_file(file: File) -> File:
except Exception as e:
frappe.log_error("File Upload Error", e)
file.db_set("s3_key", path)
- if not file.name:
- file.save()
return file
@@ -486,7 +484,8 @@ def delete_file(file: File, **kwargs) -> File:
except ClientError:
frappe.throw(_("Access denied: Could not delete file"))
except Exception as e:
- frappe.log_error(str(e), "Cloud Storage Error: Cloud not delete file")
+ print(f"EXCEPTION: {e}")
+ frappe.log_error(str(e), "Cloud Storage Error: Could not delete file")
return file
diff --git a/cloud_storage/customize.py b/cloud_storage/customize.py
index fd2fccd..56aae59 100644
--- a/cloud_storage/customize.py
+++ b/cloud_storage/customize.py
@@ -15,6 +15,8 @@ def load_customizations():
for file in files:
customizations = json.loads(Path(file).read_text())
for field in customizations.get("custom_fields"):
+ if field.get("module") != "Cloud Storage":
+ continue
existing_field = frappe.get_value("Custom Field", field.get("name"))
custom_field = (
frappe.get_doc("Custom Field", field.get("name"))
@@ -27,6 +29,8 @@ def load_customizations():
custom_field.flags.ignore_version = True
custom_field.save()
for prop in customizations.get("property_setters"):
+ if field.get("module") != "Cloud Storage":
+ continue
property_setter = frappe.get_doc(
{
"name": prop.get("name"),
diff --git a/cloud_storage/tests/conftest.py b/cloud_storage/tests/conftest.py
index 9ad5950..7d8857e 100644
--- a/cloud_storage/tests/conftest.py
+++ b/cloud_storage/tests/conftest.py
@@ -1,59 +1,57 @@
-import os
-from pathlib import Path
-from unittest.mock import MagicMock, patch
-
-import frappe
-import pytest
-from frappe.defaults import *
-from frappe.utils import get_bench_path
-
-
-def _get_logger(*args, **kwargs):
- from frappe.utils.logger import get_logger
-
- return get_logger(
- module=None,
- with_more_info=False,
- allow_site=True,
- filter=None,
- max_size=100_000,
- file_count=20,
- stream_only=True,
- )
-
-
-@pytest.fixture(scope="module")
-def monkeymodule():
- with pytest.MonkeyPatch.context() as mp:
- yield mp
-
-
-@pytest.fixture(scope="session", autouse=True)
-def db_instance():
- frappe.logger = _get_logger
-
- currentsite = "test_site"
- sites = Path(get_bench_path()) / "sites"
- if (sites / "currentsite.txt").is_file():
- currentsite = (sites / "currentsite.txt").read_text()
-
- frappe.init(site=currentsite, sites_path=sites)
- frappe.connect()
- frappe.db.commit = MagicMock()
- yield frappe.db
-
-
-@pytest.fixture(scope="module", autouse=True)
-def patch_frappe_conf(monkeymodule):
- monkeymodule.setattr(
- "frappe.conf.cloud_storage_settings",
- {
- "access_key": "test",
- "secret": "test_secret",
- "region": "us-east-1",
- "bucket": "test_bucket",
- "endpoint_url": "https://test.imgainarys3.edu",
- "expiration": 110,
- "folder": "test_folder",
- },
- )
+from pathlib import Path
+from unittest.mock import MagicMock
+
+import frappe
+import pytest
+from frappe.utils import get_bench_path
+
+
+def _get_logger(*args, **kwargs):
+ from frappe.utils.logger import get_logger
+
+ return get_logger(
+ module=None,
+ with_more_info=False,
+ allow_site=True,
+ filter=None,
+ max_size=100_000,
+ file_count=20,
+ stream_only=True,
+ )
+
+
+@pytest.fixture(scope="module")
+def monkeymodule():
+ with pytest.MonkeyPatch.context() as mp:
+ yield mp
+
+
+@pytest.fixture(scope="session", autouse=True)
+def db_instance():
+ frappe.logger = _get_logger
+
+ currentsite = "test_site"
+ sites = Path(get_bench_path()) / "sites"
+ if (sites / "currentsite.txt").is_file():
+ currentsite = (sites / "currentsite.txt").read_text()
+
+ frappe.init(site=currentsite, sites_path=sites)
+ frappe.connect()
+ frappe.db.commit = MagicMock()
+ yield frappe.db
+
+
+@pytest.fixture(scope="module", autouse=True)
+def patch_frappe_conf(monkeymodule):
+ monkeymodule.setattr(
+ "frappe.conf.cloud_storage_settings",
+ {
+ "access_key": "test",
+ "secret": "test_secret",
+ "region": "us-east-1",
+ "bucket": "test_bucket",
+ "endpoint_url": "https://test.imgainarys3.edu",
+ "expiration": 110,
+ "folder": "test_folder",
+ },
+ )
diff --git a/cloud_storage/tests/fixtures/aticonrust.svg b/cloud_storage/tests/fixtures/aticonrust.svg
new file mode 100644
index 0000000..4418b08
--- /dev/null
+++ b/cloud_storage/tests/fixtures/aticonrust.svg
@@ -0,0 +1,64 @@
+
+
+
+
diff --git a/cloud_storage/tests/fixtures/atlogo_rust.svg b/cloud_storage/tests/fixtures/atlogo_rust.svg
new file mode 100644
index 0000000..888c4ac
--- /dev/null
+++ b/cloud_storage/tests/fixtures/atlogo_rust.svg
@@ -0,0 +1,256 @@
+
+
+
+
diff --git a/cloud_storage/tests/test_file.py b/cloud_storage/tests/test_file.py
index 7e28d6d..41b5312 100644
--- a/cloud_storage/tests/test_file.py
+++ b/cloud_storage/tests/test_file.py
@@ -1,175 +1,126 @@
-from unittest.mock import MagicMock, patch
+from io import BytesIO
+from pathlib import Path
import frappe
-from boto3.exceptions import S3UploadFailedError
-from botocore.exceptions import ClientError
-from frappe.tests.utils import FrappeTestCase
-
-from cloud_storage.cloud_storage.overrides.file import (
- CustomFile,
- delete_file,
- upload_file,
- write_file,
-)
-
-
-class TestFile(FrappeTestCase):
- @patch("cloud_storage.cloud_storage.overrides.file.upload_file")
- @patch("cloud_storage.cloud_storage.overrides.file.strip_special_chars")
- @patch("frappe.get_all")
- @patch("frappe.conf")
- def test_write_file(self, config, get_all, strip_chars, upload_file):
- file = MagicMock()
-
- # test local fallback
- config.cloud_storage_settings = None
- write_file(file)
- assert file.save_file_on_filesystem.call_count == 1
-
- config.cloud_storage_settings = {"use_local": True}
- write_file(file)
- assert file.save_file_on_filesystem.call_count == 2
-
- config.cloud_storage_settings = {"use_local": False}
- file.attached_to_doctype = "Data Import"
- write_file(file)
- assert file.save_file_on_filesystem.call_count == 3
-
- # test file upload with autoname
- file.attached_to_doctype = None
- file.name = None
- file.file_name = "test_file.png"
- strip_chars.return_value = "test_file.png"
- upload_file.return_value = file
- get_all.return_value = []
- write_file(file)
- upload_file.assert_called_with(file)
-
- # test file upload without autoname
- file.name = "test_file"
- file.file_name = "test_file.png"
- upload_file.return_value = file
- write_file(file)
- upload_file.assert_called_with(file)
-
- @patch("cloud_storage.cloud_storage.overrides.file.get_cloud_storage_client")
- @patch("cloud_storage.cloud_storage.overrides.file.get_file_path")
- def test_upload_file(self, file_path, client):
- # setup file
- file = MagicMock()
- file.content_type = "image/jpeg"
-
- # test general exception
- client.return_value.put_object.side_effect = TypeError
- upload_file(file)
- assert client.return_value.put_object.call_count == 1
-
- # test upload errors
- client.return_value.put_object.side_effect = S3UploadFailedError
- with self.assertRaises(frappe.ValidationError):
- upload_file(file)
- assert client.return_value.put_object.call_count == 2
-
- # test upload success
- client.return_value.put_object.side_effect = True
- file_path.return_value = "/path/to/s3/bucket/location"
- upload_file(file)
- assert client.return_value.put_object.call_count == 3
-
- @patch("cloud_storage.cloud_storage.overrides.file.get_cloud_storage_client")
- @patch("frappe.conf")
- def test_delete_file(self, config, client):
- file = MagicMock()
-
- # test local fallback
- config.cloud_storage_settings = None
- delete_file(file)
- assert file.delete_file_from_filesystem.call_count == 1
- assert client.return_value.delete_object.call_count == 0
-
- config.cloud_storage_settings = {"use_local": True}
- delete_file(file)
- assert file.delete_file_from_filesystem.call_count == 2
- assert client.return_value.delete_object.call_count == 0
-
- # test skip folder deletion
- config.cloud_storage_settings = {"use_local": False}
- file.is_folder = True
- delete_file(file)
- assert client.return_value.delete_object.call_count == 0
-
- # test skip file deletion from missing url or key param
- file.is_folder = False
- file.file_url = None
- delete_file(file)
- assert client.return_value.delete_object.call_count == 0
-
- file.file_url = "/api/method/retrieve"
- delete_file(file)
- assert client.return_value.delete_object.call_count == 0
-
- file.file_url = "/api/method/retrieve?key="
- delete_file(file)
- assert client.return_value.delete_object.call_count == 0
-
- # test general exception
- file.file_url = "/api/method/retrieve?key=/path/to/s3/bucket/location"
- client.return_value.delete_object.side_effect = TypeError
- delete_file(file)
- assert client.return_value.delete_object.call_count == 1
-
- # test upload errors
- client.return_value.delete_object.side_effect = ClientError(
- error_response={"Error": {}}, operation_name="delete_file"
- )
- with self.assertRaises(frappe.ValidationError):
- delete_file(file)
- assert client.return_value.delete_object.call_count == 2
-
- # test upload success
- client.bucket = "bucket"
- client.return_value.delete_object.side_effect = True
- delete_file(file)
- assert client.return_value.delete_object.call_count == 3
-
- @patch("cloud_storage.cloud_storage.overrides.file.has_user_permission")
- @patch("frappe.has_permission")
- @patch("frappe.get_doc")
- def test_file_permission(self, get_doc, has_permission, has_user_permission):
- # test file access for owner
- file = CustomFile({"doctype": "File", "owner": "Administrator"})
- self.assertEqual(file.has_permission(), True)
- self.assertEqual(file.has_permission(user="Administrator"), True)
-
- # test file access for non-owner user
- has_permission.return_value = True
- assert file.has_permission(user="Administrator") is True
- assert file.has_permission(user="support@agritheory.dev") is True
- has_permission.return_value = False
- assert file.has_permission(user="Administrator") is True
- assert file.has_permission(user="support@agritheory.dev") is False
-
- # test file access for attached doctypes
- file = CustomFile(
- {
- "doctype": "File",
- "owner": "Administrator",
- "attached_to_doctype": "Sales Order",
- "attached_to_name": "SO-0001",
- }
- )
-
- # test file access for doc permissions
- get_doc.return_value = MagicMock()
- get_doc.return_value.has_permission.return_value = True
- assert file.has_permission(user="Administrator") is True
- assert file.has_permission(user="support@agritheory.dev") is True
-
- # test file access for custom user permissions
- get_doc.return_value.has_permission.return_value = False
- has_user_permission.return_value = True
- assert file.has_permission(user="Administrator") is True
- assert file.has_permission(user="support@agritheory.dev") is True
- has_user_permission.return_value = False
- assert file.has_permission(user="Administrator") is True
- assert file.has_permission(user="support@agritheory.dev") is False
+import pytest
+from moto import mock_s3
+from werkzeug.datastructures import FileMultiDict
+
+from cloud_storage.cloud_storage.overrides.file import CustomFile, retrieve
+
+
+@pytest.fixture
+def example_file_record_0():
+ return Path(__file__).parent / "fixtures" / "aticonrusthex.png"
+
+
+@pytest.fixture
+def example_file_record_1():
+ return Path(__file__).parent / "fixtures" / "aticonrust.svg"
+
+
+@pytest.fixture
+def example_file_record_2():
+ return Path(__file__).parent / "fixtures" / "atlogo_rust.svg"
+
+
+@pytest.fixture
+def get_cloud_storage_client_fixture():
+ return frappe.call("cloud_storage.cloud_storage.overrides.file.get_cloud_storage_client")
+
+
+# helper function
+def create_upload_file(file_path: Path, **kwargs) -> CustomFile:
+ f = BytesIO(file_path.resolve().read_bytes())
+ f.seek(0)
+
+ # simulate a Frappe client -> server request file object using Werkzeug
+ files = FileMultiDict()
+ files.add_file("file", f, kwargs.get("file_name"))
+
+ frappe.set_user("Administrator")
+ frappe.local.request = frappe._dict()
+ frappe.local.request.method = kwargs.get("method") or "POST"
+ frappe.local.request.files = files
+ frappe.local.form_dict = frappe._dict()
+ frappe.local.form_dict.is_private = False
+ frappe.local.form_dict.doctype = kwargs.get("doctype") or "User"
+ frappe.local.form_dict.docname = kwargs.get("docname") or "Administrator"
+ frappe.local.form_dict.fieldname = kwargs.get("fieldname") or None
+ frappe.local.form_dict.file_url = kwargs.get("file_url") or None
+ frappe.local.form_dict.folder = kwargs.get("folder") or "Home"
+ frappe.local.form_dict.file_name = kwargs.get("file_name") or None
+ frappe.local.form_dict.optimize = kwargs.get("optimize") or False
+ file = frappe.call("frappe.handler.upload_file")
+ return file
+
+
+@mock_s3
+def test_config(get_cloud_storage_client_fixture):
+ c = get_cloud_storage_client_fixture
+ assert c.bucket == "test_bucket"
+ assert c.folder == "test_folder"
+ assert c.expiration == 110
+ assert c._endpoint._endpoint_prefix == "s3"
+ assert c._endpoint.host == "https://test.imgainarys3.edu"
+
+
+@mock_s3
+def test_upload_file(example_file_record_0):
+ frappe.set_user("Administrator")
+ file = create_upload_file(example_file_record_0, file_name="aticonrusthex.png")
+ assert frappe.db.exists("File", file.name)
+ assert file.attached_to_doctype == "User"
+ assert file.attached_to_name == "Administrator"
+ assert file.attached_to_field is None
+ assert file.folder == "Home"
+ assert file.file_name == "aticonrusthex.png"
+ assert file.content_hash is not None
+ assert (
+ file.file_url == "/api/method/retrieve?key=test_folder/User/Administrator/aticonrusthex.png"
+ )
+ assert file.is_private == 0 # makes the delete file test easier
+ assert file.s3_key is not None
+ assert len(file.file_association) == 1
+ assert file.file_association[0].link_doctype == "User"
+ assert file.file_association[0].link_name == "Administrator"
+
+ # Test manual association
+ file.append("file_association", {"link_doctype": "Module Def", "link_name": "Cloud Storage"})
+ file.save()
+ assert len(file.file_association) == 2
+ assert file.file_association[1].link_doctype == "Module Def"
+ assert file.file_association[1].link_name == "Cloud Storage"
+
+
+@mock_s3
+def test_upload_file_with_multiple_association(example_file_record_1):
+ _file = create_upload_file(example_file_record_1, file_name="aticonrust.svg")
+ file = create_upload_file(
+ example_file_record_1,
+ doctype="Module Def",
+ docname="Automation",
+ file_name="aticonrust.svg",
+ )
+
+ _file.load_from_db()
+ assert frappe.db.exists("File", _file.name) is not None
+ assert frappe.db.exists("File", file.name) is None
+ assert len(_file.file_association) == 2
+ assert _file.file_association[0].link_doctype == "User"
+ assert _file.file_association[0].link_name == "Administrator"
+ assert _file.file_association[1].link_doctype == "Module Def"
+ assert _file.file_association[1].link_name == "Automation"
+
+
+@mock_s3
+def test_delete_file(example_file_record_2):
+ frappe.set_user("Administrator")
+ file = create_upload_file(example_file_record_2, file_name="atlogo_rust.svg")
+ s3_key = file.s3_key
+ file.delete()
+
+ assert not frappe.db.exists("File", file.name)
+
+ with pytest.raises(frappe.exceptions.DoesNotExistError):
+ retrieve(s3_key)
diff --git a/cloud_storage/tests/test_file_association.py b/cloud_storage/tests/test_file_association.py
deleted file mode 100644
index ec01b31..0000000
--- a/cloud_storage/tests/test_file_association.py
+++ /dev/null
@@ -1,88 +0,0 @@
-from io import BytesIO
-from pathlib import Path
-
-import frappe
-import pytest
-from moto import mock_s3
-
-
-@pytest.fixture
-def example_file_record_0():
- return Path(__file__).parent / "fixtures" / "aticonrusthex.png"
-
-
-@pytest.fixture
-def get_cloud_storage_client_fixture():
- return frappe.call("cloud_storage.cloud_storage.overrides.file.get_cloud_storage_client")
-
-
-@mock_s3
-def test_config(get_cloud_storage_client_fixture):
- c = get_cloud_storage_client_fixture
- assert c.bucket == "test_bucket"
- assert c.folder == "test_folder"
- assert c.expiration == 110
- assert c._endpoint._endpoint_prefix == "s3"
- assert c._endpoint.host == "https://test.imgainarys3.edu"
-
-
-# helper function
-def create_upload_file(file_path, **kwargs):
- f = BytesIO(file_path.resolve().read_bytes())
- f.seek(0)
- frappe.set_user("Administrator")
- frappe.local.request = frappe._dict()
- frappe.local.request.method = kwargs.get("method") or "POST"
- frappe.local.request.files = f
- frappe.local.form_dict = frappe._dict()
- frappe.local.form_dict.is_private = True
- frappe.local.form_dict.doctype = kwargs.get("doctype") or "User"
- frappe.local.form_dict.docname = kwargs.get("docname") or "Administrator"
- frappe.local.form_dict.fieldname = kwargs.get("fieldname") or "image"
- frappe.local.form_dict.file_url = kwargs.get("file_url") or None
- frappe.local.form_dict.folder = kwargs.get("folder") or "Home"
- frappe.local.form_dict.file_name = kwargs.get("file_name") or "aticonrusthex.png"
- frappe.local.form_dict.optimize = kwargs.get("optimize") or False
- return frappe.call("frappe.handler.upload_file")
-
-
-@mock_s3
-def test_upload_file(example_file_record_0):
- file = create_upload_file(example_file_record_0)
- assert frappe.db.exists("File", file.name)
- assert file.attached_to_doctype == "User"
- assert file.attached_to_name == "Administrator"
- assert file.attached_to_field == "image"
- assert file.folder == "Home"
- assert file.file_name == "aticonrusthex.png"
- assert file.content_hash is None
- assert (
- file.file_url == "/api/method/retrieve?key=test_folder/User/Administrator/aticonrusthex.png"
- )
- assert file.is_private == 1
- assert len(file.file_association) == 1
- assert file.file_association[0].link_doctype == "User"
- assert file.file_association[0].link_name == "Administrator"
- file.append("file_association", {"link_doctype": "Module Def", "link_name": "Cloud Storage"})
- file.save()
- assert len(file.file_association) == 2
- assert file.file_association[1].link_doctype == "Module Def"
- assert file.file_association[1].link_name == "Cloud Storage"
-
-
-# @mock_s3
-# def test_upload_file_with_association(example_file_record_0):
-# file = create_upload_file(example_file_record_0)
-# second_association = create_upload_file(example_file_record_0, doctype='Module Def', docname="Cloud Storage")
-# print(second_association.__dict__)
-# assert frappe.db.exists('File', file.name)
-# assert file.attached_to_doctype == 'User'
-# assert file.attached_to_name == 'Administrator'
-# assert file.attached_to_field == 'image'
-# assert file.folder == 'Home'
-# assert file.file_name == 'aticonrusthex.png'
-# assert file.file_url == '/api/method/retrieve?key=test_folder/User/Administrator/aticonrusthex.png'
-# assert file.is_private == 1
-# assert len(file.file_association) == 1
-# assert file.file_association[0].link_doctype == 'User'
-# assert file.file_association[0].link_name == 'Administrator'
diff --git a/package.json b/package.json
index 01bcb31..8e75daf 100644
--- a/package.json
+++ b/package.json
@@ -1,5 +1,6 @@
{
"name": "@agritheory/cloud_storage",
+ "type": "module",
"dependencies": {
"madr": "^3.0.0"
},
diff --git a/pyproject.toml b/pyproject.toml
index 3abea04..ee3098d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -10,9 +10,10 @@ dynamic = ["version"]
dependencies = [
"boto3==1.28.10",
"python-magic~=0.4.27",
- "pytest",
- 'moto[s3]'
+ "pytest~=7.2.2",
+ "moto~=4.1.6"
]
+moto = ["S3"]
[build-system]
requires = ["flit_core >=3.4,<4"]