From 1452f34bf903e940aee11d8a5eeec928da5712ef Mon Sep 17 00:00:00 2001 From: Heather Kusmierz Date: Mon, 6 May 2024 07:55:36 -0400 Subject: [PATCH] test: fix delete file test (#63) * ci: add frappe black to CI (#57) * ci: add frappe black to CI * chore: black * ci: fix v15 tests * ci: run redis server --------- Co-authored-by: Tyler Matteson Co-authored-by: Heather Kusmierz Co-authored-by: Rohan Bansal --- .backportrc.json | 28 +- .github/helper/install.sh | 2 +- .github/validate_customizations.py | 164 ++++ .github/workflows/lint.yaml | 53 +- .../workflows/{frappe.yaml => pytest.yaml} | 11 +- .github/workflows/release.yaml | 1 + .pre-commit-config.yaml | 49 +- .prettierrc.js => .prettierrc.cjs | 0 cloud_storage/cloud_storage/custom/file.json | 808 ++++++++---------- .../file_association/file_association.json | 111 ++- .../doctype/file_version/file_version.json | 86 +- cloud_storage/cloud_storage/overrides/file.py | 87 +- cloud_storage/customize.py | 4 + cloud_storage/tests/conftest.py | 116 ++- cloud_storage/tests/fixtures/aticonrust.svg | 64 ++ cloud_storage/tests/fixtures/atlogo_rust.svg | 256 ++++++ cloud_storage/tests/test_file.py | 297 +++---- cloud_storage/tests/test_file_association.py | 88 -- package.json | 1 + pyproject.toml | 5 +- 20 files changed, 1244 insertions(+), 987 deletions(-) create mode 100644 .github/validate_customizations.py rename .github/workflows/{frappe.yaml => pytest.yaml} (83%) rename .prettierrc.js => .prettierrc.cjs (100%) create mode 100644 cloud_storage/tests/fixtures/aticonrust.svg create mode 100644 cloud_storage/tests/fixtures/atlogo_rust.svg delete mode 100644 cloud_storage/tests/test_file_association.py diff --git a/.backportrc.json b/.backportrc.json index 6feb957..e48e270 100644 --- a/.backportrc.json +++ b/.backportrc.json @@ -1,17 +1,11 @@ -{ - "repoOwner": "agritheory", - "repoName": "cloud_storage", - "targetBranchChoices": [ - "version-14", - "version-15" - ], - "targetBranches": [ - "version-14", - "version-15" - ], - "autoMerge": true, - "autoMergeMethod": "squash", - "branchLabelMapping": { - "^auto-backport-to-(.+)$": "$1" - } -} \ No newline at end of file +{ + "repoOwner": "agritheory", + "repoName": "cloud_storage", + "targetBranchChoices": ["version-14", "version-15"], + "targetBranches": ["version-14", "version-15"], + "autoMerge": true, + "autoMergeMethod": "squash", + "branchLabelMapping": { + "^auto-backport-to-(.+)$": "$1" + } +} diff --git a/.github/helper/install.sh b/.github/helper/install.sh index 30d0d12..8990865 100644 --- a/.github/helper/install.sh +++ b/.github/helper/install.sh @@ -10,7 +10,7 @@ sudo apt install libcups2-dev redis-server mariadb-client-10.6 pip install frappe-bench -bench init --skip-redis-config-generation --skip-assets --python "$(which python)" --frappe-branch version-15 frappe-bench +bench init --skip-assets --python "$(which python)" --frappe-branch version-15 frappe-bench mkdir ~/frappe-bench/sites/test_site cp -r "${GITHUB_WORKSPACE}/.github/helper/site_config.json" ~/frappe-bench/sites/test_site/ diff --git a/.github/validate_customizations.py b/.github/validate_customizations.py new file mode 100644 index 0000000..f942698 --- /dev/null +++ b/.github/validate_customizations.py @@ -0,0 +1,164 @@ +import json +import pathlib +import sys + + +def scrub(txt: str) -> str: + return txt.replace(" ", "_").replace("-", "_").lower() + + +def unscrub(txt: str) -> str: + return txt.replace("_", " ").replace("-", " ").title() + + +def get_customized_doctypes(): + apps_dir = pathlib.Path(__file__).resolve().parent.parent.parent + apps_order = pathlib.Path(__file__).resolve().parent.parent.parent.parent / "sites" / "apps.txt" + apps_order = apps_order.read_text().split("\n") + customized_doctypes = {} + for _app_dir in apps_order: + app_dir = (apps_dir / _app_dir).resolve() + if not app_dir.is_dir(): + continue + modules = (app_dir / _app_dir / "modules.txt").read_text().split("\n") + for module in modules: + if not (app_dir / _app_dir / scrub(module) / "custom").exists(): + continue + for custom_file in list((app_dir / _app_dir / scrub(module) / "custom").glob("**/*.json")): + if custom_file.stem in customized_doctypes: + customized_doctypes[custom_file.stem].append(custom_file.resolve()) + else: + customized_doctypes[custom_file.stem] = [custom_file.resolve()] + + return dict(sorted(customized_doctypes.items())) + + +def validate_module(customized_doctypes, set_module=False): + exceptions = [] + app_dir = pathlib.Path(__file__).resolve().parent.parent + this_app = app_dir.stem + for doctype, customize_files in customized_doctypes.items(): + for customize_file in customize_files: + if not this_app in str(customize_file): + continue + module = customize_file.parent.parent.stem + file_contents = json.loads(customize_file.read_text()) + if file_contents.get("custom_fields"): + for custom_field in file_contents.get("custom_fields"): + if set_module: + custom_field["module"] = unscrub(module) + continue + if not custom_field.get("module"): + exceptions.append( + f"Custom Field for {custom_field.get('dt')} in {this_app} '{custom_field.get('fieldname')}' does not have a module key" + ) + continue + elif custom_field.get("module") != unscrub(module): + exceptions.append( + f"Custom Field for {custom_field.get('dt')} in {this_app} '{custom_field.get('fieldname')}' has module key ({custom_field.get('module')}) associated with another app" + ) + continue + if file_contents.get("property_setters"): + for ps in file_contents.get("property_setters"): + if set_module: + ps["module"] = unscrub(module) + continue + if not ps.get("module"): + exceptions.append( + f"Property Setter for {ps.get('doc_type')} in {this_app} '{ps.get('property')}' on {ps.get('field_name')} does not have a module key" + ) + continue + elif ps.get("module") != unscrub(module): + exceptions.append( + f"Property Setter for {ps.get('doc_type')} in {this_app} '{ps.get('property')}' on {ps.get('field_name')} has module key ({ps.get('module')}) associated with another app" + ) + continue + if set_module: + with customize_file.open("w", encoding="UTF-8") as target: + json.dump(file_contents, target, sort_keys=True, indent=2) + + return exceptions + + +def validate_no_custom_perms(customized_doctypes): + exceptions = [] + this_app = pathlib.Path(__file__).resolve().parent.parent.stem + for doctype, customize_files in customized_doctypes.items(): + for customize_file in customize_files: + if not this_app in str(customize_file): + continue + file_contents = json.loads(customize_file.read_text()) + if file_contents.get("custom_perms"): + exceptions.append(f"Customization for {doctype} in {this_app} contains custom permissions") + return exceptions + + +def validate_duplicate_customizations(customized_doctypes): + exceptions = [] + common_fields = {} + common_property_setters = {} + app_dir = pathlib.Path(__file__).resolve().parent.parent + this_app = app_dir.stem + for doctype, customize_files in customized_doctypes.items(): + if len(customize_files) == 1: + continue + common_fields[doctype] = {} + common_property_setters[doctype] = {} + for customize_file in customize_files: + module = customize_file.parent.parent.stem + app = customize_file.parent.parent.parent.parent.stem + file_contents = json.loads(customize_file.read_text()) + if file_contents.get("custom_fields"): + fields = [cf.get("fieldname") for cf in file_contents.get("custom_fields")] + common_fields[doctype][module] = fields + if file_contents.get("property_setters"): + ps = [ps.get("name") for ps in file_contents.get("property_setters")] + common_property_setters[doctype][module] = ps + + for doctype, module_and_fields in common_fields.items(): + if this_app not in module_and_fields.keys(): + continue + this_modules_fields = module_and_fields.pop(this_app) + for module, fields in module_and_fields.items(): + for field in fields: + if field in this_modules_fields: + exceptions.append( + f"Custom Field for {unscrub(doctype)} in {this_app} '{field}' also appears in customizations for {module}" + ) + + for doctype, module_and_ps in common_property_setters.items(): + if this_app not in module_and_ps.keys(): + continue + this_modules_ps = module_and_ps.pop(this_app) + for module, ps in module_and_ps.items(): + for p in ps: + if p in this_modules_ps: + exceptions.append( + f"Property Setter for {unscrub(doctype)} in {this_app} on '{p}' also appears in customizations for {module}" + ) + + return exceptions + + +def validate_customizations(set_module): + customized_doctypes = get_customized_doctypes() + exceptions = validate_no_custom_perms(customized_doctypes) + exceptions += validate_module(customized_doctypes, set_module) + exceptions += validate_duplicate_customizations(customized_doctypes) + + return exceptions + + +if __name__ == "__main__": + exceptions = [] + set_module = False + for arg in sys.argv: + if arg == "--set-module": + set_module = True + exceptions.append(validate_customizations(set_module)) + + if exceptions: + for exception in exceptions: + [print(e) for e in exception] # TODO: colorize + + sys.exit(1) if all(exceptions) else sys.exit(0) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index c3d22bf..a02a968 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -3,8 +3,12 @@ name: Linters on: push: branches: + - version-14 - version-15 pull_request: + branches: + - version-14 + - version-15 env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -25,8 +29,8 @@ jobs: with: python-version: '3.10' - - name: Install mypy - run: pip install mypy + - name: Install mypy and types + run: python -m pip install mypy types-python-dateutil types-pytz --no-input - name: Run mypy uses: sasanquaneuf/mypy-github-action@releases/v1 @@ -35,6 +39,27 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + black: + needs: [ py_json_merge ] + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ github.head_ref }} + fetch-depth: 2 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Install Black (Frappe) + run: pip install git+https://github.com/frappe/black.git + + - name: Run Black (Frappe) + run: black --check . + prettier: needs: [ py_json_merge ] runs-on: ubuntu-latest @@ -46,9 +71,17 @@ jobs: fetch-depth: 2 - name: Prettify code - uses: creyD/prettier_action@v4.3 + uses: rutajdash/prettier-cli-action@v1.0.0 with: - commit_message: "style: prettify code" + config_path: ./.prettierrc.js + ignore_path: ./.prettierignore + + - name: Prettier Output + if: ${{ failure() }} + shell: bash + run: | + echo "The following files are not formatted:" + echo "${{steps.prettier-run.outputs.prettier_output}}" >> $GITHUB_OUTPUT json_diff: needs: [ py_json_merge ] @@ -62,7 +95,7 @@ jobs: - name: Find JSON changes id: changed-json - uses: tj-actions/changed-files@v37 + uses: tj-actions/changed-files@v43 with: files: | **/*.json @@ -106,14 +139,6 @@ jobs: echo "D,${file}" >> base/mrd.txt done - - name: Setup requirements and script - run: | - pip install rich - pip install json_source_map - git clone --depth 1 https://gist.github.com/3eea518743067f1b971114f1a2016f69 fsjd - - - name: Diff table - run: python3 fsjd/frappe_schema_json_diff.py base/mrd.txt head/acmr.txt 1 py_json_merge: runs-on: ubuntu-latest @@ -124,7 +149,7 @@ jobs: run: git clone --depth 1 https://gist.github.com/f1bf2c11f78331b2417189c385022c28.git validate_json - name: Validate JSON - run: python3 validate_json/validate_json.py ./ + run: python3 validate_json/validate_json.py ./cloud_storage/cloud_storage/ - name: Compile run: python3 -m compileall -q ./ diff --git a/.github/workflows/frappe.yaml b/.github/workflows/pytest.yaml similarity index 83% rename from .github/workflows/frappe.yaml rename to .github/workflows/pytest.yaml index fbb7997..683b29d 100644 --- a/.github/workflows/frappe.yaml +++ b/.github/workflows/pytest.yaml @@ -50,15 +50,16 @@ jobs: with: node-version: 18 check-latest: true + cache: 'yarn' - name: Add to Hosts run: echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts - name: Cache pip - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/pyproject.toml', '**/setup.py') }} + key: ${{ runner.os }}-pip-${{ hashFiles('**/pyproject.toml') }} restore-keys: | ${{ runner.os }}-pip- ${{ runner.os }}- @@ -67,7 +68,7 @@ jobs: id: yarn-cache-dir-path run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT - - uses: actions/cache@v3 + - uses: actions/cache@v4 id: yarn-cache with: path: ${{ steps.yarn-cache-dir-path.outputs.dir }} @@ -82,9 +83,7 @@ jobs: - name: Run Tests working-directory: /home/runner/frappe-bench run: | - bench --site test_site set-config allow_tests true - bench --site test_site run-tests --app cloud_storage source env/bin/activate - pytest ./apps/cloud_storage/cloud_storage/tests/test_file_association.py --disable-warnings -s + pytest ./apps/cloud_storage/cloud_storage/tests/ -s env: TYPE: server diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index b9c3392..b4d9f60 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -3,6 +3,7 @@ name: Release on: push: branches: + - version-14 - version-15 jobs: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 442e8e4..1fbe0ae 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,8 +7,8 @@ repos: rev: v4.3.0 hooks: - id: trailing-whitespace - files: "cloud_storage.*" - exclude: ".*json$|.*txt$|.*csv|.*md|.*svg" + files: 'cloud_storage.*' + exclude: '.*json$|.*txt$|.*csv|.*md|.*svg' - id: check-yaml - id: no-commit-to-branch args: ['--branch', 'develop'] @@ -37,14 +37,28 @@ repos: types_or: [javascript, vue, scss] # Ignore any files that might contain jinja / bundles exclude: | - (?x)^( - .*node_modules.*| - cloud_storage/public/dist/.*| - cloud_storage/public/js/lib/.*| - cloud_storage/templates/includes/.*| - cloud_storage/www/website_script.js - )$ + (?x)^( + .*node_modules.*| + cloud_storage/public/dist/.*| + cloud_storage/public/js/lib/.*| + cloud_storage/templates/includes/.* + )$ + - repo: https://github.com/pre-commit/mirrors-eslint + rev: v8.44.0 + hooks: + - id: eslint + types_or: [javascript] + args: ['--quiet'] + # Ignore any files that might contain jinja / bundles + exclude: | + (?x)^( + .*node_modules.*| + cloud_storage/public/dist/.*| + cloud_storage/public/js/lib/.*| + cloud_storage/templates/includes/.*| + cloud_storage/www/website_script.js + )$ - repo: https://github.com/pre-commit/mirrors-eslint rev: v8.44.0 @@ -71,9 +85,18 @@ repos: rev: 6.0.0 hooks: - id: flake8 - additional_dependencies: ['flake8-bugbear',] + additional_dependencies: ['flake8-bugbear'] + + - repo: local + hooks: + - id: validate_customizations + always_run: true + name: .github/validate_customizations.py + entry: python .github/validate_customizations.py + language: system + types: [python] ci: - autoupdate_schedule: weekly - skip: [] - submodules: false + autoupdate_schedule: weekly + skip: [] + submodules: false diff --git a/.prettierrc.js b/.prettierrc.cjs similarity index 100% rename from .prettierrc.js rename to .prettierrc.cjs diff --git a/cloud_storage/cloud_storage/custom/file.json b/cloud_storage/cloud_storage/custom/file.json index 11617aa..70a7500 100644 --- a/cloud_storage/cloud_storage/custom/file.json +++ b/cloud_storage/cloud_storage/custom/file.json @@ -1,468 +1,342 @@ { - "custom_fields": [ - { - "_assign": null, - "_comments": null, - "_liked_by": null, - "_user_tags": null, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "collapsible_depends_on": null, - "columns": 0, - "creation": "2023-02-03 12:03:51.204553", - "default": null, - "depends_on": null, - "description": null, - "docstatus": 0, - "dt": "File", - "fetch_from": null, - "fetch_if_empty": 0, - "fieldname": "s3_key", - "fieldtype": "Data", - "hidden": 0, - "hide_border": 0, - "hide_days": 0, - "hide_seconds": 0, - "idx": 23, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_preview": 0, - "in_standard_filter": 0, - "insert_after": "uploaded_to_google_drive", - "is_system_generated": 0, - "is_virtual": 0, - "label": "S3 Key", - "length": 0, - "mandatory_depends_on": null, - "modified": "2023-07-06 02:04:01.131489", - "modified_by": "Administrator", - "module": null, - "name": "File-s3_key", - "no_copy": 0, - "non_negative": 0, - "options": null, - "owner": "Administrator", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "print_width": null, - "read_only": 1, - "read_only_depends_on": null, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "sort_options": 0, - "translatable": 0, - "unique": 0, - "width": null - }, - { - "_assign": null, - "_comments": null, - "_liked_by": null, - "_user_tags": null, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "collapsible_depends_on": null, - "columns": 0, - "creation": "2023-02-15 14:00:22.073075", - "default": null, - "depends_on": null, - "description": null, - "docstatus": 0, - "dt": "File", - "fetch_from": null, - "fetch_if_empty": 0, - "fieldname": "sharing_link", - "fieldtype": "Data", - "hidden": 0, - "hide_border": 0, - "hide_days": 0, - "hide_seconds": 0, - "idx": 24, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_preview": 0, - "in_standard_filter": 0, - "insert_after": "s3_key", - "is_system_generated": 0, - "is_virtual": 0, - "label": "Sharing Link", - "length": 0, - "mandatory_depends_on": null, - "modified": "2023-07-06 02:04:01.272045", - "modified_by": "Administrator", - "module": null, - "name": "File-sharing_link", - "no_copy": 0, - "non_negative": 0, - "options": null, - "owner": "Administrator", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "print_width": null, - "read_only": 1, - "read_only_depends_on": null, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "sort_options": 0, - "translatable": 1, - "unique": 0, - "width": null - }, - { - "_assign": null, - "_comments": null, - "_liked_by": null, - "_user_tags": null, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "collapsible_depends_on": null, - "columns": 0, - "creation": "2023-03-20 12:48:48.264679", - "default": null, - "depends_on": null, - "description": null, - "docstatus": 0, - "dt": "File", - "fetch_from": null, - "fetch_if_empty": 0, - "fieldname": "section_break_1lqhx", - "fieldtype": "Section Break", - "hidden": 0, - "hide_border": 0, - "hide_days": 0, - "hide_seconds": 0, - "idx": 25, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_preview": 0, - "in_standard_filter": 0, - "insert_after": "sharing_link", - "is_system_generated": 0, - "is_virtual": 0, - "label": null, - "length": 0, - "mandatory_depends_on": null, - "modified": "2023-07-06 02:04:01.389708", - "modified_by": "Administrator", - "module": null, - "name": "File-section_break_1lqhx", - "no_copy": 0, - "non_negative": 0, - "options": null, - "owner": "Administrator", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "print_width": null, - "read_only": 0, - "read_only_depends_on": null, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "sort_options": 0, - "translatable": 0, - "unique": 0, - "width": null - }, - { - "_assign": null, - "_comments": null, - "_liked_by": null, - "_user_tags": null, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "collapsible_depends_on": null, - "columns": 0, - "creation": "2023-03-20 12:48:48.360631", - "default": null, - "depends_on": null, - "description": null, - "docstatus": 0, - "dt": "File", - "fetch_from": null, - "fetch_if_empty": 0, - "fieldname": "file_association", - "fieldtype": "Table", - "hidden": 0, - "hide_border": 0, - "hide_days": 0, - "hide_seconds": 0, - "idx": 26, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_preview": 0, - "in_standard_filter": 0, - "insert_after": "section_break_1lqhx", - "is_system_generated": 0, - "is_virtual": 0, - "label": "", - "length": 0, - "mandatory_depends_on": null, - "modified": "2023-07-06 02:04:01.499473", - "modified_by": "Administrator", - "module": null, - "name": "File-file_association", - "no_copy": 0, - "non_negative": 0, - "options": "File Association", - "owner": "Administrator", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "print_width": null, - "read_only": 0, - "read_only_depends_on": null, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "sort_options": 0, - "translatable": 0, - "unique": 0, - "width": null - }, - { - "_assign": null, - "_comments": null, - "_liked_by": null, - "_user_tags": null, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "collapsible_depends_on": null, - "columns": 0, - "creation": "2023-07-06 05:39:34.115162", - "default": null, - "depends_on": null, - "description": null, - "docstatus": 0, - "dt": "File", - "fetch_from": null, - "fetch_if_empty": 0, - "fieldname": "section_break_1xyhd", - "fieldtype": "Section Break", - "hidden": 0, - "hide_border": 0, - "hide_days": 0, - "hide_seconds": 0, - "idx": 27, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_preview": 0, - "in_standard_filter": 0, - "insert_after": "file_association", - "is_system_generated": 0, - "is_virtual": 0, - "label": null, - "length": 0, - "mandatory_depends_on": null, - "modified": "2023-07-06 05:39:34.115162", - "modified_by": "Administrator", - "module": null, - "name": "File-section_break_1xyhd", - "no_copy": 0, - "non_negative": 0, - "options": null, - "owner": "Administrator", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "print_width": null, - "read_only": 0, - "read_only_depends_on": null, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "sort_options": 0, - "translatable": 0, - "unique": 0, - "width": null - }, - { - "_assign": null, - "_comments": null, - "_liked_by": null, - "_user_tags": null, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "collapsible_depends_on": null, - "columns": 0, - "creation": "2023-07-06 05:39:57.024056", - "default": null, - "depends_on": null, - "description": null, - "docstatus": 0, - "dt": "File", - "fetch_from": null, - "fetch_if_empty": 0, - "fieldname": "versions", - "fieldtype": "Table", - "hidden": 0, - "hide_border": 0, - "hide_days": 0, - "hide_seconds": 0, - "idx": 28, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_preview": 0, - "in_standard_filter": 0, - "insert_after": "section_break_1xyhd", - "is_system_generated": 0, - "is_virtual": 0, - "label": "", - "length": 0, - "mandatory_depends_on": null, - "modified": "2023-07-06 05:39:57.024056", - "modified_by": "Administrator", - "module": null, - "name": "File-versions", - "no_copy": 0, - "non_negative": 0, - "options": "File Version", - "owner": "Administrator", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "print_width": null, - "read_only": 0, - "read_only_depends_on": null, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "sort_options": 0, - "translatable": 0, - "unique": 0, - "width": null - } - ], - "custom_perms": [], - "doctype": "File", - "links": [], - "property_setters": [ - { - "_assign": null, - "_comments": null, - "_liked_by": null, - "_user_tags": null, - "creation": "2023-07-06 02:04:01.645547", - "default_value": null, - "doc_type": "File", - "docstatus": 0, - "doctype_or_field": "DocField", - "field_name": "preview", - "idx": 0, - "is_system_generated": 0, - "modified": "2023-07-06 02:04:01.645547", - "modified_by": "Administrator", - "module": null, - "name": "File-preview-collapsible_depends_on", - "owner": "Administrator", - "property": "collapsible_depends_on", - "property_type": "Data", - "row_name": null, - "value": "eval:true" - }, - { - "_assign": null, - "_comments": null, - "_liked_by": null, - "_user_tags": null, - "creation": "2023-07-06 02:04:01.636840", - "default_value": null, - "doc_type": "File", - "docstatus": 0, - "doctype_or_field": "DocField", - "field_name": "preview", - "idx": 0, - "is_system_generated": 0, - "modified": "2023-07-06 02:04:01.636840", - "modified_by": "Administrator", - "module": null, - "name": "File-preview-collapsible", - "owner": "Administrator", - "property": "collapsible", - "property_type": "Check", - "row_name": null, - "value": "1" - }, - { - "_assign": null, - "_comments": null, - "_liked_by": null, - "_user_tags": null, - "creation": "2023-07-06 02:04:01.627758", - "default_value": null, - "doc_type": "File", - "docstatus": 0, - "doctype_or_field": "DocField", - "field_name": "section_break_8", - "idx": 0, - "is_system_generated": 0, - "modified": "2023-07-06 02:04:01.627758", - "modified_by": "Administrator", - "module": null, - "name": "File-section_break_8-permlevel", - "owner": "Administrator", - "property": "permlevel", - "property_type": "Int", - "row_name": null, - "value": "1" - }, - { - "_assign": null, - "_comments": null, - "_liked_by": null, - "_user_tags": null, - "creation": "2023-07-06 02:04:01.616923", - "default_value": null, - "doc_type": "File", - "docstatus": 0, - "doctype_or_field": "DocField", - "field_name": "section_break_8", - "idx": 0, - "is_system_generated": 0, - "modified": "2023-07-06 02:04:01.616923", - "modified_by": "Administrator", - "module": null, - "name": "File-section_break_8-label", - "owner": "Administrator", - "property": "label", - "property_type": "Data", - "row_name": null, - "value": "Storage Details" - } - ], - "sync_on_migrate": 1 -} \ No newline at end of file + "custom_fields": [ + { + "_assign": null, + "_comments": null, + "_liked_by": null, + "_user_tags": null, + "allow_in_quick_entry": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "collapsible_depends_on": null, + "columns": 0, + "creation": "2023-02-03 12:03:51.204553", + "default": null, + "depends_on": null, + "description": null, + "docstatus": 0, + "dt": "File", + "fetch_from": null, + "fetch_if_empty": 0, + "fieldname": "s3_key", + "fieldtype": "Data", + "hidden": 0, + "hide_border": 0, + "hide_days": 0, + "hide_seconds": 0, + "idx": 23, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_preview": 0, + "in_standard_filter": 0, + "insert_after": "uploaded_to_google_drive", + "is_system_generated": 0, + "is_virtual": 0, + "label": "S3 Key", + "length": 0, + "mandatory_depends_on": null, + "modified": "2023-04-28 00:05:56.057223", + "modified_by": "Administrator", + "module": "Cloud Storage", + "name": "File-s3_key", + "no_copy": 0, + "non_negative": 0, + "options": null, + "owner": "Administrator", + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "print_width": null, + "read_only": 1, + "read_only_depends_on": null, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "translatable": 0, + "unique": 0, + "width": null + }, + { + "_assign": null, + "_comments": null, + "_liked_by": null, + "_user_tags": null, + "allow_in_quick_entry": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "collapsible_depends_on": null, + "columns": 0, + "creation": "2023-02-15 14:00:22.073075", + "default": null, + "depends_on": null, + "description": null, + "docstatus": 0, + "dt": "File", + "fetch_from": null, + "fetch_if_empty": 0, + "fieldname": "sharing_link", + "fieldtype": "Data", + "hidden": 0, + "hide_border": 0, + "hide_days": 0, + "hide_seconds": 0, + "idx": 24, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_preview": 0, + "in_standard_filter": 0, + "insert_after": "s3_key", + "is_system_generated": 0, + "is_virtual": 0, + "label": "Sharing Link", + "length": 0, + "mandatory_depends_on": null, + "modified": "2023-04-28 00:05:56.148197", + "modified_by": "Administrator", + "module": "Cloud Storage", + "name": "File-sharing_link", + "no_copy": 0, + "non_negative": 0, + "options": null, + "owner": "Administrator", + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "print_width": null, + "read_only": 1, + "read_only_depends_on": null, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "translatable": 1, + "unique": 0, + "width": null + }, + { + "_assign": null, + "_comments": null, + "_liked_by": null, + "_user_tags": null, + "allow_in_quick_entry": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "collapsible_depends_on": null, + "columns": 0, + "creation": "2023-03-20 12:48:48.264679", + "default": null, + "depends_on": null, + "description": null, + "docstatus": 0, + "dt": "File", + "fetch_from": null, + "fetch_if_empty": 0, + "fieldname": "section_break_1lqhx", + "fieldtype": "Section Break", + "hidden": 0, + "hide_border": 0, + "hide_days": 0, + "hide_seconds": 0, + "idx": 25, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_preview": 0, + "in_standard_filter": 0, + "insert_after": "sharing_link", + "is_system_generated": 0, + "is_virtual": 0, + "label": null, + "length": 0, + "mandatory_depends_on": null, + "modified": "2023-04-28 00:05:56.255829", + "modified_by": "Administrator", + "module": "Cloud Storage", + "name": "File-section_break_1lqhx", + "no_copy": 0, + "non_negative": 0, + "options": null, + "owner": "Administrator", + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "print_width": null, + "read_only": 0, + "read_only_depends_on": null, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "translatable": 0, + "unique": 0, + "width": null + }, + { + "_assign": null, + "_comments": null, + "_liked_by": null, + "_user_tags": null, + "allow_in_quick_entry": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "collapsible_depends_on": null, + "columns": 0, + "creation": "2023-03-20 12:48:48.360631", + "default": null, + "depends_on": null, + "description": null, + "docstatus": 0, + "dt": "File", + "fetch_from": null, + "fetch_if_empty": 0, + "fieldname": "file_association", + "fieldtype": "Table", + "hidden": 0, + "hide_border": 0, + "hide_days": 0, + "hide_seconds": 0, + "idx": 26, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_preview": 0, + "in_standard_filter": 0, + "insert_after": "section_break_1lqhx", + "is_system_generated": 0, + "is_virtual": 0, + "label": "", + "length": 0, + "mandatory_depends_on": null, + "modified": "2023-04-28 00:05:56.364321", + "modified_by": "Administrator", + "module": "Cloud Storage", + "name": "File-file_association", + "no_copy": 0, + "non_negative": 0, + "options": "File Association", + "owner": "Administrator", + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "print_width": null, + "read_only": 0, + "read_only_depends_on": null, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "translatable": 0, + "unique": 0, + "width": null + } + ], + "custom_perms": [], + "doctype": "File", + "links": [], + "property_setters": [ + { + "_assign": null, + "_comments": null, + "_liked_by": null, + "_user_tags": null, + "creation": "2023-04-28 00:05:56.507961", + "default_value": null, + "doc_type": "File", + "docstatus": 0, + "doctype_or_field": "DocField", + "field_name": "section_break_8", + "idx": 0, + "is_system_generated": 0, + "modified": "2023-04-28 00:05:56.507961", + "modified_by": "Administrator", + "module": "Cloud Storage", + "name": "File-section_break_8-label", + "owner": "Administrator", + "property": "label", + "property_type": "Data", + "row_name": null, + "value": "Storage Details" + }, + { + "_assign": null, + "_comments": null, + "_liked_by": null, + "_user_tags": null, + "creation": "2023-04-28 00:05:56.499487", + "default_value": null, + "doc_type": "File", + "docstatus": 0, + "doctype_or_field": "DocField", + "field_name": "section_break_8", + "idx": 0, + "is_system_generated": 0, + "modified": "2023-04-28 00:05:56.499487", + "modified_by": "Administrator", + "module": "Cloud Storage", + "name": "File-section_break_8-permlevel", + "owner": "Administrator", + "property": "permlevel", + "property_type": "Int", + "row_name": null, + "value": "1" + }, + { + "_assign": null, + "_comments": null, + "_liked_by": null, + "_user_tags": null, + "creation": "2023-04-28 00:05:56.491129", + "default_value": null, + "doc_type": "File", + "docstatus": 0, + "doctype_or_field": "DocField", + "field_name": "preview", + "idx": 0, + "is_system_generated": 0, + "modified": "2023-04-28 00:05:56.491129", + "modified_by": "Administrator", + "module": "Cloud Storage", + "name": "File-preview-collapsible", + "owner": "Administrator", + "property": "collapsible", + "property_type": "Check", + "row_name": null, + "value": "1" + }, + { + "_assign": null, + "_comments": null, + "_liked_by": null, + "_user_tags": null, + "creation": "2023-04-28 00:05:56.481095", + "default_value": null, + "doc_type": "File", + "docstatus": 0, + "doctype_or_field": "DocField", + "field_name": "preview", + "idx": 0, + "is_system_generated": 0, + "modified": "2023-04-28 00:05:56.481095", + "modified_by": "Administrator", + "module": "Cloud Storage", + "name": "File-preview-collapsible_depends_on", + "owner": "Administrator", + "property": "collapsible_depends_on", + "property_type": "Data", + "row_name": null, + "value": "eval:true" + } + ], + "sync_on_migrate": 1 +} diff --git a/cloud_storage/cloud_storage/doctype/file_association/file_association.json b/cloud_storage/cloud_storage/doctype/file_association/file_association.json index e1fc88c..bac4e0e 100644 --- a/cloud_storage/cloud_storage/doctype/file_association/file_association.json +++ b/cloud_storage/cloud_storage/doctype/file_association/file_association.json @@ -1,59 +1,54 @@ { - "actions": [], - "creation": "2023-03-20 12:48:05.086487", - "default_view": "List", - "doctype": "DocType", - "editable_grid": 1, - "engine": "InnoDB", - "field_order": [ - "link_doctype", - "link_name", - "user", - "timestamp" - ], - "fields": [ - { - "fieldname": "link_doctype", - "fieldtype": "Link", - "in_list_view": 1, - "label": "Document Type", - "options": "DocType", - "reqd": 1 - }, - { - "fieldname": "link_name", - "fieldtype": "Dynamic Link", - "in_list_view": 1, - "label": "Document Name", - "options": "link_doctype", - "reqd": 1 - }, - { - "fieldname": "timestamp", - "fieldtype": "Datetime", - "in_list_view": 1, - "label": "Timestamp", - "read_only": 1 - }, - { - "fieldname": "user", - "fieldtype": "Link", - "in_list_view": 1, - "label": "User", - "options": "User" - } - ], - "istable": 1, - "links": [], - "modified": "2023-09-27 23:42:57.356690", - "modified_by": "Administrator", - "module": "Cloud Storage", - "name": "File Association", - "owner": "Administrator", - "permissions": [], - "quick_entry": 1, - "sort_field": "modified", - "sort_order": "DESC", - "states": [], - "track_changes": 1 -} \ No newline at end of file + "actions": [], + "creation": "2023-03-20 12:48:05.086487", + "default_view": "List", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": ["link_doctype", "link_name", "user", "timestamp"], + "fields": [ + { + "fieldname": "link_doctype", + "fieldtype": "Link", + "in_list_view": 1, + "label": "Document Type", + "options": "DocType", + "reqd": 1 + }, + { + "fieldname": "link_name", + "fieldtype": "Dynamic Link", + "in_list_view": 1, + "label": "Document Name", + "options": "link_doctype", + "reqd": 1 + }, + { + "fieldname": "timestamp", + "fieldtype": "Datetime", + "in_list_view": 1, + "label": "Timestamp", + "read_only": 1 + }, + { + "fieldname": "user", + "fieldtype": "Link", + "in_list_view": 1, + "label": "User", + "options": "User" + } + ], + "istable": 1, + "links": [], + "modified": "2023-09-27 23:42:57.356690", + "modified_by": "Administrator", + "module": "Cloud Storage", + "name": "File Association", + "owner": "Administrator", + "permissions": [], + "quick_entry": 1, + "sort_field": "modified", + "sort_order": "DESC", + "states": [], + "track_changes": 1 +} diff --git a/cloud_storage/cloud_storage/doctype/file_version/file_version.json b/cloud_storage/cloud_storage/doctype/file_version/file_version.json index 0b57e6d..fdc6092 100644 --- a/cloud_storage/cloud_storage/doctype/file_version/file_version.json +++ b/cloud_storage/cloud_storage/doctype/file_version/file_version.json @@ -1,46 +1,42 @@ { - "actions": [], - "creation": "2023-07-06 05:38:35.891393", - "doctype": "DocType", - "editable_grid": 1, - "engine": "InnoDB", - "field_order": [ - "version", - "user", - "timestamp" - ], - "fields": [ - { - "fieldname": "version", - "fieldtype": "Data", - "in_list_view": 1, - "label": "Version", - "read_only": 1 - }, - { - "fieldname": "timestamp", - "fieldtype": "Datetime", - "in_list_view": 1, - "label": "Timestamp", - "read_only": 1 - }, - { - "fieldname": "user", - "fieldtype": "Link", - "in_list_view": 1, - "label": "User", - "options": "User" - } - ], - "istable": 1, - "links": [], - "modified": "2023-09-27 23:45:54.628246", - "modified_by": "Administrator", - "module": "Cloud Storage", - "name": "File Version", - "owner": "Administrator", - "permissions": [], - "sort_field": "modified", - "sort_order": "DESC", - "states": [] -} \ No newline at end of file + "actions": [], + "creation": "2023-07-06 05:38:35.891393", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": ["version", "user", "timestamp"], + "fields": [ + { + "fieldname": "version", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Version", + "read_only": 1 + }, + { + "fieldname": "timestamp", + "fieldtype": "Datetime", + "in_list_view": 1, + "label": "Timestamp", + "read_only": 1 + }, + { + "fieldname": "user", + "fieldtype": "Link", + "in_list_view": 1, + "label": "User", + "options": "User" + } + ], + "istable": 1, + "links": [], + "modified": "2023-09-27 23:45:54.628246", + "modified_by": "Administrator", + "module": "Cloud Storage", + "name": "File Version", + "owner": "Administrator", + "permissions": [], + "sort_field": "modified", + "sort_order": "DESC", + "states": [] +} diff --git a/cloud_storage/cloud_storage/overrides/file.py b/cloud_storage/cloud_storage/overrides/file.py index c51d616..039b645 100644 --- a/cloud_storage/cloud_storage/overrides/file.py +++ b/cloud_storage/cloud_storage/overrides/file.py @@ -27,9 +27,48 @@ class CustomFile(File): + @File.is_remote_file.getter + def is_remote_file(self) -> bool: + if self.file_url: # type: ignore + return self.file_url.startswith(URL_PREFIXES) # type: ignore + return not self.content + def has_permission(self, ptype: Optional[str] = None, user: Optional[str] = None) -> bool: return has_permission(self, ptype, user) + def validate(self) -> None: + self.associate_files() + if self.flags.cloud_storage or self.flags.ignore_file_validate: + return + if not self.is_remote_file: + super().validate() + else: + self.validate_file_url() + + def after_insert(self) -> File: + if self.attached_to_doctype and self.attached_to_name and not self.file_association: # type: ignore + if not self.content_hash and "/api/method/retrieve" in self.file_url: # type: ignore + associated_doc = frappe.get_value("File", {"file_url": self.file_url}, "name") # type: ignore + else: + associated_doc = frappe.get_value( + "File", + {"content_hash": self.content_hash, "name": ["!=", self.name], "is_folder": False}, # type: ignore + ) + if associated_doc: + self.db_set( + "file_url", "" + ) # this is done to prevent deletion of the remote file with the delete_file hook + rename_doc( + self.doctype, + self.name, + associated_doc, + merge=True, + force=True, + show_alert=False, + ignore_permissions=True, + # validate=False, + ) + def on_trash(self) -> None: user_roles = frappe.get_roles(frappe.session.user) if ( @@ -64,7 +103,7 @@ def associate_files( path = get_file_path(self, client.folder) self.file_url = FILE_URL.format(path=path) if not self.content_hash and "/api/method/retrieve" in self.file_url: # type: ignore - associated_doc = frappe.get_value("File", {"file_url": self.file_url}, "name") + associated_doc = frappe.get_value("File", {"file_url": self.file_url}, "name") # type: ignore else: associated_doc = frappe.get_value( "File", @@ -76,7 +115,7 @@ def associate_files( existing_file.attached_to_name = attached_to_name self.content_hash = existing_file.content_hash # if a File exists already where this association should be, we continue validating that File at this time - # the original File will then be removed in the after insert hook + # the original File will then be removed in the after insert hook (also avoids recursion issues) self = existing_file existing_attachment = list( @@ -98,39 +137,6 @@ def associate_files( if associated_doc and associated_doc != self.name: self.save() - def validate(self) -> None: - self.associate_files() - if self.flags.cloud_storage or self.flags.ignore_file_validate: - return - if not self.is_remote_file: - super().validate() - else: - self.validate_file_url() - - def after_insert(self) -> File: - if self.attached_to_doctype and self.attached_to_name and not self.file_association: # type: ignore - if not self.content_hash and "/api/method/retrieve" in self.file_url: - associated_doc = frappe.get_value("File", {"file_url": self.file_url}, "name") - else: - associated_doc = frappe.get_value( - "File", - {"content_hash": self.content_hash, "name": ["!=", self.name], "is_folder": False}, - "name", - ) - if associated_doc: - self.db_set( - "file_url", "" - ) # this is done to prevent deletion of the remote file with the delete_file hook - rename_doc( - self.doctype, - self.name, - associated_doc, - merge=True, - force=True, - show_alert=False, - ignore_permissions=True, - ) - def add_file_version(self, version_id): self.append( "versions", @@ -161,12 +167,6 @@ def remove_file_association(self, dt: str, dn: str) -> None: association.idx = idx self.save() - @property - def is_remote_file(self) -> bool: - if self.file_url: - return self.file_url.startswith(URL_PREFIXES) - return not self.content - def get_content(self) -> bytes: if self.is_folder: frappe.throw(_("Cannot get file contents of a Folder")) @@ -394,8 +394,6 @@ def upload_file(file: File) -> File: except Exception as e: frappe.log_error("File Upload Error", e) file.db_set("s3_key", path) - if not file.name: - file.save() return file @@ -486,7 +484,8 @@ def delete_file(file: File, **kwargs) -> File: except ClientError: frappe.throw(_("Access denied: Could not delete file")) except Exception as e: - frappe.log_error(str(e), "Cloud Storage Error: Cloud not delete file") + print(f"EXCEPTION: {e}") + frappe.log_error(str(e), "Cloud Storage Error: Could not delete file") return file diff --git a/cloud_storage/customize.py b/cloud_storage/customize.py index fd2fccd..56aae59 100644 --- a/cloud_storage/customize.py +++ b/cloud_storage/customize.py @@ -15,6 +15,8 @@ def load_customizations(): for file in files: customizations = json.loads(Path(file).read_text()) for field in customizations.get("custom_fields"): + if field.get("module") != "Cloud Storage": + continue existing_field = frappe.get_value("Custom Field", field.get("name")) custom_field = ( frappe.get_doc("Custom Field", field.get("name")) @@ -27,6 +29,8 @@ def load_customizations(): custom_field.flags.ignore_version = True custom_field.save() for prop in customizations.get("property_setters"): + if field.get("module") != "Cloud Storage": + continue property_setter = frappe.get_doc( { "name": prop.get("name"), diff --git a/cloud_storage/tests/conftest.py b/cloud_storage/tests/conftest.py index 9ad5950..7d8857e 100644 --- a/cloud_storage/tests/conftest.py +++ b/cloud_storage/tests/conftest.py @@ -1,59 +1,57 @@ -import os -from pathlib import Path -from unittest.mock import MagicMock, patch - -import frappe -import pytest -from frappe.defaults import * -from frappe.utils import get_bench_path - - -def _get_logger(*args, **kwargs): - from frappe.utils.logger import get_logger - - return get_logger( - module=None, - with_more_info=False, - allow_site=True, - filter=None, - max_size=100_000, - file_count=20, - stream_only=True, - ) - - -@pytest.fixture(scope="module") -def monkeymodule(): - with pytest.MonkeyPatch.context() as mp: - yield mp - - -@pytest.fixture(scope="session", autouse=True) -def db_instance(): - frappe.logger = _get_logger - - currentsite = "test_site" - sites = Path(get_bench_path()) / "sites" - if (sites / "currentsite.txt").is_file(): - currentsite = (sites / "currentsite.txt").read_text() - - frappe.init(site=currentsite, sites_path=sites) - frappe.connect() - frappe.db.commit = MagicMock() - yield frappe.db - - -@pytest.fixture(scope="module", autouse=True) -def patch_frappe_conf(monkeymodule): - monkeymodule.setattr( - "frappe.conf.cloud_storage_settings", - { - "access_key": "test", - "secret": "test_secret", - "region": "us-east-1", - "bucket": "test_bucket", - "endpoint_url": "https://test.imgainarys3.edu", - "expiration": 110, - "folder": "test_folder", - }, - ) +from pathlib import Path +from unittest.mock import MagicMock + +import frappe +import pytest +from frappe.utils import get_bench_path + + +def _get_logger(*args, **kwargs): + from frappe.utils.logger import get_logger + + return get_logger( + module=None, + with_more_info=False, + allow_site=True, + filter=None, + max_size=100_000, + file_count=20, + stream_only=True, + ) + + +@pytest.fixture(scope="module") +def monkeymodule(): + with pytest.MonkeyPatch.context() as mp: + yield mp + + +@pytest.fixture(scope="session", autouse=True) +def db_instance(): + frappe.logger = _get_logger + + currentsite = "test_site" + sites = Path(get_bench_path()) / "sites" + if (sites / "currentsite.txt").is_file(): + currentsite = (sites / "currentsite.txt").read_text() + + frappe.init(site=currentsite, sites_path=sites) + frappe.connect() + frappe.db.commit = MagicMock() + yield frappe.db + + +@pytest.fixture(scope="module", autouse=True) +def patch_frappe_conf(monkeymodule): + monkeymodule.setattr( + "frappe.conf.cloud_storage_settings", + { + "access_key": "test", + "secret": "test_secret", + "region": "us-east-1", + "bucket": "test_bucket", + "endpoint_url": "https://test.imgainarys3.edu", + "expiration": 110, + "folder": "test_folder", + }, + ) diff --git a/cloud_storage/tests/fixtures/aticonrust.svg b/cloud_storage/tests/fixtures/aticonrust.svg new file mode 100644 index 0000000..4418b08 --- /dev/null +++ b/cloud_storage/tests/fixtures/aticonrust.svg @@ -0,0 +1,64 @@ + + + + + + + + + + image/svg+xml + + + + + + + + + diff --git a/cloud_storage/tests/fixtures/atlogo_rust.svg b/cloud_storage/tests/fixtures/atlogo_rust.svg new file mode 100644 index 0000000..888c4ac --- /dev/null +++ b/cloud_storage/tests/fixtures/atlogo_rust.svg @@ -0,0 +1,256 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + diff --git a/cloud_storage/tests/test_file.py b/cloud_storage/tests/test_file.py index 7e28d6d..41b5312 100644 --- a/cloud_storage/tests/test_file.py +++ b/cloud_storage/tests/test_file.py @@ -1,175 +1,126 @@ -from unittest.mock import MagicMock, patch +from io import BytesIO +from pathlib import Path import frappe -from boto3.exceptions import S3UploadFailedError -from botocore.exceptions import ClientError -from frappe.tests.utils import FrappeTestCase - -from cloud_storage.cloud_storage.overrides.file import ( - CustomFile, - delete_file, - upload_file, - write_file, -) - - -class TestFile(FrappeTestCase): - @patch("cloud_storage.cloud_storage.overrides.file.upload_file") - @patch("cloud_storage.cloud_storage.overrides.file.strip_special_chars") - @patch("frappe.get_all") - @patch("frappe.conf") - def test_write_file(self, config, get_all, strip_chars, upload_file): - file = MagicMock() - - # test local fallback - config.cloud_storage_settings = None - write_file(file) - assert file.save_file_on_filesystem.call_count == 1 - - config.cloud_storage_settings = {"use_local": True} - write_file(file) - assert file.save_file_on_filesystem.call_count == 2 - - config.cloud_storage_settings = {"use_local": False} - file.attached_to_doctype = "Data Import" - write_file(file) - assert file.save_file_on_filesystem.call_count == 3 - - # test file upload with autoname - file.attached_to_doctype = None - file.name = None - file.file_name = "test_file.png" - strip_chars.return_value = "test_file.png" - upload_file.return_value = file - get_all.return_value = [] - write_file(file) - upload_file.assert_called_with(file) - - # test file upload without autoname - file.name = "test_file" - file.file_name = "test_file.png" - upload_file.return_value = file - write_file(file) - upload_file.assert_called_with(file) - - @patch("cloud_storage.cloud_storage.overrides.file.get_cloud_storage_client") - @patch("cloud_storage.cloud_storage.overrides.file.get_file_path") - def test_upload_file(self, file_path, client): - # setup file - file = MagicMock() - file.content_type = "image/jpeg" - - # test general exception - client.return_value.put_object.side_effect = TypeError - upload_file(file) - assert client.return_value.put_object.call_count == 1 - - # test upload errors - client.return_value.put_object.side_effect = S3UploadFailedError - with self.assertRaises(frappe.ValidationError): - upload_file(file) - assert client.return_value.put_object.call_count == 2 - - # test upload success - client.return_value.put_object.side_effect = True - file_path.return_value = "/path/to/s3/bucket/location" - upload_file(file) - assert client.return_value.put_object.call_count == 3 - - @patch("cloud_storage.cloud_storage.overrides.file.get_cloud_storage_client") - @patch("frappe.conf") - def test_delete_file(self, config, client): - file = MagicMock() - - # test local fallback - config.cloud_storage_settings = None - delete_file(file) - assert file.delete_file_from_filesystem.call_count == 1 - assert client.return_value.delete_object.call_count == 0 - - config.cloud_storage_settings = {"use_local": True} - delete_file(file) - assert file.delete_file_from_filesystem.call_count == 2 - assert client.return_value.delete_object.call_count == 0 - - # test skip folder deletion - config.cloud_storage_settings = {"use_local": False} - file.is_folder = True - delete_file(file) - assert client.return_value.delete_object.call_count == 0 - - # test skip file deletion from missing url or key param - file.is_folder = False - file.file_url = None - delete_file(file) - assert client.return_value.delete_object.call_count == 0 - - file.file_url = "/api/method/retrieve" - delete_file(file) - assert client.return_value.delete_object.call_count == 0 - - file.file_url = "/api/method/retrieve?key=" - delete_file(file) - assert client.return_value.delete_object.call_count == 0 - - # test general exception - file.file_url = "/api/method/retrieve?key=/path/to/s3/bucket/location" - client.return_value.delete_object.side_effect = TypeError - delete_file(file) - assert client.return_value.delete_object.call_count == 1 - - # test upload errors - client.return_value.delete_object.side_effect = ClientError( - error_response={"Error": {}}, operation_name="delete_file" - ) - with self.assertRaises(frappe.ValidationError): - delete_file(file) - assert client.return_value.delete_object.call_count == 2 - - # test upload success - client.bucket = "bucket" - client.return_value.delete_object.side_effect = True - delete_file(file) - assert client.return_value.delete_object.call_count == 3 - - @patch("cloud_storage.cloud_storage.overrides.file.has_user_permission") - @patch("frappe.has_permission") - @patch("frappe.get_doc") - def test_file_permission(self, get_doc, has_permission, has_user_permission): - # test file access for owner - file = CustomFile({"doctype": "File", "owner": "Administrator"}) - self.assertEqual(file.has_permission(), True) - self.assertEqual(file.has_permission(user="Administrator"), True) - - # test file access for non-owner user - has_permission.return_value = True - assert file.has_permission(user="Administrator") is True - assert file.has_permission(user="support@agritheory.dev") is True - has_permission.return_value = False - assert file.has_permission(user="Administrator") is True - assert file.has_permission(user="support@agritheory.dev") is False - - # test file access for attached doctypes - file = CustomFile( - { - "doctype": "File", - "owner": "Administrator", - "attached_to_doctype": "Sales Order", - "attached_to_name": "SO-0001", - } - ) - - # test file access for doc permissions - get_doc.return_value = MagicMock() - get_doc.return_value.has_permission.return_value = True - assert file.has_permission(user="Administrator") is True - assert file.has_permission(user="support@agritheory.dev") is True - - # test file access for custom user permissions - get_doc.return_value.has_permission.return_value = False - has_user_permission.return_value = True - assert file.has_permission(user="Administrator") is True - assert file.has_permission(user="support@agritheory.dev") is True - has_user_permission.return_value = False - assert file.has_permission(user="Administrator") is True - assert file.has_permission(user="support@agritheory.dev") is False +import pytest +from moto import mock_s3 +from werkzeug.datastructures import FileMultiDict + +from cloud_storage.cloud_storage.overrides.file import CustomFile, retrieve + + +@pytest.fixture +def example_file_record_0(): + return Path(__file__).parent / "fixtures" / "aticonrusthex.png" + + +@pytest.fixture +def example_file_record_1(): + return Path(__file__).parent / "fixtures" / "aticonrust.svg" + + +@pytest.fixture +def example_file_record_2(): + return Path(__file__).parent / "fixtures" / "atlogo_rust.svg" + + +@pytest.fixture +def get_cloud_storage_client_fixture(): + return frappe.call("cloud_storage.cloud_storage.overrides.file.get_cloud_storage_client") + + +# helper function +def create_upload_file(file_path: Path, **kwargs) -> CustomFile: + f = BytesIO(file_path.resolve().read_bytes()) + f.seek(0) + + # simulate a Frappe client -> server request file object using Werkzeug + files = FileMultiDict() + files.add_file("file", f, kwargs.get("file_name")) + + frappe.set_user("Administrator") + frappe.local.request = frappe._dict() + frappe.local.request.method = kwargs.get("method") or "POST" + frappe.local.request.files = files + frappe.local.form_dict = frappe._dict() + frappe.local.form_dict.is_private = False + frappe.local.form_dict.doctype = kwargs.get("doctype") or "User" + frappe.local.form_dict.docname = kwargs.get("docname") or "Administrator" + frappe.local.form_dict.fieldname = kwargs.get("fieldname") or None + frappe.local.form_dict.file_url = kwargs.get("file_url") or None + frappe.local.form_dict.folder = kwargs.get("folder") or "Home" + frappe.local.form_dict.file_name = kwargs.get("file_name") or None + frappe.local.form_dict.optimize = kwargs.get("optimize") or False + file = frappe.call("frappe.handler.upload_file") + return file + + +@mock_s3 +def test_config(get_cloud_storage_client_fixture): + c = get_cloud_storage_client_fixture + assert c.bucket == "test_bucket" + assert c.folder == "test_folder" + assert c.expiration == 110 + assert c._endpoint._endpoint_prefix == "s3" + assert c._endpoint.host == "https://test.imgainarys3.edu" + + +@mock_s3 +def test_upload_file(example_file_record_0): + frappe.set_user("Administrator") + file = create_upload_file(example_file_record_0, file_name="aticonrusthex.png") + assert frappe.db.exists("File", file.name) + assert file.attached_to_doctype == "User" + assert file.attached_to_name == "Administrator" + assert file.attached_to_field is None + assert file.folder == "Home" + assert file.file_name == "aticonrusthex.png" + assert file.content_hash is not None + assert ( + file.file_url == "/api/method/retrieve?key=test_folder/User/Administrator/aticonrusthex.png" + ) + assert file.is_private == 0 # makes the delete file test easier + assert file.s3_key is not None + assert len(file.file_association) == 1 + assert file.file_association[0].link_doctype == "User" + assert file.file_association[0].link_name == "Administrator" + + # Test manual association + file.append("file_association", {"link_doctype": "Module Def", "link_name": "Cloud Storage"}) + file.save() + assert len(file.file_association) == 2 + assert file.file_association[1].link_doctype == "Module Def" + assert file.file_association[1].link_name == "Cloud Storage" + + +@mock_s3 +def test_upload_file_with_multiple_association(example_file_record_1): + _file = create_upload_file(example_file_record_1, file_name="aticonrust.svg") + file = create_upload_file( + example_file_record_1, + doctype="Module Def", + docname="Automation", + file_name="aticonrust.svg", + ) + + _file.load_from_db() + assert frappe.db.exists("File", _file.name) is not None + assert frappe.db.exists("File", file.name) is None + assert len(_file.file_association) == 2 + assert _file.file_association[0].link_doctype == "User" + assert _file.file_association[0].link_name == "Administrator" + assert _file.file_association[1].link_doctype == "Module Def" + assert _file.file_association[1].link_name == "Automation" + + +@mock_s3 +def test_delete_file(example_file_record_2): + frappe.set_user("Administrator") + file = create_upload_file(example_file_record_2, file_name="atlogo_rust.svg") + s3_key = file.s3_key + file.delete() + + assert not frappe.db.exists("File", file.name) + + with pytest.raises(frappe.exceptions.DoesNotExistError): + retrieve(s3_key) diff --git a/cloud_storage/tests/test_file_association.py b/cloud_storage/tests/test_file_association.py deleted file mode 100644 index ec01b31..0000000 --- a/cloud_storage/tests/test_file_association.py +++ /dev/null @@ -1,88 +0,0 @@ -from io import BytesIO -from pathlib import Path - -import frappe -import pytest -from moto import mock_s3 - - -@pytest.fixture -def example_file_record_0(): - return Path(__file__).parent / "fixtures" / "aticonrusthex.png" - - -@pytest.fixture -def get_cloud_storage_client_fixture(): - return frappe.call("cloud_storage.cloud_storage.overrides.file.get_cloud_storage_client") - - -@mock_s3 -def test_config(get_cloud_storage_client_fixture): - c = get_cloud_storage_client_fixture - assert c.bucket == "test_bucket" - assert c.folder == "test_folder" - assert c.expiration == 110 - assert c._endpoint._endpoint_prefix == "s3" - assert c._endpoint.host == "https://test.imgainarys3.edu" - - -# helper function -def create_upload_file(file_path, **kwargs): - f = BytesIO(file_path.resolve().read_bytes()) - f.seek(0) - frappe.set_user("Administrator") - frappe.local.request = frappe._dict() - frappe.local.request.method = kwargs.get("method") or "POST" - frappe.local.request.files = f - frappe.local.form_dict = frappe._dict() - frappe.local.form_dict.is_private = True - frappe.local.form_dict.doctype = kwargs.get("doctype") or "User" - frappe.local.form_dict.docname = kwargs.get("docname") or "Administrator" - frappe.local.form_dict.fieldname = kwargs.get("fieldname") or "image" - frappe.local.form_dict.file_url = kwargs.get("file_url") or None - frappe.local.form_dict.folder = kwargs.get("folder") or "Home" - frappe.local.form_dict.file_name = kwargs.get("file_name") or "aticonrusthex.png" - frappe.local.form_dict.optimize = kwargs.get("optimize") or False - return frappe.call("frappe.handler.upload_file") - - -@mock_s3 -def test_upload_file(example_file_record_0): - file = create_upload_file(example_file_record_0) - assert frappe.db.exists("File", file.name) - assert file.attached_to_doctype == "User" - assert file.attached_to_name == "Administrator" - assert file.attached_to_field == "image" - assert file.folder == "Home" - assert file.file_name == "aticonrusthex.png" - assert file.content_hash is None - assert ( - file.file_url == "/api/method/retrieve?key=test_folder/User/Administrator/aticonrusthex.png" - ) - assert file.is_private == 1 - assert len(file.file_association) == 1 - assert file.file_association[0].link_doctype == "User" - assert file.file_association[0].link_name == "Administrator" - file.append("file_association", {"link_doctype": "Module Def", "link_name": "Cloud Storage"}) - file.save() - assert len(file.file_association) == 2 - assert file.file_association[1].link_doctype == "Module Def" - assert file.file_association[1].link_name == "Cloud Storage" - - -# @mock_s3 -# def test_upload_file_with_association(example_file_record_0): -# file = create_upload_file(example_file_record_0) -# second_association = create_upload_file(example_file_record_0, doctype='Module Def', docname="Cloud Storage") -# print(second_association.__dict__) -# assert frappe.db.exists('File', file.name) -# assert file.attached_to_doctype == 'User' -# assert file.attached_to_name == 'Administrator' -# assert file.attached_to_field == 'image' -# assert file.folder == 'Home' -# assert file.file_name == 'aticonrusthex.png' -# assert file.file_url == '/api/method/retrieve?key=test_folder/User/Administrator/aticonrusthex.png' -# assert file.is_private == 1 -# assert len(file.file_association) == 1 -# assert file.file_association[0].link_doctype == 'User' -# assert file.file_association[0].link_name == 'Administrator' diff --git a/package.json b/package.json index 01bcb31..8e75daf 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,6 @@ { "name": "@agritheory/cloud_storage", + "type": "module", "dependencies": { "madr": "^3.0.0" }, diff --git a/pyproject.toml b/pyproject.toml index 3abea04..ee3098d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,9 +10,10 @@ dynamic = ["version"] dependencies = [ "boto3==1.28.10", "python-magic~=0.4.27", - "pytest", - 'moto[s3]' + "pytest~=7.2.2", + "moto~=4.1.6" ] +moto = ["S3"] [build-system] requires = ["flit_core >=3.4,<4"]