feat: versioned files (#36)

* wip: versioning doctype * wip: file versioning * fix: remove print * wip: override upload dialog * feat: run name and content validations on adding file * feat: setup rename for name-conflicts * feat: show existing filenames for content hash conflicts * fix: error text on filename conflict * feat: override existing file instead of creating a new one * ci: update pre-commit config * test: fix write_file test * feat: add file association if content hash matches * fix: catch errors while stripping EXIF data from file * ci: update json-diff dependency * fix: allow error message translations * test: fix write_file test * fix: set values in database if a record exists already * test: fix write_file test * test: disable db logger in test * fix: avoid file size checks since that data may not exist * fix: allow skipping content hash conflicts * Revert "fix: allow skipping content hash conflicts" This reverts commit e731eb0. * fix: optimize content validation logic * fix: check file permission with full object * fix: file permission for orphaned files * style: ignore type errors * fix: handle file attachments from existing library * fix: version table indexes when associations are removed * fix: add metadata to version and file association tables * fix: allow keeping file name structure * fix: index numbering issues for version table --------- Co-authored-by: Rohan Bansal <[email protected]>
agritheory · Oct 17, 2023 · 331c1ef · 331c1ef
1 parent 38398f0
commit 331c1ef
Show file tree

Hide file tree

Showing 15 changed files with 1,575 additions and 147 deletions.
diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml
@@ -62,7 +62,7 @@ jobs:
 
       - name: Find JSON changes
         id: changed-json
-        uses: tj-actions/changed-files@v23.1
+        uses: tj-actions/changed-files@v37
         with:
           files: |
             **/*.json

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -7,8 +7,8 @@ repos:
     rev: v4.3.0
     hooks:
       - id: trailing-whitespace
-        files: 'frappe.*'
-        exclude: '.*json$|.*txt$|.*csv|.*md|.*svg'
+        files: "cloud_storage.*"
+        exclude: ".*json$|.*txt$|.*csv|.*md|.*svg"
       - id: check-yaml
       - id: no-commit-to-branch
         args: ['--branch', 'develop']
@@ -23,7 +23,7 @@ repos:
     rev: v2.34.0
     hooks:
       - id: pyupgrade
-        args: ['--py310-plus']
+        args: ['--py38-plus']
 
   - repo: https://github.com/frappe/black
     rev: 951ccf4d5bb0d692b457a5ebc4215d755618eb68
@@ -37,22 +37,28 @@ repos:
         types_or: [javascript]
         # Ignore any files that might contain jinja / bundles
         exclude: |
-          (?x)^(
-              frappe/public/dist/.*|
-              .*node_modules.*|
+            (?x)^(
               .*boilerplate.*|
-              frappe/www/website_script.js|
-              frappe/templates/includes/.*|
-              frappe/public/js/lib/.*
-          )$
+              .*node_modules.*|
+              cloud_storage/public/dist/.*|
+              cloud_storage/public/js/lib/.*|
+              cloud_storage/templates/includes/.*|
+              cloud_storage/www/website_script.js
+            )$
+
+
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.12.0
+    hooks:
+      - id: isort
 
   - repo: https://github.com/PyCQA/flake8
     rev: 5.0.4
     hooks:
       - id: flake8
-        additional_dependencies: ['flake8-bugbear']
+        additional_dependencies: ['flake8-bugbear',]
 
 ci:
-  autoupdate_schedule: weekly
-  skip: []
-  submodules: false
+    autoupdate_schedule: weekly
+    skip: []
+    submodules: false
diff --git a/cloud_storage/cloud_storage/custom/file.json b/cloud_storage/cloud_storage/custom/file.json
@@ -38,7 +38,7 @@
    "label": "S3 Key",
    "length": 0,
    "mandatory_depends_on": null,
-   "modified": "2023-04-28 00:05:56.057223",
+   "modified": "2023-07-06 02:04:01.131489",
    "modified_by": "Administrator",
    "module": null,
    "name": "File-s3_key",
@@ -56,6 +56,7 @@
    "report_hide": 0,
    "reqd": 0,
    "search_index": 0,
+   "sort_options": 0,
    "translatable": 0,
    "unique": 0,
    "width": null
@@ -98,7 +99,7 @@
    "label": "Sharing Link",
    "length": 0,
    "mandatory_depends_on": null,
-   "modified": "2023-04-28 00:05:56.148197",
+   "modified": "2023-07-06 02:04:01.272045",
    "modified_by": "Administrator",
    "module": null,
    "name": "File-sharing_link",
@@ -116,6 +117,7 @@
    "report_hide": 0,
    "reqd": 0,
    "search_index": 0,
+   "sort_options": 0,
    "translatable": 1,
    "unique": 0,
    "width": null
@@ -158,7 +160,7 @@
    "label": null,
    "length": 0,
    "mandatory_depends_on": null,
-   "modified": "2023-04-28 00:05:56.255829",
+   "modified": "2023-07-06 02:04:01.389708",
    "modified_by": "Administrator",
    "module": null,
    "name": "File-section_break_1lqhx",
@@ -176,6 +178,7 @@
    "report_hide": 0,
    "reqd": 0,
    "search_index": 0,
+   "sort_options": 0,
    "translatable": 0,
    "unique": 0,
    "width": null
@@ -218,7 +221,7 @@
    "label": "",
    "length": 0,
    "mandatory_depends_on": null,
-   "modified": "2023-04-28 00:05:56.364321",
+   "modified": "2023-07-06 02:04:01.499473",
    "modified_by": "Administrator",
    "module": null,
    "name": "File-file_association",
@@ -236,6 +239,129 @@
    "report_hide": 0,
    "reqd": 0,
    "search_index": 0,
+   "sort_options": 0,
+   "translatable": 0,
+   "unique": 0,
+   "width": null
+  },
+  {
+   "_assign": null,
+   "_comments": null,
+   "_liked_by": null,
+   "_user_tags": null,
+   "allow_in_quick_entry": 0,
+   "allow_on_submit": 0,
+   "bold": 0,
+   "collapsible": 0,
+   "collapsible_depends_on": null,
+   "columns": 0,
+   "creation": "2023-07-06 05:39:34.115162",
+   "default": null,
+   "depends_on": null,
+   "description": null,
+   "docstatus": 0,
+   "dt": "File",
+   "fetch_from": null,
+   "fetch_if_empty": 0,
+   "fieldname": "section_break_1xyhd",
+   "fieldtype": "Section Break",
+   "hidden": 0,
+   "hide_border": 0,
+   "hide_days": 0,
+   "hide_seconds": 0,
+   "idx": 27,
+   "ignore_user_permissions": 0,
+   "ignore_xss_filter": 0,
+   "in_global_search": 0,
+   "in_list_view": 0,
+   "in_preview": 0,
+   "in_standard_filter": 0,
+   "insert_after": "file_association",
+   "is_system_generated": 0,
+   "is_virtual": 0,
+   "label": null,
+   "length": 0,
+   "mandatory_depends_on": null,
+   "modified": "2023-07-06 05:39:34.115162",
+   "modified_by": "Administrator",
+   "module": null,
+   "name": "File-section_break_1xyhd",
+   "no_copy": 0,
+   "non_negative": 0,
+   "options": null,
+   "owner": "Administrator",
+   "permlevel": 0,
+   "precision": "",
+   "print_hide": 0,
+   "print_hide_if_no_value": 0,
+   "print_width": null,
+   "read_only": 0,
+   "read_only_depends_on": null,
+   "report_hide": 0,
+   "reqd": 0,
+   "search_index": 0,
+   "sort_options": 0,
+   "translatable": 0,
+   "unique": 0,
+   "width": null
+  },
+  {
+   "_assign": null,
+   "_comments": null,
+   "_liked_by": null,
+   "_user_tags": null,
+   "allow_in_quick_entry": 0,
+   "allow_on_submit": 0,
+   "bold": 0,
+   "collapsible": 0,
+   "collapsible_depends_on": null,
+   "columns": 0,
+   "creation": "2023-07-06 05:39:57.024056",
+   "default": null,
+   "depends_on": null,
+   "description": null,
+   "docstatus": 0,
+   "dt": "File",
+   "fetch_from": null,
+   "fetch_if_empty": 0,
+   "fieldname": "versions",
+   "fieldtype": "Table",
+   "hidden": 0,
+   "hide_border": 0,
+   "hide_days": 0,
+   "hide_seconds": 0,
+   "idx": 28,
+   "ignore_user_permissions": 0,
+   "ignore_xss_filter": 0,
+   "in_global_search": 0,
+   "in_list_view": 0,
+   "in_preview": 0,
+   "in_standard_filter": 0,
+   "insert_after": "section_break_1xyhd",
+   "is_system_generated": 0,
+   "is_virtual": 0,
+   "label": "",
+   "length": 0,
+   "mandatory_depends_on": null,
+   "modified": "2023-07-06 05:39:57.024056",
+   "modified_by": "Administrator",
+   "module": null,
+   "name": "File-versions",
+   "no_copy": 0,
+   "non_negative": 0,
+   "options": "File Version",
+   "owner": "Administrator",
+   "permlevel": 0,
+   "precision": "",
+   "print_hide": 0,
+   "print_hide_if_no_value": 0,
+   "print_width": null,
+   "read_only": 0,
+   "read_only_depends_on": null,
+   "report_hide": 0,
+   "reqd": 0,
+   "search_index": 0,
+   "sort_options": 0,
    "translatable": 0,
    "unique": 0,
    "width": null
@@ -250,44 +376,44 @@
    "_comments": null,
    "_liked_by": null,
    "_user_tags": null,
-   "creation": "2023-04-28 00:05:56.507961",
+   "creation": "2023-07-06 02:04:01.645547",
    "default_value": null,
    "doc_type": "File",
    "docstatus": 0,
    "doctype_or_field": "DocField",
-   "field_name": "section_break_8",
+   "field_name": "preview",
    "idx": 0,
    "is_system_generated": 0,
-   "modified": "2023-04-28 00:05:56.507961",
+   "modified": "2023-07-06 02:04:01.645547",
    "modified_by": "Administrator",
    "module": null,
-   "name": "File-section_break_8-label",
+   "name": "File-preview-collapsible_depends_on",
    "owner": "Administrator",
-   "property": "label",
+   "property": "collapsible_depends_on",
    "property_type": "Data",
    "row_name": null,
-   "value": "Storage Details"
+   "value": "eval:true"
   },
   {
    "_assign": null,
    "_comments": null,
    "_liked_by": null,
    "_user_tags": null,
-   "creation": "2023-04-28 00:05:56.499487",
+   "creation": "2023-07-06 02:04:01.636840",
    "default_value": null,
    "doc_type": "File",
    "docstatus": 0,
    "doctype_or_field": "DocField",
-   "field_name": "section_break_8",
+   "field_name": "preview",
    "idx": 0,
    "is_system_generated": 0,
-   "modified": "2023-04-28 00:05:56.499487",
+   "modified": "2023-07-06 02:04:01.636840",
    "modified_by": "Administrator",
    "module": null,
-   "name": "File-section_break_8-permlevel",
+   "name": "File-preview-collapsible",
    "owner": "Administrator",
-   "property": "permlevel",
-   "property_type": "Int",
+   "property": "collapsible",
+   "property_type": "Check",
    "row_name": null,
    "value": "1"
   },
@@ -296,21 +422,21 @@
    "_comments": null,
    "_liked_by": null,
    "_user_tags": null,
-   "creation": "2023-04-28 00:05:56.491129",
+   "creation": "2023-07-06 02:04:01.627758",
    "default_value": null,
    "doc_type": "File",
    "docstatus": 0,
    "doctype_or_field": "DocField",
-   "field_name": "preview",
+   "field_name": "section_break_8",
    "idx": 0,
    "is_system_generated": 0,
-   "modified": "2023-04-28 00:05:56.491129",
+   "modified": "2023-07-06 02:04:01.627758",
    "modified_by": "Administrator",
    "module": null,
-   "name": "File-preview-collapsible",
+   "name": "File-section_break_8-permlevel",
    "owner": "Administrator",
-   "property": "collapsible",
-   "property_type": "Check",
+   "property": "permlevel",
+   "property_type": "Int",
    "row_name": null,
    "value": "1"
   },
@@ -319,24 +445,24 @@
    "_comments": null,
    "_liked_by": null,
    "_user_tags": null,
-   "creation": "2023-04-28 00:05:56.481095",
+   "creation": "2023-07-06 02:04:01.616923",
    "default_value": null,
    "doc_type": "File",
    "docstatus": 0,
    "doctype_or_field": "DocField",
-   "field_name": "preview",
+   "field_name": "section_break_8",
    "idx": 0,
    "is_system_generated": 0,
-   "modified": "2023-04-28 00:05:56.481095",
+   "modified": "2023-07-06 02:04:01.616923",
    "modified_by": "Administrator",
    "module": null,
-   "name": "File-preview-collapsible_depends_on",
+   "name": "File-section_break_8-label",
    "owner": "Administrator",
-   "property": "collapsible_depends_on",
+   "property": "label",
    "property_type": "Data",
    "row_name": null,
-   "value": "eval:true"
+   "value": "Storage Details"
   }
  ],
  "sync_on_migrate": 1
-}
+}