Skip to content

Commit

Permalink
feat: import && export graph datoms transit
Browse files Browse the repository at this point in the history
  • Loading branch information
@tiensonqin
tiensonqin committed Dec 29, 2024
1 parent 23b97bd commit 665207d
Show file tree
Hide file tree
Showing 8 changed files with 164 additions and 124 deletions.
2 changes: 1 addition & 1 deletion src/main/frontend/components/container.cljs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1056,7 +1056,7 @@
[:a#download.hidden]
[:a#download-as-edn-v2.hidden]
[:a#download-as-json-v2.hidden]
[:a#download-as-json-debug.hidden]
[:a#download-as-transit-debug.hidden]
[:a#download-as-sqlite-db.hidden]
[:a#download-as-roam-json.hidden]
[:a#download-as-html.hidden]
Expand Down
6 changes: 3 additions & 3 deletions src/main/frontend/components/export.cljs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,9 +94,9 @@
(t :export-zip)]])
(when db-based?
[:div
[:a.font-medium {:on-click #(export/export-repo-as-debug-json! current-repo)}
"Export debug JSON"]
[:p.text-sm.opacity-70 "Any sensitive data will be removed in the exported json file, you can send it to us for debugging."]])
[:a.font-medium {:on-click #(export/export-repo-as-debug-transit! current-repo)}
"Export debug transit file"]
[:p.text-sm.opacity-70 "Any sensitive data will be removed in the exported transit file, you can send it to us for debugging."]])

(when (util/electron?)
[:div
Expand Down
194 changes: 117 additions & 77 deletions src/main/frontend/components/imports.cljs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@
:error))))

(defn- lsq-import-handler
[e & {:keys [sqlite? graph-name]}]
[e & {:keys [sqlite? debug-transit? graph-name]}]
(let [file (first (array-seq (.-files (.-target e))))
file-name (some-> (gobj/get file "name")
(string/lower-case))
Expand Down Expand Up @@ -98,6 +98,31 @@
(js/console.error e)))
(.readAsArrayBuffer reader file))))

debug-transit?
(let [graph-name (string/trim graph-name)]
(cond
(string/blank? graph-name)
(notification/show! "Empty graph name." :error)

(repo-handler/graph-already-exists? graph-name)
(notification/show! "Please specify another name as another graph with this name already exists!" :error)

:else
(do
(state/set-state! :graph/importing :logseq)
(let [reader (js/FileReader.)
import-f import-handler/import-from-debug-transit!]
(set! (.-onload reader)
(fn [e]
(let [text (.. e -target -result)]
(import-f
graph-name
text
#(do
(state/set-state! :graph/importing nil)
(finished-cb))))))
(.readAsText reader file)))))

(or edn? json?)
(do
(state/set-state! :graph/importing :logseq)
Expand Down Expand Up @@ -142,11 +167,11 @@
(rum/defcs set-graph-name-dialog
< rum/reactive
(rum/local "" ::input)
[state sqlite-input-e opts]
[state input-e opts]
(let [*input (::input state)
on-submit #(if (repo/invalid-graph-name? @*input)
(repo/invalid-graph-name-warning)
(lsq-import-handler sqlite-input-e (assoc opts :graph-name @*input)))]
(lsq-import-handler input-e (assoc opts :graph-name @*input)))]
[:div.container
[:div.sm:flex.sm:items-start
[:div.mt-3.text-center.sm:mt-0.sm:text-left
Expand Down Expand Up @@ -184,74 +209,74 @@
;; (js/console.log "[form] submit: " e (js->clj e))
(on-submit-fn (js->clj e :keywordize-keys true))
(shui/dialog-close!)))
[convert-all-tags-input set-convert-all-tags-input!] (rum/use-state true)]
[convert-all-tags-input set-convert-all-tags-input!] (rum/use-state true)]

(shui/form-provider form-ctx
[:form
{:on-submit on-submit-valid}

(shui/form-field {:name "graph-name"}
(fn [field error]
(shui/form-item
(shui/form-label "New graph name")
(shui/form-control
(shui/input (merge {:placeholder "Graph name"} field)))
(when error
(shui/form-description
[:b.text-red-800 (:message error)])))))

(shui/form-field {:name "convert-all-tags?"}
(fn [field]
(shui/form-item
{:class "pt-3 flex justify-start items-center space-x-3 space-y-0 my-3 pr-3"}
(shui/form-label "Import all tags")
(shui/form-control
(shui/checkbox {:checked (:value field)
:on-checked-change (fn [e]
((:onChange field) e)
(set-convert-all-tags-input! (not convert-all-tags-input)))})))))

(shui/form-field {:name "tag-classes"}
(fn [field _error]
(shui/form-item
{:class "pt-3"}
(shui/form-label "Import specific tags")
(shui/form-control
(shui/input (merge field
{:placeholder "tag 1, tag 2" :disabled convert-all-tags-input})))
(shui/form-description "Tags are case insensitive"))))

(shui/form-field {:name "remove-inline-tags?"}
(fn [field]
(shui/form-item
{:class "pt-3 flex justify-start items-center space-x-3 space-y-0 my-3 pr-3"}
(shui/form-label "Remove inline tags")
(shui/form-description "Default behavior for DB graphs")
(shui/form-control
(shui/checkbox {:checked (:value field)
:on-checked-change (:onChange field)})))))

(shui/form-field {:name "property-classes"}
(fn [field _error]
(shui/form-item
{:class "pt-3"}
(shui/form-label "Import additional tags from property values")
(shui/form-control
(shui/input (merge {:placeholder "e.g. type"} field)))
(shui/form-description
"Properties are case insensitive and separated by commas"))))

(shui/form-field {:name "property-parent-classes"}
(fn [field _error]
(shui/form-item
{:class "pt-3"}
(shui/form-label "Import tag parents from property values")
(shui/form-control
(shui/input (merge {:placeholder "e.g. parent"} field)))
(shui/form-description
"Properties are case insensitive and separated by commas"))))

(shui/button {:type "submit" :class "right-0 mt-3"} "Submit")]))])
[:form
{:on-submit on-submit-valid}

(shui/form-field {:name "graph-name"}
(fn [field error]
(shui/form-item
(shui/form-label "New graph name")
(shui/form-control
(shui/input (merge {:placeholder "Graph name"} field)))
(when error
(shui/form-description
[:b.text-red-800 (:message error)])))))

(shui/form-field {:name "convert-all-tags?"}
(fn [field]
(shui/form-item
{:class "pt-3 flex justify-start items-center space-x-3 space-y-0 my-3 pr-3"}
(shui/form-label "Import all tags")
(shui/form-control
(shui/checkbox {:checked (:value field)
:on-checked-change (fn [e]
((:onChange field) e)
(set-convert-all-tags-input! (not convert-all-tags-input)))})))))

(shui/form-field {:name "tag-classes"}
(fn [field _error]
(shui/form-item
{:class "pt-3"}
(shui/form-label "Import specific tags")
(shui/form-control
(shui/input (merge field
{:placeholder "tag 1, tag 2" :disabled convert-all-tags-input})))
(shui/form-description "Tags are case insensitive"))))

(shui/form-field {:name "remove-inline-tags?"}
(fn [field]
(shui/form-item
{:class "pt-3 flex justify-start items-center space-x-3 space-y-0 my-3 pr-3"}
(shui/form-label "Remove inline tags")
(shui/form-description "Default behavior for DB graphs")
(shui/form-control
(shui/checkbox {:checked (:value field)
:on-checked-change (:onChange field)})))))

(shui/form-field {:name "property-classes"}
(fn [field _error]
(shui/form-item
{:class "pt-3"}
(shui/form-label "Import additional tags from property values")
(shui/form-control
(shui/input (merge {:placeholder "e.g. type"} field)))
(shui/form-description
"Properties are case insensitive and separated by commas"))))

(shui/form-field {:name "property-parent-classes"}
(fn [field _error]
(shui/form-item
{:class "pt-3"}
(shui/form-label "Import tag parents from property values")
(shui/form-control
(shui/input (merge {:placeholder "e.g. parent"} field)))
(shui/form-description
"Properties are case insensitive and separated by commas"))))

(shui/button {:type "submit" :class "right-0 mt-3"} "Submit")]))])

(defn- counts-from-entities
[entities]
Expand Down Expand Up @@ -416,14 +441,14 @@
(rum/defc import-indicator
[importing?]
(rum/use-effect!
(fn []
(when (and importing? (not (shui-dialog/get-modal :import-indicator)))
(shui/dialog-open! indicator-progress
{:id :import-indicator
:content-props
{:onPointerDownOutside #(.preventDefault %)
:onOpenAutoFocus #(.preventDefault %)}})))
[importing?])
(fn []
(when (and importing? (not (shui-dialog/get-modal :import-indicator)))
(shui/dialog-open! indicator-progress
{:id :import-indicator
:content-props
{:onPointerDownOutside #(.preventDefault %)
:onOpenAutoFocus #(.preventDefault %)}})))
[importing?])
[:<>])

(rum/defc importer < rum/reactive
Expand Down Expand Up @@ -468,6 +493,21 @@
(import-file-to-db-handler e {}))
1000)}]])

(when (or (util/electron?) util/web-platform?)
[:label.action-input.flex.items-center.mx-2.my-2
[:span.as-flex-center [:i (svg/logo 28)]]
[:span.flex.flex-col
[[:strong "Debug Transit"]
[:small "Import debug transit file into a new DB graph"]]]
;; Test form style changes
#_[:a.button {:on-click #(import-file-to-db-handler nil {:import-graph-fn js/alert})} "Open"]
[:input.absolute.hidden
{:id "import-debug-transit"
:type "file"
:on-change (fn [e]
(shui/dialog-open!
#(set-graph-name-dialog e {:debug-transit? true})))}]])

(when (and (util/electron?) support-file-based?)
[:label.action-input.flex.items-center.mx-2.my-2
[:span.as-flex-center [:i (svg/logo 28)]]
Expand Down
13 changes: 5 additions & 8 deletions src/main/frontend/handler/export.cljs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -190,16 +190,13 @@
(.setAttribute anchor "download" filename)
(.click anchor)))))

(defn export-repo-as-debug-json!
(defn export-repo-as-debug-transit!
[repo]
(p/let [result (export-common-handler/<get-debug-datoms repo)
json-str (-> result
bean/->js
js/JSON.stringify)
filename (file-name (str repo "-debug-datoms") :json)
data-str (str "data:text/json;charset=utf-8,"
(js/encodeURIComponent json-str))]
(when-let [anchor (gdom/getElement "download-as-json-debug")]
filename (file-name (str repo "-debug-datoms") :transit)
data-str (str "data:text/transit;charset=utf-8,"
(js/encodeURIComponent result))]
(when-let [anchor (gdom/getElement "download-as-transit-debug")]
(.setAttribute anchor "href" data-str)
(.setAttribute anchor "download" filename)
(.click anchor))))
Expand Down
3 changes: 1 addition & 2 deletions src/main/frontend/handler/export/common.cljs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -191,8 +191,7 @@
(defn <get-debug-datoms
[repo]
(when-let [^object worker @db-browser/*worker]
(p/let [result (.get-debug-datoms worker repo)]
(ldb/read-transit-str result))))
(.get-debug-datoms worker repo)))

(defn <get-all-page->content
[repo]
Expand Down
15 changes: 14 additions & 1 deletion src/main/frontend/handler/import.cljs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,8 @@
[frontend.persist-db :as persist-db]
[promesa.core :as p]
[frontend.db.async :as db-async]
[logseq.db.sqlite.util :as sqlite-util]))
[logseq.db.sqlite.util :as sqlite-util]
[logseq.db :as ldb]))

(defn index-files!
"Create file structure, then parse into DB (client only)"
Expand Down Expand Up @@ -291,3 +292,15 @@
(async/go
(async/<! (import-from-tree! clj-data tree-vec-translate-json))
(finished-ok-handler nil)))) ;; it was designed to accept a list of imported page names but now deprecated

(defn import-from-debug-transit!
[bare-graph-name raw finished-ok-handler]
(let [graph (str config/db-version-prefix bare-graph-name)
datoms (ldb/read-transit-str raw)]
(p/do!
(persist-db/<new graph {:import-type "debug-transit"
:datoms datoms})
(state/add-repo! {:url graph})
(repo-handler/restore-and-setup-repo! graph {:import-type "debug-transit"})
(state/set-current-repo! graph)
(finished-ok-handler nil))))
23 changes: 15 additions & 8 deletions src/main/frontend/worker/db_worker.cljs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,7 @@
[logseq.outliner.op :as outliner-op]
[me.tonsky.persistent-sorted-set :as set :refer [BTSet]]
[promesa.core :as p]
[shadow.cljs.modern :refer [defclass]]
[datascript.impl.entity :as de]))
[shadow.cljs.modern :refer [defclass]]))

(defonce *sqlite worker-state/*sqlite)
(defonce *sqlite-conns worker-state/*sqlite-conns)
Expand Down Expand Up @@ -290,7 +289,7 @@
(.exec db "PRAGMA journal_mode=WAL"))

(defn- create-or-open-db!
[repo {:keys [config import-type]}]
[repo {:keys [config import-type datoms]}]
(when-not (worker-state/get-sqlite-conn repo)
(p/let [[db search-db client-ops-db :as dbs] (get-dbs repo)
storage (new-sqlite-storage repo {})
Expand All @@ -308,12 +307,19 @@
(search/create-tables-and-triggers! search-db)
(let [schema (sqlite-util/get-schema repo)
conn (sqlite-common-db/get-storage-conn storage schema)
client-ops-conn (when-not @*publishing? (sqlite-common-db/get-storage-conn client-ops-storage client-op/schema-in-db))
initial-data-exists? (and (d/entity @conn :logseq.class/Root)
(= "db" (:kv/value (d/entity @conn :logseq.kv/db-type))))]
_ (when datoms
(let [data (map (fn [datom]
[:db/add (:e datom) (:a datom) (:v datom)]) datoms)]
(d/transact! conn data {:initial-db? true})))
client-ops-conn (when-not @*publishing? (sqlite-common-db/get-storage-conn
client-ops-storage
client-op/schema-in-db))
initial-data-exists? (when (nil? datoms)
(and (d/entity @conn :logseq.class/Root)
(= "db" (:kv/value (d/entity @conn :logseq.kv/db-type)))))]
(swap! *datascript-conns assoc repo conn)
(swap! *client-ops-conns assoc repo client-ops-conn)
(when (and db-based? (not initial-data-exists?))
(when (and db-based? (not initial-data-exists?) (not datoms))
(let [config (or config "")
initial-data (sqlite-create-graph/build-db-initial-data config
(when import-type {:import-type import-type}))]
Expand Down Expand Up @@ -732,7 +738,8 @@
(get-debug-datoms
[this repo]
(when-let [db (worker-state/get-sqlite-conn repo)]
(ldb/write-transit-str (worker-export/get-debug-datoms db))))
(let [conn (worker-state/get-datascript-conn repo)]
(ldb/write-transit-str (worker-export/get-debug-datoms conn db)))))

(get-all-pages
[this repo]
Expand Down
Loading

0 comments on commit 665207d

Please sign in to comment.