feat: import && export graph datoms transit

src/main/frontend/components/container.cljs

@@ -1056,7 +1056,7 @@
       [:a#download.hidden]
       [:a#download-as-edn-v2.hidden]
       [:a#download-as-json-v2.hidden]
-      [:a#download-as-json-debug.hidden]
+      [:a#download-as-transit-debug.hidden]
       [:a#download-as-sqlite-db.hidden]
       [:a#download-as-roam-json.hidden]
       [:a#download-as-html.hidden]

src/main/frontend/components/export.cljs

@@ -94,9 +94,9 @@
             (t :export-zip)]])
         (when db-based?
           [:div
-           [:a.font-medium {:on-click #(export/export-repo-as-debug-json! current-repo)}
-            "Export debug JSON"]
-           [:p.text-sm.opacity-70 "Any sensitive data will be removed in the exported json file, you can send it to us for debugging."]])
+           [:a.font-medium {:on-click #(export/export-repo-as-debug-transit! current-repo)}
+            "Export debug transit file"]
+           [:p.text-sm.opacity-70 "Any sensitive data will be removed in the exported transit file, you can send it to us for debugging."]])
 
         (when (util/electron?)
           [:div

src/main/frontend/components/imports.cljs

@@ -69,7 +69,7 @@
                           :error))))
 
 (defn- lsq-import-handler
-  [e & {:keys [sqlite? graph-name]}]
+  [e & {:keys [sqlite? debug-transit? graph-name]}]
   (let [file      (first (array-seq (.-files (.-target e))))
         file-name (some-> (gobj/get file "name")
                           (string/lower-case))
@@ -98,6 +98,31 @@
                                        (js/console.error e)))
             (.readAsArrayBuffer reader file))))
 
+      debug-transit?
+      (let [graph-name (string/trim graph-name)]
+        (cond
+          (string/blank? graph-name)
+          (notification/show! "Empty graph name." :error)
+
+          (repo-handler/graph-already-exists? graph-name)
+          (notification/show! "Please specify another name as another graph with this name already exists!" :error)
+
+          :else
+          (do
+            (state/set-state! :graph/importing :logseq)
+            (let [reader (js/FileReader.)
+                  import-f import-handler/import-from-debug-transit!]
+              (set! (.-onload reader)
+                    (fn [e]
+                      (let [text (.. e -target -result)]
+                        (import-f
+                         graph-name
+                         text
+                         #(do
+                            (state/set-state! :graph/importing nil)
+                            (finished-cb))))))
+              (.readAsText reader file)))))
+
       (or edn? json?)
       (do
         (state/set-state! :graph/importing :logseq)
@@ -142,11 +167,11 @@
 (rum/defcs set-graph-name-dialog
   < rum/reactive
   (rum/local "" ::input)
-  [state sqlite-input-e opts]
+  [state input-e opts]
   (let [*input (::input state)
         on-submit #(if (repo/invalid-graph-name? @*input)
                      (repo/invalid-graph-name-warning)
-                     (lsq-import-handler sqlite-input-e (assoc opts :graph-name @*input)))]
+                     (lsq-import-handler input-e (assoc opts :graph-name @*input)))]
     [:div.container
      [:div.sm:flex.sm:items-start
       [:div.mt-3.text-center.sm:mt-0.sm:text-left
@@ -184,74 +209,74 @@
                             ;; (js/console.log "[form] submit: " e (js->clj e))
                             (on-submit-fn (js->clj e :keywordize-keys true))
                             (shui/dialog-close!)))
-        [convert-all-tags-input set-convert-all-tags-input!] (rum/use-state true)]
+         [convert-all-tags-input set-convert-all-tags-input!] (rum/use-state true)]
 
      (shui/form-provider form-ctx
-       [:form
-        {:on-submit on-submit-valid}
-
-        (shui/form-field {:name "graph-name"}
-                         (fn [field error]
-                           (shui/form-item
-                            (shui/form-label "New graph name")
-                            (shui/form-control
-                             (shui/input (merge {:placeholder "Graph name"} field)))
-                            (when error
-                              (shui/form-description
-                               [:b.text-red-800 (:message error)])))))
-
-        (shui/form-field {:name "convert-all-tags?"}
-                         (fn [field]
-                           (shui/form-item
-                            {:class "pt-3 flex justify-start items-center space-x-3 space-y-0 my-3 pr-3"}
-                            (shui/form-label "Import all tags")
-                            (shui/form-control
-                             (shui/checkbox {:checked (:value field)
-                                             :on-checked-change (fn [e]
-                                                                  ((:onChange field) e)
-                                                                  (set-convert-all-tags-input! (not convert-all-tags-input)))})))))
-
-        (shui/form-field {:name "tag-classes"}
-                         (fn [field _error]
-                           (shui/form-item
-                            {:class "pt-3"}
-                            (shui/form-label "Import specific tags")
-                            (shui/form-control
-                             (shui/input (merge field
-                                                {:placeholder "tag 1, tag 2" :disabled convert-all-tags-input})))
-                            (shui/form-description "Tags are case insensitive"))))
-
-        (shui/form-field {:name "remove-inline-tags?"}
-                         (fn [field]
-                           (shui/form-item
-                            {:class "pt-3 flex justify-start items-center space-x-3 space-y-0 my-3 pr-3"}
-                            (shui/form-label "Remove inline tags")
-                            (shui/form-description "Default behavior for DB graphs")
-                            (shui/form-control
-                             (shui/checkbox {:checked (:value field)
-                                             :on-checked-change (:onChange field)})))))
-
-        (shui/form-field {:name "property-classes"}
-                         (fn [field _error]
-                           (shui/form-item
-                            {:class "pt-3"}
-                            (shui/form-label "Import additional tags from property values")
-                            (shui/form-control
-                             (shui/input (merge {:placeholder "e.g. type"} field)))
-                            (shui/form-description
-                             "Properties are case insensitive and separated by commas"))))
-
-        (shui/form-field {:name "property-parent-classes"}
-                         (fn [field _error]
-                           (shui/form-item
-                            {:class "pt-3"}
-                            (shui/form-label "Import tag parents from property values")
-                            (shui/form-control
-                             (shui/input (merge {:placeholder "e.g. parent"} field)))
-                            (shui/form-description
-                             "Properties are case insensitive and separated by commas"))))
-
-        (shui/button {:type "submit" :class "right-0 mt-3"} "Submit")]))])
+                         [:form
+                          {:on-submit on-submit-valid}
+
+                          (shui/form-field {:name "graph-name"}
+                                           (fn [field error]
+                                             (shui/form-item
+                                              (shui/form-label "New graph name")
+                                              (shui/form-control
+                                               (shui/input (merge {:placeholder "Graph name"} field)))
+                                              (when error
+                                                (shui/form-description
+                                                 [:b.text-red-800 (:message error)])))))
+
+                          (shui/form-field {:name "convert-all-tags?"}
+                                           (fn [field]
+                                             (shui/form-item
+                                              {:class "pt-3 flex justify-start items-center space-x-3 space-y-0 my-3 pr-3"}
+                                              (shui/form-label "Import all tags")
+                                              (shui/form-control
+                                               (shui/checkbox {:checked (:value field)
+                                                               :on-checked-change (fn [e]
+                                                                                    ((:onChange field) e)
+                                                                                    (set-convert-all-tags-input! (not convert-all-tags-input)))})))))
+
+                          (shui/form-field {:name "tag-classes"}
+                                           (fn [field _error]
+                                             (shui/form-item
+                                              {:class "pt-3"}
+                                              (shui/form-label "Import specific tags")
+                                              (shui/form-control
+                                               (shui/input (merge field
+                                                                  {:placeholder "tag 1, tag 2" :disabled convert-all-tags-input})))
+                                              (shui/form-description "Tags are case insensitive"))))
+
+                          (shui/form-field {:name "remove-inline-tags?"}
+                                           (fn [field]
+                                             (shui/form-item
+                                              {:class "pt-3 flex justify-start items-center space-x-3 space-y-0 my-3 pr-3"}
+                                              (shui/form-label "Remove inline tags")
+                                              (shui/form-description "Default behavior for DB graphs")
+                                              (shui/form-control
+                                               (shui/checkbox {:checked (:value field)
+                                                               :on-checked-change (:onChange field)})))))
+
+                          (shui/form-field {:name "property-classes"}
+                                           (fn [field _error]
+                                             (shui/form-item
+                                              {:class "pt-3"}
+                                              (shui/form-label "Import additional tags from property values")
+                                              (shui/form-control
+                                               (shui/input (merge {:placeholder "e.g. type"} field)))
+                                              (shui/form-description
+                                               "Properties are case insensitive and separated by commas"))))
+
+                          (shui/form-field {:name "property-parent-classes"}
+                                           (fn [field _error]
+                                             (shui/form-item
+                                              {:class "pt-3"}
+                                              (shui/form-label "Import tag parents from property values")
+                                              (shui/form-control
+                                               (shui/input (merge {:placeholder "e.g. parent"} field)))
+                                              (shui/form-description
+                                               "Properties are case insensitive and separated by commas"))))
+
+                          (shui/button {:type "submit" :class "right-0 mt-3"} "Submit")]))])
 
 (defn- counts-from-entities
   [entities]
@@ -416,14 +441,14 @@
 (rum/defc import-indicator
   [importing?]
   (rum/use-effect!
-    (fn []
-      (when (and importing? (not (shui-dialog/get-modal :import-indicator)))
-        (shui/dialog-open! indicator-progress
-          {:id :import-indicator
-           :content-props
-           {:onPointerDownOutside #(.preventDefault %)
-            :onOpenAutoFocus #(.preventDefault %)}})))
-    [importing?])
+   (fn []
+     (when (and importing? (not (shui-dialog/get-modal :import-indicator)))
+       (shui/dialog-open! indicator-progress
+                          {:id :import-indicator
+                           :content-props
+                           {:onPointerDownOutside #(.preventDefault %)
+                            :onOpenAutoFocus #(.preventDefault %)}})))
+   [importing?])
   [:<>])
 
 (rum/defc importer < rum/reactive
@@ -468,6 +493,21 @@
                                       (import-file-to-db-handler e {}))
                                     1000)}]])
 
+          (when (or (util/electron?) util/web-platform?)
+            [:label.action-input.flex.items-center.mx-2.my-2
+             [:span.as-flex-center [:i (svg/logo 28)]]
+             [:span.flex.flex-col
+              [[:strong "Debug Transit"]
+               [:small "Import debug transit file into a new DB graph"]]]
+             ;; Test form style changes
+             #_[:a.button {:on-click #(import-file-to-db-handler nil {:import-graph-fn js/alert})} "Open"]
+             [:input.absolute.hidden
+              {:id "import-debug-transit"
+               :type "file"
+               :on-change (fn [e]
+                            (shui/dialog-open!
+                             #(set-graph-name-dialog e {:debug-transit? true})))}]])
+
           (when (and (util/electron?) support-file-based?)
             [:label.action-input.flex.items-center.mx-2.my-2
              [:span.as-flex-center [:i (svg/logo 28)]]

src/main/frontend/handler/export.cljs

@@ -190,16 +190,13 @@
         (.setAttribute anchor "download" filename)
         (.click anchor)))))
 
-(defn export-repo-as-debug-json!
+(defn export-repo-as-debug-transit!
   [repo]
   (p/let [result (export-common-handler/<get-debug-datoms repo)
-          json-str (-> result
-                       bean/->js
-                       js/JSON.stringify)
-          filename (file-name (str repo "-debug-datoms") :json)
-          data-str (str "data:text/json;charset=utf-8,"
-                        (js/encodeURIComponent json-str))]
-    (when-let [anchor (gdom/getElement "download-as-json-debug")]
+          filename (file-name (str repo "-debug-datoms") :transit)
+          data-str (str "data:text/transit;charset=utf-8,"
+                        (js/encodeURIComponent result))]
+    (when-let [anchor (gdom/getElement "download-as-transit-debug")]
       (.setAttribute anchor "href" data-str)
       (.setAttribute anchor "download" filename)
       (.click anchor))))

src/main/frontend/handler/export/common.cljs

@@ -191,8 +191,7 @@
 (defn <get-debug-datoms
   [repo]
   (when-let [^object worker @db-browser/*worker]
-    (p/let [result (.get-debug-datoms worker repo)]
-      (ldb/read-transit-str result))))
+    (.get-debug-datoms worker repo)))
 
 (defn <get-all-page->content
   [repo]

src/main/frontend/handler/import.cljs

@@ -27,7 +27,8 @@
             [frontend.persist-db :as persist-db]
             [promesa.core :as p]
             [frontend.db.async :as db-async]
-            [logseq.db.sqlite.util :as sqlite-util]))
+            [logseq.db.sqlite.util :as sqlite-util]
+            [logseq.db :as ldb]))
 
 (defn index-files!
   "Create file structure, then parse into DB (client only)"
@@ -291,3 +292,15 @@
     (async/go
       (async/<! (import-from-tree! clj-data tree-vec-translate-json))
       (finished-ok-handler nil)))) ;; it was designed to accept a list of imported page names but now deprecated
+
+(defn import-from-debug-transit!
+  [bare-graph-name raw finished-ok-handler]
+  (let [graph (str config/db-version-prefix bare-graph-name)
+        datoms (ldb/read-transit-str raw)]
+    (p/do!
+     (persist-db/<new graph {:import-type "debug-transit"
+                             :datoms datoms})
+     (state/add-repo! {:url graph})
+     (repo-handler/restore-and-setup-repo! graph {:import-type "debug-transit"})
+     (state/set-current-repo! graph)
+     (finished-ok-handler nil))))

src/main/frontend/worker/db_worker.cljs

@@ -37,8 +37,7 @@
             [logseq.outliner.op :as outliner-op]
             [me.tonsky.persistent-sorted-set :as set :refer [BTSet]]
             [promesa.core :as p]
-            [shadow.cljs.modern :refer [defclass]]
-            [datascript.impl.entity :as de]))
+            [shadow.cljs.modern :refer [defclass]]))
 
 (defonce *sqlite worker-state/*sqlite)
 (defonce *sqlite-conns worker-state/*sqlite-conns)
@@ -290,7 +289,7 @@
   (.exec db "PRAGMA journal_mode=WAL"))
 
 (defn- create-or-open-db!
-  [repo {:keys [config import-type]}]
+  [repo {:keys [config import-type datoms]}]
   (when-not (worker-state/get-sqlite-conn repo)
     (p/let [[db search-db client-ops-db :as dbs] (get-dbs repo)
             storage (new-sqlite-storage repo {})
@@ -308,12 +307,19 @@
       (search/create-tables-and-triggers! search-db)
       (let [schema (sqlite-util/get-schema repo)
             conn (sqlite-common-db/get-storage-conn storage schema)
-            client-ops-conn (when-not @*publishing? (sqlite-common-db/get-storage-conn client-ops-storage client-op/schema-in-db))
-            initial-data-exists? (and (d/entity @conn :logseq.class/Root)
-                                      (= "db" (:kv/value (d/entity @conn :logseq.kv/db-type))))]
+            _ (when datoms
+                (let [data (map (fn [datom]
+                                  [:db/add (:e datom) (:a datom) (:v datom)]) datoms)]
+                  (d/transact! conn data {:initial-db? true})))
+            client-ops-conn (when-not @*publishing? (sqlite-common-db/get-storage-conn
+                                                     client-ops-storage
+                                                     client-op/schema-in-db))
+            initial-data-exists? (when (nil? datoms)
+                                   (and (d/entity @conn :logseq.class/Root)
+                                        (= "db" (:kv/value (d/entity @conn :logseq.kv/db-type)))))]
         (swap! *datascript-conns assoc repo conn)
         (swap! *client-ops-conns assoc repo client-ops-conn)
-        (when (and db-based? (not initial-data-exists?))
+        (when (and db-based? (not initial-data-exists?) (not datoms))
           (let [config (or config "")
                 initial-data (sqlite-create-graph/build-db-initial-data config
                                                                         (when import-type {:import-type import-type}))]
@@ -732,7 +738,8 @@
   (get-debug-datoms
    [this repo]
    (when-let [db (worker-state/get-sqlite-conn repo)]
-     (ldb/write-transit-str (worker-export/get-debug-datoms db))))
+     (let [conn (worker-state/get-datascript-conn repo)]
+       (ldb/write-transit-str (worker-export/get-debug-datoms conn db)))))
 
   (get-all-pages
    [this repo]

src/main/frontend/worker/export.cljs

@@ -53,7 +53,7 @@
                  (common-file/block->content repo db (:block/uuid e) {} {})])))))
 
 (defn get-debug-datoms
-  [^Object db]
+  [conn ^Object db]
   (some->> (.exec db #js {:sql "select content from kvs"
                           :rowMode "array"})
            bean/->clj
@@ -61,26 +61,10 @@
                      (let [result (sqlite-util/transit-read (first result))]
                        (when (map? result)
                          (:keys result)))))
-           (group-by first)
-           (mapcat (fn [[_id col]]
-                     (let [ident (some (fn [[_e a v _t]]
-                                         (when (= a :db/ident)
-                                           v)) col)
-                           journal (some (fn [[_e a v _t]]
-                                           (when (= a :block/journal-day)
-                                             v)) col)]
-                       (map
-                        (fn [[e a v t]]
-                          (cond
-                            (and (contains? #{:block/title :block/name} a)
-                                 (not (or ident journal)))
-                            [e a (str "debug " e) t]
-
-                            (= a :block/uuid)
-                            [e a (str v) t]
-
-                            :else
-                            [e a v t]))
-                        col))))
-           (distinct)
-           (sort-by first)))
+           (map (fn [[e a v t]]
+                  (if (and (contains? #{:block/title :block/name} a)
+                           (let [entity (d/entity @conn e)]
+                             (and (not (:db/ident entity))
+                                  (not (ldb/journal? entity)))))
+                    (d/datom e a (str "debug " e) t)
+                    (d/datom e a v t))))))