enhance: add import edn graph option

Also fix lint and move other db import fns to db ns

scripts/src/logseq/tasks/dev/db_and_file_graphs.clj

@@ -118,8 +118,8 @@
                        (map str)
                        (into [;; e.g. block/properties :title
                               "block/properties :"
-                              ;; anything org mode except for org.babashka
-                              "org[^\\.]"
+                              ;; anything org mode except for org.babashka or urls like schema.org
+                              "[^\\.]org[^\\.]"
                               "#+BEGIN_"
                               "#+END_"
                               "pre-block"]))

src/main/frontend/components/imports.cljs

@@ -12,6 +12,7 @@
             [frontend.fs :as fs]
             [frontend.handler.file-based.import :as file-import-handler]
             [frontend.handler.db-based.editor :as db-editor-handler]
+            [frontend.handler.db-based.import :as db-import-handler]
             [frontend.handler.import :as import-handler]
             [frontend.handler.notification :as notification]
             [frontend.handler.repo :as repo-handler]
@@ -70,7 +71,7 @@
                           :error))))
 
 (defn- lsq-import-handler
-  [e & {:keys [sqlite? debug-transit? graph-name]}]
+  [e & {:keys [sqlite? debug-transit? graph-name db-edn?]}]
   (let [file      (first (array-seq (.-files (.-target e))))
         file-name (some-> (gobj/get file "name")
                           (string/lower-case))
@@ -91,7 +92,7 @@
             (set! (.-onload reader)
                   (fn []
                     (let [buffer (.-result ^js reader)]
-                      (import-handler/import-from-sqlite-db! buffer graph-name finished-cb)
+                      (db-import-handler/import-from-sqlite-db! buffer graph-name finished-cb)
                       (shui/dialog-close!))))
             (set! (.-onerror reader) (fn [e] (js/console.error e)))
             (set! (.-onabort reader) (fn [e]
@@ -99,7 +100,7 @@
                                        (js/console.error e)))
             (.readAsArrayBuffer reader file))))
 
-      debug-transit?
+      (or debug-transit? db-edn?)
       (let [graph-name (string/trim graph-name)]
         (cond
           (string/blank? graph-name)
@@ -112,7 +113,9 @@
           (do
             (state/set-state! :graph/importing :logseq)
             (let [reader (js/FileReader.)
-                  import-f import-handler/import-from-debug-transit!]
+                  import-f (if db-edn?
+                             db-import-handler/import-from-edn!
+                             db-import-handler/import-from-debug-transit!)]
               (set! (.-onload reader)
                     (fn [e]
                       (let [text (.. e -target -result)]
@@ -121,7 +124,9 @@
                          text
                          #(do
                             (state/set-state! :graph/importing nil)
-                            (finished-cb))))))
+                            (finished-cb)
+                            ;; graph input not closing
+                            (shui/dialog-close-all!))))))
               (.readAsText reader file)))))
 
       (or edn? json?)
@@ -449,7 +454,7 @@
    [importing?])
   [:<>])
 
-(rum/defc importer < rum/reactive
+(rum/defc ^:large-vars/cleanup-todo importer < rum/reactive
   [{:keys [query-params]}]
   (let [support-file-based? (config/local-file-based-graph? (state/get-current-repo))
         importing? (state/sub :graph/importing)]
@@ -497,8 +502,6 @@
              [:span.flex.flex-col
               [[:strong "Debug Transit"]
                [:small "Import debug transit file into a new DB graph"]]]
-             ;; Test form style changes
-             #_[:a.button {:on-click #(import-file-to-db-handler nil {:import-graph-fn js/alert})} "Open"]
              [:input.absolute.hidden
               {:id "import-debug-transit"
                :type "file"
@@ -506,11 +509,24 @@
                             (shui/dialog-open!
                              #(set-graph-name-dialog e {:debug-transit? true})))}]])
 
+          (when (or (util/electron?) util/web-platform?)
+            [:label.action-input.flex.items-center.mx-2.my-2
+             [:span.as-flex-center [:i (svg/logo 28)]]
+             [:span.flex.flex-col
+              [[:strong "EDN to DB graph"]
+               [:small "Import a DB graph's EDN export into a new DB graph"]]]
+             [:input.absolute.hidden
+              {:id "import-db-edn"
+               :type "file"
+               :on-change (fn [e]
+                            (shui/dialog-open!
+                             #(set-graph-name-dialog e {:db-edn? true})))}]])
+
           (when (and (util/electron?) support-file-based?)
             [:label.action-input.flex.items-center.mx-2.my-2
              [:span.as-flex-center [:i (svg/logo 28)]]
              [:span.flex.flex-col
-              [[:strong "EDN / JSON"]
+              [[:strong "EDN / JSON to plain text graph"]
                [:small (t :on-boarding/importing-lsq-desc)]]]
              [:input.absolute.hidden
               {:id "import-lsq"

src/main/frontend/handler/db_based/import.cljs

@@ -0,0 +1,100 @@
+(ns frontend.handler.db-based.import
+  (:require [clojure.edn :as edn]
+            [frontend.config :as config]
+            [frontend.db :as db]
+            [frontend.handler.notification :as notification]
+            [frontend.handler.repo :as repo-handler]
+            [frontend.persist-db :as persist-db]
+            [frontend.state :as state]
+            [logseq.db :as ldb]
+            [logseq.db.sqlite.export :as sqlite-export]
+            [logseq.db.sqlite.util :as sqlite-util]
+            [logseq.shui.ui :as shui]
+            [promesa.core :as p]))
+
+(defn import-from-sqlite-db!
+  [buffer bare-graph-name finished-ok-handler]
+  (let [graph (str config/db-version-prefix bare-graph-name)]
+    (->
+     (p/do!
+      (persist-db/<import-db graph buffer)
+      (state/add-repo! {:url graph})
+      (repo-handler/restore-and-setup-repo! graph {:import-type :sqlite-db})
+      (state/set-current-repo! graph)
+      (persist-db/<export-db graph {})
+      (db/transact! graph (sqlite-util/import-tx :sqlite-db))
+      (finished-ok-handler))
+     (p/catch
+      (fn [e]
+        (js/console.error e)
+        (notification/show!
+         (str (.-message e))
+         :error))))))
+
+(defn import-from-debug-transit!
+  [bare-graph-name raw finished-ok-handler]
+  (let [graph (str config/db-version-prefix bare-graph-name)
+        datoms (ldb/read-transit-str raw)]
+    (p/do!
+     (persist-db/<new graph {:import-type :debug-transit
+                             :datoms datoms})
+     (state/add-repo! {:url graph})
+     (repo-handler/restore-and-setup-repo! graph {:import-type :debug-transit})
+     (db/transact! graph (sqlite-util/import-tx :debug-transit))
+     (state/set-current-repo! graph)
+     (finished-ok-handler))))
+
+(defn- build-and-transact-import-edn
+  "Similar to db-based.export/import-submit"
+  [export-map]
+  (let [{:keys [init-tx block-props-tx misc-tx error] :as _txs}
+        (try
+          (sqlite-export/build-import export-map (db/get-db) {})
+          (catch :default e
+            (js/console.error "Import EDN error: " e)
+            {:error "An unexpected error occurred during import. See the javascript console for details."}))]
+    ;; (cljs.pprint/pprint _txs)
+    (if error
+      (notification/show! error :error)
+      (let [tx-meta {::sqlite-export/imported-data? true}
+            repo (state/get-current-repo)]
+        (-> (p/do
+              (db/transact! repo init-tx tx-meta)
+              (when (seq block-props-tx)
+                (db/transact! repo block-props-tx tx-meta))
+              (when (seq misc-tx)
+                (db/transact! repo misc-tx tx-meta)))
+            (p/catch (fn [e]
+                       (js/console.error "Import EDN error: " e)
+                       (notification/show! "An unexpected error occurred during import. See the javascript console for details." :error))))))))
+
+(defn import-from-edn!
+  "Creates a new DB graph and imports sqlite.build EDN file"
+  [bare-graph-name file-body finished-ok-handler]
+  (let [graph (str config/db-version-prefix bare-graph-name)
+        finished-error-handler
+        #(do
+           (state/set-state! :graph/importing nil)
+           (shui/dialog-close-all!))
+        edn-data (try
+                   (edn/read-string file-body)
+                   (catch :default e
+                     (js/console.error e)
+                     (notification/show! "The given EDN file is not valid EDN. Please fix and try again."
+                                         :error)
+                     (finished-error-handler)
+                     nil))]
+    (when (some? edn-data)
+      (-> (p/do!
+           (persist-db/<new graph {:import-type :edn})
+           (state/add-repo! {:url graph})
+           (repo-handler/restore-and-setup-repo! graph {:import-type :edn})
+           (state/set-current-repo! graph)
+           (build-and-transact-import-edn edn-data)
+           (finished-ok-handler))
+          (p/catch
+           (fn [e]
+             (js/console.error e)
+             (notification/show! (str "Unexpected error: " (.-message e))
+                                 :error)
+             (finished-error-handler)))))))

src/main/frontend/handler/import.cljs

@@ -5,7 +5,6 @@
             [clojure.edn :as edn]
             [clojure.string :as string]
             [clojure.walk :as walk]
-            [frontend.config :as config]
             [frontend.db :as db]
             [frontend.db.async :as db-async]
             [frontend.format.block :as block]
@@ -13,12 +12,8 @@
             [frontend.handler.editor :as editor]
             [frontend.handler.notification :as notification]
             [frontend.handler.page :as page-handler]
-            [frontend.handler.repo :as repo-handler]
-            [frontend.persist-db :as persist-db]
             [frontend.state :as state]
             [frontend.util :as util]
-            [logseq.db :as ldb]
-            [logseq.db.sqlite.util :as sqlite-util]
             [logseq.graph-parser.mldoc :as gp-mldoc]
             [logseq.graph-parser.whiteboard :as gp-whiteboard]
             [medley.core :as medley]
@@ -165,25 +160,6 @@
                            form))]
      (walk/postwalk tree-trans-fn tree-vec))))
 
-(defn import-from-sqlite-db!
-  [buffer bare-graph-name finished-ok-handler]
-  (let [graph (str config/db-version-prefix bare-graph-name)]
-    (->
-     (p/do!
-      (persist-db/<import-db graph buffer)
-      (state/add-repo! {:url graph})
-      (repo-handler/restore-and-setup-repo! graph {:import-type :sqlite-db})
-      (state/set-current-repo! graph)
-      (persist-db/<export-db graph {})
-      (db/transact! graph (sqlite-util/import-tx :sqlite-db))
-      (finished-ok-handler))
-     (p/catch
-      (fn [e]
-        (js/console.error e)
-        (notification/show!
-         (str (.-message e))
-         :error))))))
-
 (defn import-from-edn!
   [raw finished-ok-handler]
   (try
@@ -230,16 +206,3 @@
     (async/go
       (async/<! (import-from-tree! clj-data tree-vec-translate-json))
       (finished-ok-handler nil)))) ;; it was designed to accept a list of imported page names but now deprecated
-
-(defn import-from-debug-transit!
-  [bare-graph-name raw finished-ok-handler]
-  (let [graph (str config/db-version-prefix bare-graph-name)
-        datoms (ldb/read-transit-str raw)]
-    (p/do!
-     (persist-db/<new graph {:import-type :debug-transit
-                             :datoms datoms})
-     (state/add-repo! {:url graph})
-     (repo-handler/restore-and-setup-repo! graph {:import-type :debug-transit})
-     (db/transact! graph (sqlite-util/import-tx :debug-transit))
-     (state/set-current-repo! graph)
-     (finished-ok-handler nil))))