feat: import from logseq edn

feat: import edn with provided uuid

feat: overwrite page uuid; use properties in content

feat: error handling for importing

feat: support json import

chore: fix lint by splitting setup ui

deps/graph-parser/src/logseq/graph_parser/block.cljs

@@ -237,6 +237,10 @@
 (defn page-name->map
   "Create a page's map structure given a original page name (string).
    map as input is supported for legacy compatibility.
+   with-id?: if true, assign uuid to the map structure.
+    if the page entity already exists, no-op.
+    else, if with-id? is a uuid, the uuid is used.
+    otherwise, generate a uuid.
    with-timestamp?: assign timestampes to the map structure.
     Useful when creating new pages from references or namespaces,
     as there's no chance to introduce timestamps via editing in page"
@@ -253,9 +257,10 @@
        {:block/name page-name
         :block/original-name original-page-name}
        (when with-id?
-         (if page-entity
-           {:block/uuid (:block/uuid page-entity)}
-           {:block/uuid (d/squuid)}))
+         (let [new-uuid (cond page-entity      (:block/uuid page-entity)
+                              (uuid? with-id?) with-id?
+                              :else            (d/squuid))]
+           {:block/uuid new-uuid}))
        (when namespace?
          (let [namespace (first (gp-util/split-last "/" original-page-name))]
            (when-not (string/blank? namespace)

src/main/frontend/components/onboarding/setups.cljs

@@ -127,54 +127,124 @@
               [:small.opacity-50 label]]]))]]])))
 
 (defonce *roam-importing? (atom nil))
+(defonce *lsq-importing? (atom nil))
 (defonce *opml-importing? (atom nil))
 (defonce *opml-imported-pages (atom nil))
 
+(defn- finished-cb
+  []
+  (notification/show! "Import finished!" :success)
+  (route-handler/redirect-to-home!))
+
+(defn- roam-import-handler
+  [e]
+  (let [file      (first (array-seq (.-files (.-target e))))
+        file-name (gobj/get file "name")]
+    (if (string/ends-with? file-name ".json")
+      (do
+        (reset! *roam-importing? true)
+        (let [reader (js/FileReader.)]
+          (set! (.-onload reader)
+                (fn [e]
+                  (let [text (.. e -target -result)]
+                    (external-handler/import-from-roam-json! text
+                                                             #(do (reset! *roam-importing? false) (finished-cb))))))
+          (.readAsText reader file)))
+      (notification/show! "Please choose a JSON file."
+                          :error))))
+
+(defn- lsq-import-handler
+  [e]
+  (let [file      (first (array-seq (.-files (.-target e))))
+        file-name (gobj/get file "name")]
+    (cond (string/ends-with? file-name ".edn")
+          (do
+            (reset! *lsq-importing? true)
+            (let [reader (js/FileReader.)]
+              (set! (.-onload reader)
+                    (fn [e]
+                      (let [text (.. e -target -result)]
+                        (external-handler/import-from-edn! text
+                                                           #(do (reset! *lsq-importing? false) (finished-cb))))))
+              (.readAsText reader file)))
+
+          (string/ends-with? file-name ".json")
+          (do
+            (reset! *lsq-importing? true)
+            (let [reader (js/FileReader.)]
+              (set! (.-onload reader)
+                    (fn [e]
+                      (let [text (.. e -target -result)]
+                        (external-handler/import-from-json! text
+                                                            #(do (reset! *lsq-importing? false) (finished-cb))))))
+              (.readAsText reader file)))
+
+          :else
+          (notification/show! "Please choose an EDN or a JSON file."
+                              :error))))
+
+(defn- opml-import-handler
+  [e]
+  (let [file      (first (array-seq (.-files (.-target e))))
+        file-name (gobj/get file "name")]
+    (if (string/ends-with? file-name ".opml")
+      (do
+        (reset! *opml-importing? true)
+        (let [reader (js/FileReader.)]
+          (set! (.-onload reader)
+                (fn [e]
+                  (let [text (.. e -target -result)]
+                    (external-handler/import-from-opml! text
+                                                        (fn [pages]
+                                                          (reset! *opml-imported-pages pages)
+                                                          (reset! *opml-importing? false)
+                                                          (finished-cb))))))
+          (.readAsText reader file)))
+      (notification/show! "Please choose a OPML file."
+                          :error))))
+
 (rum/defc importer < rum/reactive
   [{:keys [query-params]}]
   (let [roam-importing? (rum/react *roam-importing?)
+        lsq-importing?  (rum/react *lsq-importing?)
         opml-importing? (rum/react *opml-importing?)
-        finished-cb     (fn []
-                          (notification/show! "Finished!" :success)
-                          (route-handler/redirect-to-home!))]
+        importing?      (or roam-importing? lsq-importing? opml-importing?)]
 
     (setups-container
      :importer
      [:article.flex.flex-col.items-center.importer.py-16.px-8
       [:section.c.text-center
        [:h1 "Do you already have notes that you want to import?"]
-       [:h2 "If they are in a JSON or Markdown format Logseq can work with them."]]
+       [:h2 "If they are in a JSON, EDN or Markdown format Logseq can work with them."]]
       [:section.d.md:flex
        [:label.action-input.flex.items-center.mx-2.my-2
-        {:disabled (or roam-importing? opml-importing?)}
+        {:disabled importing?}
         [:span.as-flex-center [:i (svg/roam-research 28)]]
         [:div.flex.flex-col
          (if roam-importing?
            (ui/loading "Importing ...")
-           [
-            [:strong "RoamResearch"]
+           [[:strong "RoamResearch"]
             [:small "Import a JSON Export of your Roam graph"]])]
         [:input.absolute.hidden
          {:id        "import-roam"
           :type      "file"
-          :on-change (fn [e]
-                       (let [file      (first (array-seq (.-files (.-target e))))
-                             file-name (gobj/get file "name")]
-                         (if (string/ends-with? file-name ".json")
-                           (do
-                             (reset! *roam-importing? true)
-                             (let [reader (js/FileReader.)]
-                               (set! (.-onload reader)
-                                     (fn [e]
-                                       (let [text (.. e -target -result)]
-                                         (external-handler/import-from-roam-json! text
-                                                                                  #(do (reset! *roam-importing? false) (finished-cb))))))
-                               (.readAsText reader file)))
-                           (notification/show! "Please choose a JSON file."
-                                               :error))))}]]
+          :on-change roam-import-handler}]]
+
+       [:label.action-input.flex.items-center.mx-2.my-2
+        {:disabled importing?}
+        [:span.as-flex-center [:i (svg/logo 28)]]
+        [:span.flex.flex-col
+         (if lsq-importing?
+           (ui/loading "Importing ...")
+           [[:strong "EDN / JSON"]
+            [:small "Import an EDN or a JSON Export of your Logseq graph"]])]
+        [:input.absolute.hidden
+         {:id        "import-lsq"
+          :type      "file"
+          :on-change lsq-import-handler}]]
 
        [:label.action-input.flex.items-center.mx-2.my-2
-        {:disabled (or roam-importing? opml-importing?)}
+        {:disabled importing?}
         [:span.as-flex-center (ui/icon "sitemap" {:style {:fontSize "26px"}})]
         [:span.flex.flex-col
          (if opml-importing?
@@ -185,24 +255,7 @@
         [:input.absolute.hidden
          {:id        "import-opml"
           :type      "file"
-          :on-change (fn [e]
-                       (let [file      (first (array-seq (.-files (.-target e))))
-                             file-name (gobj/get file "name")]
-                         (if (string/ends-with? file-name ".opml")
-                           (do
-                             (reset! *opml-importing? true)
-                             (let [reader (js/FileReader.)]
-                               (set! (.-onload reader)
-                                     (fn [e]
-                                       (let [text (.. e -target -result)]
-                                         (external-handler/import-from-opml! text
-                                                                             (fn [pages]
-                                                                               (reset! *opml-imported-pages pages)
-                                                                               (reset! *opml-importing? false)
-                                                                               (finished-cb))))))
-                               (.readAsText reader file)))
-                           (notification/show! "Please choose a OPML file."
-                                               :error))))}]]]
+          :on-change opml-import-handler}]]]
 
       (when (= "picker" (:from query-params))
         [:section.e

src/main/frontend/db.cljs

@@ -41,7 +41,7 @@
   delete-file-blocks! delete-page-blocks delete-files delete-pages-by-files
   filter-only-public-pages-and-blocks get-all-block-contents get-all-tagged-pages
   get-all-templates get-block-and-children get-block-by-uuid get-block-children sort-by-left
-  get-block-parent get-block-parents parents-collapsed? get-block-referenced-blocks
+  get-block-parent get-block-parents parents-collapsed? get-block-referenced-blocks get-all-referenced-blocks-uuid
   get-block-children-ids get-block-immediate-children get-block-page
   get-custom-css get-date-scheduled-or-deadlines
   get-file-blocks get-file-last-modified-at get-file get-file-page get-file-page-id file-exists?
@@ -158,13 +158,13 @@
                 (assoc option
                        :listen-handler listen-and-persist!))))
 
-(defn restore-graph!
-  "Restore db from serialized db cache, and swap into the current db status"
-  [repo]
+(defn restore-graph-from-text!
+  "Swap db string into the current db status
+   stored: the text to restore from"
+  [repo stored]
   (p/let [db-name (datascript-db repo)
           db-conn (d/create-conn db-schema/schema)
           _ (swap! conns assoc db-name db-conn)
-          stored (db-persist/get-serialized-graph db-name)
           _ (when stored
               (let [stored-db (try (string->db stored)
                                    (catch js/Error _e
@@ -178,6 +178,13 @@
                 (conn/reset-conn! db-conn db)))]
     (d/transact! db-conn [{:schema/version db-schema/version}])))
 
+(defn restore-graph!
+  "Restore db from serialized db cache"
+  [repo]
+  (p/let [db-name (datascript-db repo)
+          stored (db-persist/get-serialized-graph db-name)]
+    (restore-graph-from-text! repo stored)))
+
 (defn restore!
   [{:keys [repos]} _old-db-schema restore-config-handler]
   (let [repo (or (state/get-current-repo) (:url (first repos)))]

src/main/frontend/db/model.cljs

@@ -71,6 +71,11 @@
        react
        first))))
 
+(defn get-original-name
+  [page-entity]
+  (or (:block/original-name page-entity)
+      (:block/name page-entity)))
+
 (defn get-tag-pages
   [repo tag-name]
   (when tag-name
@@ -1364,6 +1369,18 @@
          (reset! blocks-count-cache n)
          n)))))
 
+(defn get-all-referenced-blocks-uuid
+  "Get all uuids of blocks with any back link exists."
+  []
+  (when-let [db (conn/get-db)]
+    (->> (d/datoms db :avet :block/uuid)
+         (map :v)
+         (map (fn [id]
+                (let [e (db-utils/entity [:block/uuid id])]
+                  (when (pos-int? (count (:block/_refs e)))
+                    id))))
+         (remove nil?))))
+
 ;; block/uuid and block/content
 (defn get-all-block-contents
   []

src/main/frontend/handler/editor.cljs

@@ -942,14 +942,16 @@
               (state/set-edit-content! input-id new-content)
               (save-block-if-changed! block new-content))))))))
 
-(defn- set-blocks-id!
+(defn set-blocks-id!
+  "Persist block uuid to file if not exists"
   [block-ids]
   (let [block-ids (remove nil? block-ids)
         col (map (fn [block-id]
-                   (let [block (db/entity [:block/uuid block-id])]
+                   (when-let [block (db/entity [:block/uuid block-id])]
                      (when-not (:block/pre-block? block)
                        [block-id :id (str block-id)])))
-                 block-ids)]
+                 block-ids)
+        col (remove nil? col)]
     (batch-set-block-property! col)))
 
 (defn copy-block-ref!
@@ -1915,10 +1917,13 @@
    0))
 
 (defn paste-blocks
+  "Given a vec of blocks, insert them into the target page.
+   keep-uuid?: if true, keep the uuid provided in the block structure."
   [blocks {:keys [content-update-fn
                   exclude-properties
                   target-block
-                  sibling?]
+                  sibling?
+                  keep-uuid?]
            :or {exclude-properties []}}]
   (let [editing-block (when-let [editing-block (state/get-edit-block)]
                         (some-> (db/pull (:db/id editing-block))
@@ -1945,10 +1950,11 @@
         (let [format (or (:block/format target-block) (state/get-preferred-format))
               blocks' (map (fn [block]
                              (paste-block-cleanup block page exclude-properties format content-update-fn))
-                        blocks)
+                           blocks)
               result (outliner-core/insert-blocks! blocks' target-block {:sibling? sibling?
                                                                          :outliner-op :paste
-                                                                         :replace-empty-target? true})]
+                                                                         :replace-empty-target? true
+                                                                         :keep-uuid? keep-uuid?})]
           (edit-last-block-after-inserted! result))))))
 
 (defn- block-tree->blocks
@@ -1965,18 +1971,24 @@
                 (assert fst-block "fst-block shouldn't be nil")
                 (assoc fst-block :block/level (:block/level block)))))))
 
-(defn insert-block-tree-after-target
+(defn insert-block-tree
   "`tree-vec`: a vector of blocks.
-   Block element: {:content :properties :children [block-1, block-2, ...]}"
-  [target-block-id sibling? tree-vec format]
+   A block element: {:content :properties :children [block-1, block-2, ...]}"
+  [tree-vec format {:keys [target-block] :as opts}]
   (let [blocks (block-tree->blocks tree-vec format)
-        target-block (db/pull target-block-id)
         page-id (:db/id (:block/page target-block))
         blocks (gp-block/with-parent-and-left page-id blocks)]
     (paste-blocks
      blocks
-     {:target-block target-block
-      :sibling? sibling?})))
+     opts)))
+
+(defn insert-block-tree-after-target
+  "`tree-vec`: a vector of blocks.
+   A block element: {:content :properties :children [block-1, block-2, ...]}"
+  [target-block-id sibling? tree-vec format]
+  (insert-block-tree tree-vec format
+                     {:target-block (db/pull target-block-id)
+                      :sibling?     sibling?}))
 
 (defn insert-template!
   ([element-id db-id]

src/main/frontend/handler/external.cljs

@@ -1,5 +1,7 @@
 (ns frontend.handler.external
-  (:require [frontend.external :as external]
+  (:require [clojure.edn :as edn]
+            [clojure.walk :as walk]
+            [frontend.external :as external]
             [frontend.handler.file :as file-handler]
             [frontend.handler.repo :as repo-handler]
             [frontend.state :as state]
@@ -13,9 +15,11 @@
             [logseq.graph-parser.date-time-util :as date-time-util]
             [frontend.handler.page :as page]
             [frontend.handler.editor :as editor]
+            [frontend.handler.notification :as notification]
             [frontend.util :as util]))
 
 (defn index-files!
+  "Create file structure, then parse into DB (client only)"
   [repo files finish-handler]
   (let [titles (->> files
                     (map :title)
@@ -95,3 +99,118 @@
          {:target-block target-block
           :sibling? sibling?})
         (finished-ok-handler [page-name])))))
+
+(defn create-page-with-exported-tree!
+  "Create page from the per page object generated in `export-repo-as-edn-v2!`
+   Return page-name (title)
+   Extension to `insert-block-tree-after-target`
+   :id       - page's uuid
+   :title    - page's title (original name)
+   :children - tree
+   "
+  [{:keys [id title children] :as tree}]
+  (let [has-children? (seq children)
+        page-format (some-> tree (:children) (first) (:format))]
+    (try (page/create! title {:redirect?  false
+                                  :format     page-format
+                                  :uuid       id})
+         (catch js/Error e
+           (notification/show! (str "Error happens when creating page " title ":\n"
+                                    e
+                                    "\nSkipped and continue the remaining import.") :error)))
+    (when has-children?
+      (let [page-block (db/entity [:block/name (util/page-name-sanity-lc title)])]
+        ;; Missing support for per block format (or deprecated?)
+        (try (editor/insert-block-tree children page-format
+                                       {:target-block page-block
+                                        :sibling?     true
+                                        :keep-uuid?   true})
+             (catch js/Error e
+               (notification/show! (str "Error happens when creating block content of page " title "\n"
+                                        e
+                                        "\nSkipped and continue the remaining import.") :error))))))
+  title)
+
+(defn- pre-transact-uuids
+  "Collect all uuids from page trees and write them to the db before hand."
+  [pages]
+  (let [uuids (map (fn [block]
+                     {:block/uuid (:uuid block)})
+                   (mapcat #(tree-seq map? :children %)
+                           pages))]
+    (db/transact! uuids)
+    pages))
+
+(defn- import-from-tree!
+  "Not rely on file system - backend compatible.
+   tree-translator-fn: translate exported tree structure to the desired tree for import"
+  [data tree-translator-fn]
+  (when-let [_repo (state/get-current-repo)]
+    (try (->> (:blocks data)
+              (map tree-translator-fn)
+              (pre-transact-uuids)
+              (mapv create-page-with-exported-tree!))
+         (editor/set-blocks-id! (db/get-all-referenced-blocks-uuid))
+         (catch js/Error e
+           (notification/show! (str "Error happens when importing:\n" e) :error)))))
+
+(defn tree-vec-translate-edn
+  "Actions to do for loading edn tree structure.
+   1) Removes namespace `:block/` from all levels of the `tree-vec`
+   2) Rename all :block/page-name to :title
+   3) Rename all :block/id to :uuid
+   4) Dissoc all :properties"
+  ([tree-vec]
+   (let [rm-kw-ns-fn #(-> %
+                          str
+                          (string/replace ":block/page-name" ":block/title")
+                          (string/replace ":block/id" ":block/uuid")
+                          (string/replace ":block/" "")
+                          keyword)
+         transform-map (fn [form]
+                         (if (map? form)
+                           ;; build a new map with the same keys but without the namespace
+                           (reduce-kv (fn [acc k v]
+                                        (if (not= :block/properties k)
+                                          (assoc acc (rm-kw-ns-fn k) v)
+                                          acc)) {} form)
+                           form))]
+     (walk/postwalk transform-map tree-vec))))
+
+(defn import-from-edn!
+  [raw finished-ok-handler]
+  (import-from-tree! (edn/read-string raw) tree-vec-translate-edn)
+  (finished-ok-handler nil)) ;; it was designed to accept a list of imported page names but now deprecated
+
+(defn tree-vec-translate-json
+  "Actions to do for loading json tree structure.
+   1) Rename all :id to :uuid
+   2) Rename all :page-name to :title
+   3) Dissoc all :properties
+   4) Rename all :format \"markdown\" to :format `:markdown`"
+  ([tree-vec]
+   (let [rm-kw-ns-fn #(-> %
+                          str
+                          (string/replace ":page-name" ":title")
+                          (string/replace ":id" ":uuid")
+                          (string/replace #"^:" "")
+                          keyword)
+         transform-map (fn [form]
+                         (if (map? form)
+                           (reduce-kv (fn [acc k v]
+                                        (if (not= :properties k)
+                                          (let [k (rm-kw-ns-fn k)
+                                                v (if (= k :format) (keyword v) v)]
+                                            (assoc acc k v))
+                                          acc)) {} form)
+                           form))
+         _ (prn tree-vec)
+         _ (prn (walk/postwalk transform-map tree-vec))]
+     (walk/postwalk transform-map tree-vec))))
+
+(defn import-from-json!
+  [raw finished-ok-handler]
+  (let [json     (js/JSON.parse raw)
+        clj-data (js->clj json :keywordize-keys true)]
+    (import-from-tree! clj-data tree-vec-translate-json))
+  (finished-ok-handler nil)) ;; it was designed to accept a list of imported page names but now deprecated

src/main/frontend/handler/graph.cljs

@@ -90,8 +90,7 @@
             namespaces (db/get-all-namespace-relation repo)
             tags (set (map second tagged-pages))
             full-pages (db/get-all-pages repo)
-            get-original-name (fn [p] (or (:block/original-name p) (:block/name p)))
-            all-pages (map get-original-name full-pages)
+            all-pages (map db/get-original-name full-pages)
             page-name->original-name (zipmap (map :block/name full-pages) all-pages)
             pages-after-journal-filter (if-not journal?
                                          (remove :block/journal? full-pages)
@@ -164,9 +163,7 @@
                        (distinct))
             nodes (build-nodes dark? page links (set tags) nodes namespaces)
             full-pages (db/get-all-pages repo)
-            get-original-name (fn [p] (or (:block/original-name p)
-                                         (:block/name p)))
-            all-pages (map get-original-name full-pages)
+            all-pages (map db/get-original-name full-pages)
             page-name->original-name (zipmap (map :block/name full-pages) all-pages)]
         (normalize-page-name
          {:nodes nodes

src/main/frontend/handler/page.cljs

@@ -116,27 +116,35 @@
         [page]))))
 
 (defn create!
+  "Create page.
+   :redirect?           - when true, redirect to the created page, otherwise return sanitized page name.
+   :split-namespace?    - when true, split hierarchical namespace into levels.
+   :create-first-block? - when true, create an empty block if the page is empty.
+   :uuid                - when set, use this uuid instead of generating a new one."
   ([title]
    (create! title {}))
-  ([title {:keys [redirect? create-first-block? format properties split-namespace? journal?]
+  ([title {:keys [redirect? create-first-block? format properties split-namespace? journal? uuid]
            :or   {redirect?           true
                   create-first-block? true
                   format              nil
                   properties          nil
-                  split-namespace?    true}}]
-   (let [title (string/trim title)
-         title (gp-util/remove-boundary-slashes title)
-         page-name (util/page-name-sanity-lc title)
-         repo (state/get-current-repo)]
+                  split-namespace?    true
+                  uuid                nil}}]
+   (let [title      (string/trim title)
+         title      (gp-util/remove-boundary-slashes title)
+         page-name  (util/page-name-sanity-lc title)
+         repo       (state/get-current-repo)
+         with-uuid? (if (uuid? uuid) uuid true)] ;; FIXME: prettier validation
      (when (db/page-empty? repo page-name)
        (let [pages    (if split-namespace?
                         (gp-util/split-namespace-pages title)
                         [title])
              format   (or format (state/get-preferred-format))
              pages    (map (fn [page]
-                             (-> (block/page-name->map page true)
+                             ;; only apply uuid to the deepest hierarchy of page to create if provided.
+                             (-> (block/page-name->map page (if (= page title) with-uuid? true))
                                  (assoc :block/format format)))
-                        pages)
+                           pages)
              txs      (->> pages
                            ;; for namespace pages, only last page need properties
                            drop-last
@@ -149,6 +157,7 @@
          (when (seq txs)
            (db/transact! txs)))
 
+       (prn "creating" page-name) ;; TODO Junyi
        (when create-first-block?
          (when (or
                 (db/page-empty? repo (:db/id (db/entity [:block/name page-name])))