fix: edn import: keep block uuid; remove redundant last block

fix: json import uuid failure

enhance: improve get-all-referenced-blocks-uuid query

src/main/frontend/components/external.cljs

@@ -1,3 +1,5 @@
+;; deprecated by the onboarding import panel frontend.components.onboarding.setups
+
 (ns frontend.components.external
   (:require [rum.core :as rum]
             [goog.object :as gobj]

src/main/frontend/db.cljs

@@ -52,7 +52,8 @@
   get-all-pages get-pages get-pages-relation get-pages-that-mentioned-page get-public-pages get-tag-pages
   journal-page? page-alias-set pull-block
   set-file-last-modified-at! page-empty? page-exists? page-empty-or-dummy? get-alias-source-page
-  set-file-content! has-children? get-namespace-pages get-all-namespace-relation get-pages-by-name-partition]
+  set-file-content! has-children? get-namespace-pages get-all-namespace-relation get-pages-by-name-partition
+  get-original-name]
 
  [frontend.db.react
   get-current-page set-key-value

src/main/frontend/db/model.cljs

@@ -1373,13 +1373,11 @@
   "Get all uuids of blocks with any back link exists."
   []
   (when-let [db (conn/get-db)]
-    (->> (d/datoms db :avet :block/uuid)
-         (map :v)
-         (map (fn [id]
-                (let [e (db-utils/entity [:block/uuid id])]
-                  (when (pos-int? (count (:block/_refs e)))
-                    id))))
-         (remove nil?))))
+    (d/q '[:find [?refed-uuid ...]
+           :where
+           ;; ?referee-b is block with ref towards ?refed-b
+           [?refed-b   :block/uuid ?refed-uuid]
+           [?referee-b :block/refs ?refed-b]] db)))
 
 ;; block/uuid and block/content
 (defn get-all-block-contents

src/main/frontend/handler/editor.cljs

@@ -943,7 +943,8 @@
               (save-block-if-changed! block new-content))))))))
 
 (defn set-blocks-id!
-  "Persist block uuid to file if not exists"
+  "Persist block uuid to file if the uuid is valid, and it's not persisted in file.
+   Accepts a list of uuids."
   [block-ids]
   (let [block-ids (remove nil? block-ids)
         col (map (fn [block-id]
@@ -1958,7 +1959,8 @@
           (edit-last-block-after-inserted! result))))))
 
 (defn- block-tree->blocks
-  [tree-vec format]
+  "keep-uuid? - maintain the existing :uuid in tree vec"
+  [tree-vec format keep-uuid?]
   (->> (outliner-core/tree-vec-flatten tree-vec)
        (map (fn [block]
               (let [content (:content block)
@@ -1967,15 +1969,18 @@
                                   (property/insert-properties format content props))
                     ast (mldoc/->edn content* (gp-mldoc/default-config format))
                     blocks (block/extract-blocks ast content* true format)
-                    fst-block (first blocks)]
+                    fst-block (first blocks)
+                    fst-block (if (and keep-uuid? (uuid? (:uuid block)))
+                                (assoc fst-block :block/uuid (:uuid block))
+                                fst-block)]
                 (assert fst-block "fst-block shouldn't be nil")
                 (assoc fst-block :block/level (:block/level block)))))))
 
 (defn insert-block-tree
   "`tree-vec`: a vector of blocks.
    A block element: {:content :properties :children [block-1, block-2, ...]}"
-  [tree-vec format {:keys [target-block] :as opts}]
-  (let [blocks (block-tree->blocks tree-vec format)
+  [tree-vec format {:keys [target-block keep-uuid?] :as opts}]
+  (let [blocks (block-tree->blocks tree-vec format keep-uuid?)
         page-id (:db/id (:block/page target-block))
         blocks (gp-block/with-parent-and-left page-id blocks)]
     (paste-blocks

src/main/frontend/handler/external.cljs

@@ -108,21 +108,22 @@
    :title    - page's title (original name)
    :children - tree
    "
-  [{:keys [id title children] :as tree}]
+  [{:keys [uuid title children] :as tree}]
   (let [has-children? (seq children)
         page-format (some-> tree (:children) (first) (:format))]
     (try (page/create! title {:redirect?  false
-                                  :format     page-format
-                                  :uuid       id})
+                              :format     page-format
+                              :uuid       uuid})
          (catch js/Error e
            (notification/show! (str "Error happens when creating page " title ":\n"
                                     e
                                     "\nSkipped and continue the remaining import.") :error)))
     (when has-children?
-      (let [page-block (db/entity [:block/name (util/page-name-sanity-lc title)])]
+      (let [page-block  (db/entity [:block/name (util/page-name-sanity-lc title)])
+            first-child (first (:block/_left page-block)) ]
         ;; Missing support for per block format (or deprecated?)
         (try (editor/insert-block-tree children page-format
-                                       {:target-block page-block
+                                       {:target-block first-child
                                         :sibling?     true
                                         :keep-uuid?   true})
              (catch js/Error e
@@ -145,37 +146,34 @@
   "Not rely on file system - backend compatible.
    tree-translator-fn: translate exported tree structure to the desired tree for import"
   [data tree-translator-fn]
-  (when-let [_repo (state/get-current-repo)]
-    (try (->> (:blocks data)
-              (map tree-translator-fn)
-              (pre-transact-uuids)
-              (mapv create-page-with-exported-tree!))
-         (editor/set-blocks-id! (db/get-all-referenced-blocks-uuid))
-         (catch js/Error e
-           (notification/show! (str "Error happens when importing:\n" e) :error)))))
+  (try (->> (:blocks data)
+            (map tree-translator-fn)
+            (pre-transact-uuids)
+            (mapv create-page-with-exported-tree!))
+       (editor/set-blocks-id! (db/get-all-referenced-blocks-uuid))
+       (catch js/Error e
+         (notification/show! (str "Error happens when importing:\n" e) :error))))
 
 (defn tree-vec-translate-edn
   "Actions to do for loading edn tree structure.
    1) Removes namespace `:block/` from all levels of the `tree-vec`
    2) Rename all :block/page-name to :title
-   3) Rename all :block/id to :uuid
-   4) Dissoc all :properties"
+   3) Rename all :block/id to :uuid"
   ([tree-vec]
-   (let [rm-kw-ns-fn #(-> %
+   (let [kw-trans-fn #(-> %
                           str
                           (string/replace ":block/page-name" ":block/title")
                           (string/replace ":block/id" ":block/uuid")
                           (string/replace ":block/" "")
                           keyword)
-         transform-map (fn [form]
-                         (if (map? form)
-                           ;; build a new map with the same keys but without the namespace
-                           (reduce-kv (fn [acc k v]
-                                        (if (not= :block/properties k)
-                                          (assoc acc (rm-kw-ns-fn k) v)
-                                          acc)) {} form)
+         map-trans-fn (fn [acc k v]
+                        (assoc acc (kw-trans-fn k) v))
+         tree-trans-fn (fn [form]
+                         (if (and (map? form)
+                                  (:block/id form))
+                           (reduce-kv map-trans-fn {} form)
                            form))]
-     (walk/postwalk transform-map tree-vec))))
+     (walk/postwalk tree-trans-fn tree-vec))))
 
 (defn import-from-edn!
   [raw finished-ok-handler]
@@ -186,27 +184,27 @@
   "Actions to do for loading json tree structure.
    1) Rename all :id to :uuid
    2) Rename all :page-name to :title
-   3) Dissoc all :properties
-   4) Rename all :format \"markdown\" to :format `:markdown`"
+   3) Rename all :format \"markdown\" to :format `:markdown`"
   ([tree-vec]
-   (let [rm-kw-ns-fn #(-> %
+   (let [kw-trans-fn #(-> %
                           str
                           (string/replace ":page-name" ":title")
                           (string/replace ":id" ":uuid")
                           (string/replace #"^:" "")
                           keyword)
-         transform-map (fn [form]
-                         (if (map? form)
-                           (reduce-kv (fn [acc k v]
-                                        (if (not= :properties k)
-                                          (let [k (rm-kw-ns-fn k)
-                                                v (if (= k :format) (keyword v) v)]
-                                            (assoc acc k v))
-                                          acc)) {} form)
-                           form))
-         _ (prn tree-vec)
-         _ (prn (walk/postwalk transform-map tree-vec))]
-     (walk/postwalk transform-map tree-vec))))
+         map-trans-fn (fn [acc k v]
+                        (cond (= :format k)
+                              (assoc acc (kw-trans-fn k) (keyword v))
+                              (= :id k)
+                              (assoc acc (kw-trans-fn k) (uuid v))
+                              :else
+                              (assoc acc (kw-trans-fn k) v)))
+         tree-trans-fn (fn [form]
+                         (if (and (map? form)
+                                  (:id form))
+                           (reduce-kv map-trans-fn {} form)
+                           form))]
+     (walk/postwalk tree-trans-fn tree-vec))))
 
 (defn import-from-json!
   [raw finished-ok-handler]

src/main/frontend/handler/page.cljs

@@ -157,7 +157,6 @@
          (when (seq txs)
            (db/transact! txs)))
 
-       (prn "creating" page-name) ;; TODO Junyi
        (when create-first-block?
          (when (or
                 (db/page-empty? repo (:db/id (db/entity [:block/name page-name])))