wip: export refactor

1. Move fns to worker
2. Don't rely on :block/file, use db blocks to build page content

src/main/frontend/components/export.cljs

@@ -22,10 +22,10 @@
      [:h1.title (t :export)]
      [:ul.mr-1
       [:li.mb-4
-       [:a.font-medium {:on-click #(export/export-repo-as-edn-v2! current-repo)}
+       [:a.font-medium {:on-click #(export/export-repo-as-edn! current-repo)}
         (t :export-edn)]]
       [:li.mb-4
-       [:a.font-medium {:on-click #(export/export-repo-as-json-v2! current-repo)}
+       [:a.font-medium {:on-click #(export/export-repo-as-json! current-repo)}
         (t :export-json)]]
       (when (config/db-based-graph? current-repo)
        [:li.mb-4

src/main/frontend/components/file.cljs

@@ -57,13 +57,7 @@
                      (if (or (nil? modified-at) (zero? modified-at))
                        (t :file/no-data)
                        (date/get-date-time-string
-                        (t/to-default-time-zone (tc/to-date-time modified-at))))]])
-
-             (when-not mobile?
-               [:td [:a.text-sm
-                     {:on-click (fn [_e]
-                                  (export-handler/download-file! file))}
-                     [:span (t :download)]]])]))]])))
+                        (t/to-default-time-zone (tc/to-date-time modified-at))))]])]))]])))
 
 (rum/defc files
   []

src/main/frontend/db_worker.cljs

@@ -15,6 +15,7 @@
             [logseq.db.sqlite.util :as sqlite-util]
             [frontend.worker.state :as worker-state]
             [frontend.worker.file :as file]
+            [frontend.worker.export :as worker-export]
             [logseq.db :as ldb]
             [frontend.worker.rtc.op-mem-layer :as op-mem-layer]
             [frontend.worker.rtc.db-listener :as rtc-db-listener]
@@ -405,6 +406,24 @@
      (worker-state/set-new-state! new-state)
      nil))
 
+  ;; Export
+  (block->content
+   [this repo block-uuid-or-page-name tree->file-opts context]
+   (when-let [conn (worker-state/get-datascript-conn repo)]
+     (worker-export/block->content repo @conn block-uuid-or-page-name
+                                   (edn/read-string tree->file-opts)
+                                   (edn/read-string context))))
+
+  (get-all-pages
+   [this repo]
+   (when-let [conn (worker-state/get-datascript-conn repo)]
+     (bean/->js (worker-export/get-all-pages repo @conn))))
+
+  (get-all-page->content
+   [this repo]
+   (when-let [conn (worker-state/get-datascript-conn repo)]
+     (bean/->js (worker-export/get-all-page->content repo @conn))))
+
   ;; RTC
   (rtc-start
    [this repo token]

src/main/frontend/handler/export.cljs

@@ -5,74 +5,24 @@
    [clojure.set :as s]
    [clojure.string :as string]
    [clojure.walk :as walk]
-   [datascript.core :as d]
    [frontend.config :as config]
    [frontend.db :as db]
    [frontend.extensions.zip :as zip]
    [frontend.external.roam-export :as roam-export]
-   [frontend.format.mldoc :as mldoc]
    [frontend.handler.notification :as notification]
    [frontend.mobile.util :as mobile-util]
-   [frontend.modules.file.core :as outliner-file]
-   [frontend.modules.outliner.tree :as outliner-tree]
    [logseq.publishing.html :as publish-html]
    [frontend.state :as state]
    [frontend.util :as util]
-   [frontend.handler.property.file :as property-file]
    [goog.dom :as gdom]
    [lambdaisland.glogi :as log]
-   [logseq.graph-parser.property :as gp-property]
-   [logseq.common.util.block-ref :as block-ref]
-   [logseq.common.util.page-ref :as page-ref]
    [promesa.core :as p]
-   [frontend.persist-db :as persist-db])
+   [frontend.persist-db :as persist-db]
+   [frontend.persist-db.browser :as db-browser]
+   [cljs-bean.core :as bean])
   (:import
    [goog.string StringBuffer]))
 
-(defn- get-page-content
-  [repo page]
-  (outliner-file/tree->file-content
-   (outliner-tree/blocks->vec-tree
-    (db/get-page-blocks-no-cache repo page) page) {:init-level 1}))
-
-(defn- get-file-content
-  [repo file-path]
-  (if-let [page-name
-           (ffirst (d/q '[:find ?pn
-                          :in $ ?path
-                          :where
-                          [?p :block/file ?f]
-                          [?p :block/name ?pn]
-                          [?f :file/path ?path]]
-                        (db/get-db repo) file-path))]
-    (get-page-content repo page-name)
-    (ffirst
-     (d/q '[:find ?content
-            :in $ ?path
-            :where
-            [?f :file/path ?path]
-            [?f :file/content ?content]]
-          (db/get-db repo) file-path))))
-
-(defn- get-blocks-contents
-  [repo root-block-uuid]
-  (->
-   (db/get-block-and-children repo root-block-uuid)
-   (outliner-tree/blocks->vec-tree (str root-block-uuid))
-   (outliner-file/tree->file-content {:init-level 1})))
-
-(defn download-file!
-  [file-path]
-  (when-let [repo (state/get-current-repo)]
-    (when-let [content (get-file-content repo file-path)]
-      (let [data (js/Blob. ["\ufeff" (array content)] ; prepend BOM
-                           (clj->js {:type "text/plain;charset=utf-8,"}))
-            anchor (gdom/getElement "download")
-            url (js/window.URL.createObjectURL data)]
-        (.setAttribute anchor "href" url)
-        (.setAttribute anchor "download" file-path)
-        (.click anchor)))))
-
 (defn download-repo-as-html!
   "download public pages as html"
   [repo]
@@ -98,28 +48,21 @@
           (.setAttribute anchor "download" "index.html")
           (.click anchor))))))
 
-(defn- get-file-contents
-  ([repo]
-   (get-file-contents repo {:init-level 1}))
-  ([repo file-opts]
-   (let [db (db/get-db repo)]
-     (->> (d/q '[:find ?n ?fp
-                 :where
-                 [?e :block/file ?f]
-                 [?f :file/path ?fp]
-                 [?e :block/name ?n]] db)
-          (mapv (fn [[page-name file-path]]
-                  [file-path
-                   (outliner-file/tree->file-content
-                    (outliner-tree/blocks->vec-tree
-                     (db/get-page-blocks-no-cache page-name) page-name)
-                    file-opts)]))))))
+(defn- <get-all-page->content
+  [repo]
+  (when-let [^object worker @db-browser/*worker]
+    (.get-all-page->content worker repo)))
 
 (defn export-repo-as-zip!
   [repo]
-  (let [files (get-file-contents repo)
-        [owner repo-name] (util/get-git-owner-and-repo repo)
-        repo-name (str owner "-" repo-name)]
+  (p/let [page->content (<get-all-page->content repo)
+          [owner repo-name] (util/get-git-owner-and-repo repo)
+          repo-name (str owner "-" repo-name)
+          files (map (fn [[title content]]
+                       [(str title "." (if (= :org (state/get-preferred-format))
+                                         "org"
+                                         "md"))
+                        content]) page->content)]
     (when (seq files)
       (p/let [zipfile (zip/make-zip repo-name files repo)]
         (when-let [anchor (gdom/getElement "download")]
@@ -127,137 +70,6 @@
           (.setAttribute anchor "download" (.-name zipfile))
           (.click anchor))))))
 
-(defn- get-embed-pages-from-ast [ast]
-  (let [result (transient #{})]
-    (doseq [item ast]
-      (walk/prewalk (fn [i]
-                      (cond
-                        (and (vector? i)
-                             (= "Macro" (first i))
-                             (= "embed" (some-> (:name (second i))
-                                                (string/lower-case)))
-                             (some-> (:arguments (second i))
-                                     first
-                                     page-ref/page-ref?))
-                        (let [arguments (:arguments (second i))
-                              page-ref (first arguments)
-                              page-name (-> page-ref
-                                            (subs 2)
-                                            (#(subs % 0 (- (count %) 2)))
-                                            (string/lower-case))]
-                          (conj! result page-name)
-                          i)
-                        :else
-                        i))
-                    item))
-    (persistent! result)))
-
-(defn- get-embed-blocks-from-ast [ast]
-  (let [result (transient #{})]
-    (doseq [item ast]
-      (walk/prewalk (fn [i]
-                      (cond
-                        (and (vector? i)
-                             (= "Macro" (first i))
-                             (= "embed" (some-> (:name (second i))
-                                                (string/lower-case)))
-                             (some-> (:arguments (second i))
-                                     (first)
-                                     block-ref/string-block-ref?))
-                        (let [arguments (:arguments (second i))
-                              block-uuid (block-ref/get-string-block-ref-id (first arguments))]
-                          (conj! result block-uuid)
-                          i)
-                        :else
-                        i)) item))
-    (persistent! result)))
-
-(defn- get-block-refs-from-ast [ast]
-  (let [result (transient #{})]
-    (doseq [item ast]
-      (walk/prewalk (fn [i]
-                      (cond
-                        (and (vector? i)
-                             (= "Block_ref" (first i))
-                             (some? (second i)))
-                        (let [block-uuid (second i)]
-                          (conj! result block-uuid)
-                          i)
-                        :else
-                        i)) item))
-    (persistent! result)))
-
-(declare get-page-page&block-refs)
-(defn get-block-page&block-refs [repo block-uuid embed-pages embed-blocks block-refs]
-  (let [block (db/entity [:block/uuid (uuid block-uuid)])
-        block-content (get-blocks-contents repo (:block/uuid block))
-        format (:block/format block)
-        ast (mldoc/->edn block-content format)
-        embed-pages-new  (get-embed-pages-from-ast ast)
-        embed-blocks-new  (get-embed-blocks-from-ast ast)
-        block-refs-new (get-block-refs-from-ast ast)
-        embed-pages-diff (s/difference embed-pages-new embed-pages)
-        embed-blocks-diff (s/difference embed-blocks-new embed-blocks)
-        block-refs-diff (s/difference block-refs-new block-refs)
-        embed-pages* (s/union embed-pages-new embed-pages)
-        embed-blocks* (s/union embed-blocks-new embed-blocks)
-        block-refs* (s/union block-refs-new block-refs)
-        [embed-pages-1 embed-blocks-1 block-refs-1]
-        (->>
-         (mapv (fn [page-name]
-                 (let [{:keys [embed-pages embed-blocks block-refs]}
-                       (get-page-page&block-refs repo page-name embed-pages* embed-blocks* block-refs*)]
-                   [embed-pages embed-blocks block-refs])) embed-pages-diff)
-         (apply mapv vector) ; [[1 2 3] [4 5 6] [7 8 9]] -> [[1 4 7] [2 5 8] [3 6 9]]
-         (mapv #(apply s/union %)))
-        [embed-pages-2 embed-blocks-2 block-refs-2]
-        (->>
-         (mapv (fn [block-uuid]
-                 (let [{:keys [embed-pages embed-blocks block-refs]}
-                       (get-block-page&block-refs repo block-uuid embed-pages* embed-blocks* block-refs*)]
-                   [embed-pages embed-blocks block-refs])) (s/union embed-blocks-diff block-refs-diff))
-         (apply mapv vector)
-         (mapv #(apply s/union %)))]
-    {:embed-pages (s/union embed-pages-1 embed-pages-2 embed-pages*)
-     :embed-blocks (s/union embed-blocks-1 embed-blocks-2 embed-blocks*)
-     :block-refs (s/union block-refs-1 block-refs-2 block-refs*)}))
-
-
-(defn get-page-page&block-refs [repo page-name embed-pages embed-blocks block-refs]
-  (let [page-name* (util/page-name-sanity-lc page-name)
-        page-content (get-page-content repo page-name*)
-        format (:block/format (db/entity [:block/name page-name*]))
-        ast (mldoc/->edn page-content format)
-        embed-pages-new (get-embed-pages-from-ast ast)
-        embed-blocks-new (get-embed-blocks-from-ast ast)
-        block-refs-new (get-block-refs-from-ast ast)
-        embed-pages-diff (s/difference embed-pages-new embed-pages)
-        embed-blocks-diff (s/difference embed-blocks-new embed-blocks)
-        block-refs-diff (s/difference block-refs-new block-refs)
-        embed-pages* (s/union embed-pages-new embed-pages)
-        embed-blocks* (s/union embed-blocks-new embed-blocks)
-        block-refs* (s/union block-refs-new block-refs)
-        [embed-pages-1 embed-blocks-1 block-refs-1]
-        (->>
-         (mapv (fn [page-name]
-                 (let [{:keys [embed-pages embed-blocks block-refs]}
-                       (get-page-page&block-refs repo page-name embed-pages* embed-blocks* block-refs*)]
-                   [embed-pages embed-blocks block-refs])) embed-pages-diff)
-         (apply mapv vector)
-         (mapv #(apply s/union %)))
-        [embed-pages-2 embed-blocks-2 block-refs-2]
-        (->>
-         (mapv (fn [block-uuid]
-                 (let [{:keys [embed-pages embed-blocks block-refs]}
-                       (get-block-page&block-refs repo block-uuid embed-pages* embed-blocks* block-refs*)]
-                   [embed-pages embed-blocks block-refs])) (s/union embed-blocks-diff block-refs-diff))
-         (apply mapv vector)
-         (mapv #(apply s/union %)))]
-    {:embed-pages (s/union embed-pages-1 embed-pages-2 embed-pages*)
-     :embed-blocks (s/union embed-blocks-1 embed-blocks-2 embed-blocks*)
-     :block-refs (s/union block-refs-1 block-refs-2 block-refs*)}))
-
-
 (defn- export-file-on-mobile [data path]
   (p/catch
    (.writeFile Filesystem (clj->js {:path path
@@ -294,56 +106,26 @@
        x))
    vec-tree))
 
-(defn- safe-keywordize
-  [block]
-  (update block :block/properties
-          (fn [properties]
-            (when (seq properties)
-              (->> (filter (fn [[k _v]]
-                             (gp-property/valid-property-name? (str k))) properties)
-                   (into {}))))))
-
-(defn- blocks
-  [repo db]
-  {:version 1
-   :blocks
-   (->> (d/q '[:find (pull ?b [*])
-               :in $
-               :where
-               [?b :block/original-name]
-               [?b :block/name]] db)
+(defn- <get-all-pages
+  [repo]
+  (when-let [^object worker @db-browser/*worker]
+    (.get-all-pages worker repo)))
 
-        (map (fn [[{:block/keys [name] :as page}]]
-               (let [whiteboard? (contains? (set (:block/type page)) "whiteboard")
-                     blocks (db/get-page-blocks-no-cache
-                             (state/get-current-repo)
-                             name
-                             {:transform? false})
-                     blocks' (if whiteboard?
-                               blocks
-                               (map (fn [b]
-                                      (let [b' (if (seq (:block/properties b))
-                                                 (update b :block/content
-                                                         (fn [content]
-                                                           (property-file/remove-properties-when-file-based
-                                                            repo (:block/format b) content)))
-                                                 b)]
-                                        (safe-keywordize b'))) blocks))
-                     children (if whiteboard?
-                                blocks'
-                                (outliner-tree/blocks->vec-tree blocks' name))
-                     page' (safe-keywordize page)]
-                 (assoc page' :block/children children))))
-        (nested-select-keys
-         [:block/id
-          :block/type
-          :block/page-name
-          :block/properties
-          :block/format
-          :block/children
-          :block/content
-          :block/created-at
-          :block/updated-at]))})
+(defn- <build-blocks
+  [repo]
+  (p/let [pages (<get-all-pages repo)]
+    {:version 1
+     :blocks
+     (nested-select-keys pages
+                         [:block/id
+                          :block/type
+                          :block/page-name
+                          :block/properties
+                          :block/format
+                          :block/children
+                          :block/content
+                          :block/created-at
+                          :block/updated-at])}))
 
 (defn- file-name [repo extension]
   (-> (string/replace repo config/local-db-prefix "")
@@ -351,15 +133,15 @@
       (str "_" (quot (util/time-ms) 1000))
       (str "." (string/lower-case (name extension)))))
 
-(defn- export-repo-as-edn-str [repo]
-  (when-let [db (db/get-db repo)]
+(defn- <export-repo-as-edn-str [repo]
+  (p/let [result (<build-blocks repo)]
     (let [sb (StringBuffer.)]
-      (pprint/pprint (blocks repo db) (StringBufferWriter. sb))
+      (pprint/pprint result (StringBufferWriter. sb))
       (str sb))))
 
-(defn export-repo-as-edn-v2!
+(defn export-repo-as-edn!
   [repo]
-  (when-let [edn-str (export-repo-as-edn-str repo)]
+  (when-let [edn-str (<export-repo-as-edn-str repo)]
     (let [data-str (some->> edn-str
                             js/encodeURIComponent
                             (str "data:text/edn;charset=utf-8,"))
@@ -380,23 +162,22 @@
        x))
    vec-tree))
 
-(defn export-repo-as-json-v2!
+(defn export-repo-as-json!
   [repo]
-  (when-let [db (db/get-db repo)]
-    (let [json-str
-          (-> (blocks repo db)
-              nested-update-id
-              clj->js
-              js/JSON.stringify)
+  (p/let [result (<build-blocks repo)
+          json-str (-> result
+                       nested-update-id
+                       clj->js
+                       js/JSON.stringify)
           filename (file-name repo :json)
           data-str (str "data:text/json;charset=utf-8,"
                         (js/encodeURIComponent json-str))]
-      (if (mobile-util/native-platform?)
-        (export-file-on-mobile json-str filename)
-        (when-let [anchor (gdom/getElement "download-as-json-v2")]
-          (.setAttribute anchor "href" data-str)
-          (.setAttribute anchor "download" filename)
-          (.click anchor))))))
+    (if (mobile-util/native-platform?)
+      (export-file-on-mobile json-str filename)
+      (when-let [anchor (gdom/getElement "download-as-json-v2")]
+        (.setAttribute anchor "href" data-str)
+        (.setAttribute anchor "download" filename)
+        (.click anchor)))))
 
 (defn export-repo-as-sqlite-db!
   [repo]
@@ -415,39 +196,25 @@
 
 ;; https://roamresearch.com/#/app/help/page/Nxz8u0vXU
 ;; export to roam json according to above spec
-(defn- roam-json [db]
-  (->> (d/q '[:find (pull ?b [*])
-              :in $
-              :where
-              [?b :block/file]
-              [?b :block/original-name]
-              [?b :block/name]] db)
-
-       (map (fn [[{:block/keys [name] :as page}]]
-              (assoc page
-                     :block/children
-                     (outliner-tree/blocks->vec-tree
-                      (db/get-page-blocks-no-cache
-                       (state/get-current-repo)
-                       name
-                       {:transform? false}) name))))
-
-       (roam-export/traverse
-        [:page/title
-         :block/string
-         :block/uid
-         :block/children])))
+(defn- <roam-data [repo]
+  (p/let [pages (<get-all-pages repo)]
+    (let [non-empty-pages (remove #(empty? (:block/children %)) pages)]
+      (roam-export/traverse
+       [:page/title
+        :block/string
+        :block/uid
+        :block/children]
+       non-empty-pages))))
 
 (defn export-repo-as-roam-json!
   [repo]
-  (when-let [db (db/get-db repo)]
-    (let [json-str
-          (-> (roam-json db)
-              clj->js
-              js/JSON.stringify)
+  (p/let [data (<roam-data repo)
+          json-str (-> data
+                       bean/->js
+                       js/JSON.stringify)
           data-str (str "data:text/json;charset=utf-8,"
                         (js/encodeURIComponent json-str))]
-      (when-let [anchor (gdom/getElement "download-as-roam-json")]
-        (.setAttribute anchor "href" data-str)
-        (.setAttribute anchor "download" (file-name (str repo "_roam") :json))
-        (.click anchor)))))
+    (when-let [anchor (gdom/getElement "download-as-roam-json")]
+      (.setAttribute anchor "href" data-str)
+      (.setAttribute anchor "download" (file-name (str repo "_roam") :json))
+      (.click anchor))))

src/main/frontend/handler/export/common.cljs

@@ -5,7 +5,6 @@
   (:refer-clojure :exclude [map filter mapcat concat remove])
   (:require [cljs.core.match :refer [match]]
             [clojure.string :as string]
-            [datascript.core :as d]
             [frontend.db :as db]
             [frontend.format.mldoc :as mldoc]
             [frontend.modules.file.core :as outliner-file]
@@ -15,7 +14,9 @@
             [logseq.common.util :as common-util]
             [frontend.handler.property.util :as pu]
             [malli.core :as m]
-            [malli.util :as mu]))
+            [malli.util :as mu]
+            [frontend.db.async :as db-async]
+            [promesa.core :as p]))
 
 ;;; TODO: split frontend.handler.export.text related states
 (def ^:dynamic *state*
@@ -179,38 +180,33 @@
                  ast-content)))
            inline-coll)))
 
-(defn- get-file-contents
+(defn- <get-file-contents
   [repo]
-  (let [db (db/get-db repo)]
-    (->> (d/q '[:find ?fp
-                :where
-                [?e :block/file ?f]
-                [?f :file/path ?fp]] db)
-         (mapv (fn [[file-path]]
-                 [file-path
-                  (db/get-file file-path)])))))
-
-(defn- get-md-file-contents
+  (db-async/<q repo '[:find ?pn ?fp ?fc
+                      :where
+                      [?e :block/original-name ?pn]
+                      [?e :block/file ?f]
+                      [?f :file/path ?fp]
+                      [?f :file/content ?fc]]))
+
+(defn- <get-md-file-contents
   [repo]
-  (filterv (fn [[path _]]
-             (let [path (string/lower-case path)]
-               (re-find #"\.(?:md|markdown)$" path)))
-           (get-file-contents repo)))
+  (p/let [result (<get-file-contents repo)]
+    (filterv (fn [[path _]]
+               (let [path (string/lower-case path)]
+                 (re-find #"\.(?:md|markdown)$" path)))
+             result)))
 
-(defn get-file-contents-with-suffix
+(defn <get-file-contents-with-suffix
   [repo]
-  (let [db (db/get-db repo)
-        md-files (get-md-file-contents repo)]
+  (p/let [md-files (<get-md-file-contents repo)]
     (->>
      md-files
-     (mapv (fn [[path content]] {:path path :content content
-                                 :names (d/q '[:find [?n ?n2]
-                                               :in $ ?p
-                                               :where [?e :file/path ?p]
-                                               [?e2 :block/file ?e]
-                                               [?e2 :block/name ?n]
-                                               [?e2 :block/original-name ?n2]] db path)
-                                 :format (common-util/get-format path)})))))
+     (mapv (fn [[page-title path content]]
+             {:path path
+              :content content
+              :title page-title
+              :format (common-util/get-format path)})))))
 
 ;;; utils (ends)
 

src/main/frontend/handler/export/opml.cljs

@@ -450,21 +450,22 @@
   "options see also `export-blocks-as-opml`"
   [files options]
   (mapv
-   (fn [{:keys [path content names format]}]
-     (when (first names)
+   (fn [{:keys [path content title format]}]
+     (when title
        (util/profile (print-str :export-files-as-opml path)
-                     [path (export-helper content format options :title (first names))])))
+                     [path (export-helper content format options :title title)])))
    files))
 
 (defn export-repo-as-opml!
   [repo]
-  (when-let [files (common/get-file-contents-with-suffix repo)]
-    (let [files (export-files-as-opml files nil)
-          zip-file-name (str repo "_opml_" (quot (util/time-ms) 1000))]
-      (p/let [zipfile (zip/make-zip zip-file-name files repo)]
-        (when-let [anchor (gdom/getElement "export-as-opml")]
-          (.setAttribute anchor "href" (js/window.URL.createObjectURL zipfile))
-          (.setAttribute anchor "download" (.-name zipfile))
-          (.click anchor))))))
+  (p/let [files (common/<get-file-contents-with-suffix repo)]
+    (when (seq files)
+      (let [files (export-files-as-opml files nil)
+            zip-file-name (str repo "_opml_" (quot (util/time-ms) 1000))]
+        (p/let [zipfile (zip/make-zip zip-file-name files repo)]
+          (when-let [anchor (gdom/getElement "export-as-opml")]
+            (.setAttribute anchor "href" (js/window.URL.createObjectURL zipfile))
+            (.setAttribute anchor "download" (.-name zipfile))
+            (.click anchor)))))))
 
 ;;; export fns (ends)

src/main/frontend/handler/export/text.cljs

@@ -495,8 +495,8 @@
   "options see also `export-blocks-as-markdown`"
   [files options]
   (mapv
-   (fn [{:keys [path content names format]}]
-     (when (first names)
+   (fn [{:keys [path content title format]}]
+     (when title
        (util/profile (print-str :export-files-as-markdown path)
                      [path (export-helper content format options)])))
    files))
@@ -504,13 +504,14 @@
 (defn export-repo-as-markdown!
   "TODO: indent-style and remove-options"
   [repo]
-  (when-let [files (util/profile :get-file-content (common/get-file-contents-with-suffix repo))]
-    (let [files (export-files-as-markdown files nil)
-          zip-file-name (str repo "_markdown_" (quot (util/time-ms) 1000))]
-      (p/let [zipfile (zip/make-zip zip-file-name files repo)]
-        (when-let [anchor (gdom/getElement "export-as-markdown")]
-          (.setAttribute anchor "href" (js/window.URL.createObjectURL zipfile))
-          (.setAttribute anchor "download" (.-name zipfile))
-          (.click anchor))))))
+  (p/let [files (util/profile :get-file-content (common/<get-file-contents-with-suffix repo))]
+    (when (seq files)
+      (let [files (export-files-as-markdown files nil)
+            zip-file-name (str repo "_markdown_" (quot (util/time-ms) 1000))]
+        (p/let [zipfile (zip/make-zip zip-file-name files repo)]
+          (when-let [anchor (gdom/getElement "export-as-markdown")]
+            (.setAttribute anchor "href" (js/window.URL.createObjectURL zipfile))
+            (.setAttribute anchor "download" (.-name zipfile))
+            (.click anchor)))))))
 
 ;;; export fns (ends)

src/main/frontend/handler/whiteboard.cljs

@@ -18,8 +18,7 @@
             [clojure.set :as set]
             [clojure.string :as string]
             [cljs-bean.core :as bean]
-            [logseq.db.sqlite.util :as sqlite-util]
-            [clojure.data :as data]))
+            [logseq.db.sqlite.util :as sqlite-util]))
 
 (defn js->clj-keywordize
   [obj]
@@ -255,39 +254,18 @@
         blocks (:block/_page react-page)]
     (whiteboard-clj->tldr react-page blocks)))
 
-(defn- get-whiteboard-blocks
-  "Given a page, return all the logseq blocks (exclude all shapes)"
-  [page-name]
-  (let [blocks (model/get-page-blocks-no-cache page-name)]
-    (remove pu/shape-block? blocks)))
-
-(defn- get-last-root-block
-  "Get the last root Logseq block in the page. Main purpose is to calculate the new :block/left id"
-  [page-name]
-  (let [page-id (:db/id (model/get-page page-name))
-        blocks (get-whiteboard-blocks page-name)
-        root-blocks (filter (fn [block] (= page-id (:db/id (:block/parent block)))) blocks)
-        root-block-left-ids (->> root-blocks
-                                 (map (fn [block] (get-in block [:block/left :db/id] nil)))
-                                 (remove nil?)
-                                 (set))
-        blocks-with-no-next (remove #(root-block-left-ids (:db/id %)) root-blocks)]
-    (when (seq blocks-with-no-next) (first blocks-with-no-next))))
-
 (defn <add-new-block!
   [page-name content]
   (p/let [repo (state/get-current-repo)
           new-block-id (db/new-block-id)
           page-entity (model/get-page page-name)
-          last-root-block (or (get-last-root-block page-name) page-entity)
           tx (sqlite-util/block-with-timestamps
-              {:block/left (select-keys last-root-block [:db/id])
-               :block/uuid new-block-id
+              {:block/uuid new-block-id
                :block/content (or content "")
                :block/format :markdown
                :block/page {:block/name (util/page-name-sanity-lc page-name)
                             :block/original-name page-name}
-               :block/parent {:block/name page-name}})
+               :block/parent (:db/id page-entity)})
           _ (db/transact! repo [tx] {:whiteboard/transact? true})]
     new-block-id))
 

src/main/frontend/worker/export.cljs

@@ -0,0 +1,68 @@
+(ns frontend.worker.export
+  "Export data"
+  (:require [logseq.db :as ldb]
+            [logseq.outliner.tree :as otree]
+            [frontend.worker.file.core :as worker-file]
+            [datascript.core :as d]
+            [logseq.common.util :as common-util]
+            [logseq.graph-parser.property :as gp-property]
+            [datascript.core :as d]))
+
+(defn block->content
+  "Converts a block including its children (recursively) to plain-text."
+  [repo db root-block-uuid-or-page-name tree->file-opts context]
+  (let [root-block-uuid (or
+                         (and (uuid? root-block-uuid-or-page-name) root-block-uuid-or-page-name)
+                         (:block/uuid (d/entity db [:block/name (common-util/page-name-sanity-lc
+                                                                 root-block-uuid-or-page-name)])))
+        init-level (or (:init-level tree->file-opts)
+                       (if (uuid? root-block-uuid-or-page-name) 1 0))
+        blocks (ldb/get-block-and-children repo db root-block-uuid)
+        tree (otree/blocks->vec-tree repo db blocks (str root-block-uuid))]
+    (worker-file/tree->file-content repo db tree
+                                    (assoc tree->file-opts :init-level init-level)
+                                    context)))
+
+(defn- safe-keywordize
+  [block]
+  (update block :block/properties
+          (fn [properties]
+            (when (seq properties)
+              (->> (filter (fn [[k _v]]
+                             (gp-property/valid-property-name? (str k))) properties)
+                   (into {}))))))
+
+(defn get-all-pages
+  "Get all pages and their children blocks."
+  [repo db]
+  (->> (d/q '[:find (pull ?b [*])
+              :in $
+              :where
+              [?b :block/original-name]
+              [?b :block/name]] db)
+
+       (map (fn [[{:block/keys [name] :as page}]]
+              (let [whiteboard? (contains? (set (:block/type page)) "whiteboard")
+                    blocks (ldb/get-page-blocks db name {})
+                    blocks' (if whiteboard?
+                              blocks
+                              (map (fn [b]
+                                     (let [b' (if (seq (:block/properties b))
+                                                (update b :block/content
+                                                        (fn [content]
+                                                          (gp-property/remove-properties (:block/format b) content)))
+                                                b)]
+                                       (safe-keywordize b'))) blocks))
+                    children (if whiteboard?
+                               blocks'
+                               (otree/blocks->vec-tree repo db blocks' name))
+                    page' (safe-keywordize page)]
+                (assoc page' :block/children children))))))
+
+(defn get-all-page->content
+  [repo db]
+  (->> (d/datoms db :avet :block/name)
+       (map (fn [d]
+              (let [e (d/entity db (:e d))]
+                [(:block/original-name e)
+                 (block->content repo db (:v d) {} {})])))))

src/test/frontend/handler/export_test.cljs

@@ -15,11 +15,11 @@
     (string/trim "
 - 1
   id:: 61506710-484c-46d5-9983-3d1651ec02c8
-	- 2
-	  id:: 61506711-5638-4899-ad78-187bdc2eaffc
-		- 3
-		  id:: 61506712-3007-407e-b6d3-d008a8dfa88b
-		- ((61506712-3007-407e-b6d3-d008a8dfa88b))
+        - 2
+          id:: 61506711-5638-4899-ad78-187bdc2eaffc
+                - 3
+                  id:: 61506712-3007-407e-b6d3-d008a8dfa88b
+                - ((61506712-3007-407e-b6d3-d008a8dfa88b))
 - 4
   id:: 61506712-b8a7-491d-ad84-b71651c3fdab")}
    {:file/path "pages/page2.md"
@@ -27,7 +27,7 @@
     (string/trim "
 - 3
   id:: 97a00e55-48c3-48d8-b9ca-417b16e3a616
-	- {{embed [[page1]]}}")}])
+        - {{embed [[page1]]}}")}])
 
 (use-fixtures :once
   {:before (fn []
@@ -44,18 +44,18 @@
                                                               {:remove-options #{:property}})))
     (string/trim "
 - 1
-	- 2
-		- 3
-		- 3")
+        - 2
+                - 3
+                - 3")
     "61506710-484c-46d5-9983-3d1651ec02c8"
 
     (string/trim "
 - 3
-	- 1
-		- 2
-			- 3
-			- 3
-	- 4")
+        - 1
+                - 2
+                        - 3
+                        - 3
+        - 4")
     "97a00e55-48c3-48d8-b9ca-417b16e3a616"))
 
 
@@ -65,25 +65,25 @@
     (string/trim "
 - 1
   id:: 61506710-484c-46d5-9983-3d1651ec02c8
-	- 2
-	  id:: 61506711-5638-4899-ad78-187bdc2eaffc
-		- 3
-		  id:: 61506712-3007-407e-b6d3-d008a8dfa88b
-		- 3")
+        - 2
+          id:: 61506711-5638-4899-ad78-187bdc2eaffc
+                - 3
+                  id:: 61506712-3007-407e-b6d3-d008a8dfa88b
+                - 3")
     "61506710-484c-46d5-9983-3d1651ec02c8"
 
     (string/trim "
 - 3
   id:: 97a00e55-48c3-48d8-b9ca-417b16e3a616
-	- 1
-	  id:: 61506710-484c-46d5-9983-3d1651ec02c8
-		- 2
-		  id:: 61506711-5638-4899-ad78-187bdc2eaffc
-			- 3
-			  id:: 61506712-3007-407e-b6d3-d008a8dfa88b
-			- 3
-	- 4
-	  id:: 61506712-b8a7-491d-ad84-b71651c3fdab")
+        - 1
+          id:: 61506710-484c-46d5-9983-3d1651ec02c8
+                - 2
+                  id:: 61506711-5638-4899-ad78-187bdc2eaffc
+                        - 3
+                          id:: 61506712-3007-407e-b6d3-d008a8dfa88b
+                        - 3
+        - 4
+          id:: 61506712-b8a7-491d-ad84-b71651c3fdab")
     "97a00e55-48c3-48d8-b9ca-417b16e3a616"))
 
 (deftest export-blocks-as-markdown-level<N
@@ -93,13 +93,13 @@
                                                                       :other-options {:keep-only-level<=N 2}})))
     (string/trim "
 - 1
-	- 2")
+        - 2")
     "61506710-484c-46d5-9983-3d1651ec02c8"
 
     (string/trim "
 - 3
-	- 1
-	- 4")
+        - 1
+        - 4")
     "97a00e55-48c3-48d8-b9ca-417b16e3a616"))
 
 (deftest export-blocks-as-markdown-newline-after-block
@@ -110,24 +110,24 @@
     (string/trim "
 - 1
 
-	- 2
+        - 2
 
-		- 3
+                - 3
 
-		- 3")
+                - 3")
     "61506710-484c-46d5-9983-3d1651ec02c8"
     (string/trim "
 - 3
 
-	- 1
+        - 1
 
-		- 2
+                - 2
 
-			- 3
+                        - 3
 
-			- 3
+                        - 3
 
-	- 4")
+        - 4")
     "97a00e55-48c3-48d8-b9ca-417b16e3a616"))
 
 
@@ -143,12 +143,11 @@
        [{:path "pages/page2.md" :content (:file/content (nth test-files 1)) :names ["page2"] :format :markdown}])))
 
 (deftest-async export-repo-as-edn-str
-  (p/do!
-   (let [edn-output (edn/read-string
-                     (@#'export/export-repo-as-edn-str (state/get-current-repo)))]
-     (is (= #{:version :blocks} (set (keys edn-output)))
-         "Correct top-level keys")
-     (is (= (sort (concat (map :block/original-name default-db/built-in-pages)
-                          ["page1" "page2"]))
-            (sort (map :block/page-name (:blocks edn-output))))
-         "Correct pages"))))
+  (p/let [edn-output (edn/read-string
+                      (@#'export/<export-repo-as-edn-str (state/get-current-repo)))]
+    (is (= #{:version :blocks} (set (keys edn-output)))
+        "Correct top-level keys")
+    (is (= (sort (concat (map :block/original-name default-db/built-in-pages)
+                         ["page1" "page2"]))
+           (sort (map :block/page-name (:blocks edn-output))))
+        "Correct pages")))