Move outliner tree to the outliner dep

deps/db/deps.edn

@@ -3,7 +3,8 @@
  {datascript/datascript {:git/url "https://github.com/logseq/datascript" ;; fork
                          :sha     "21fc7880c7042fb1d9086135d162ea7a91681f89"}
   com.cognitect/transit-cljs   {:mvn/version "0.8.280"}
-  cljs-bean/cljs-bean          {:mvn/version "1.5.0"}}
+  cljs-bean/cljs-bean          {:mvn/version "1.5.0"}
+  logseq/graph-parser          {:local/root "deps/graph-parser"}}
 
  :aliases
  {:clj-kondo

deps/db/src/logseq/db.cljs

@@ -8,7 +8,42 @@
             [logseq.db.frontend.property :as db-property]
             [logseq.db.frontend.property.util :as db-property-util]
             [logseq.db.sqlite.util :as sqlite-util]
-            [clojure.string :as string]))
+            [clojure.string :as string]
+            [logseq.graph-parser.util :as gp-util]
+            [logseq.db.frontend.content :as db-content]))
+
+;; Use it as an input argument for datalog queries
+(def block-attrs
+  '[:db/id
+    :block/uuid
+    :block/parent
+    :block/left
+    :block/collapsed?
+    :block/collapsed-properties
+    :block/format
+    :block/refs
+    :block/_refs
+    :block/path-refs
+    :block/tags
+    :block/link
+    :block/content
+    :block/marker
+    :block/priority
+    :block/properties
+    :block/properties-order
+    :block/properties-text-values
+    :block/pre-block?
+    :block/scheduled
+    :block/deadline
+    :block/repeated?
+    :block/created-at
+    :block/updated-at
+    ;; TODO: remove this in later releases
+    :block/heading-level
+    :block/file
+    {:block/page [:db/id :block/name :block/original-name :block/journal-day]}
+    {:block/_parent ...}])
+
 
 (defn create-default-pages!
   "Creates default pages if one of the default pages does not exist. This
@@ -57,3 +92,44 @@
     (when-not file-based?
       (create-built-in-properties! db-conn))
     db-conn))
+
+(defn sort-by-left
+  ([blocks parent]
+   (sort-by-left blocks parent {:check? true}))
+  ([blocks parent {:keys [_check?]}]
+   (let [blocks (gp-util/distinct-by :db/id blocks)
+         left->blocks (reduce (fn [acc b] (assoc acc (:db/id (:block/left b)) b)) {} blocks)]
+     (loop [block parent
+            result []]
+       (if-let [next (get left->blocks (:db/id block))]
+         (recur next (conj result next))
+         (vec result))))))
+
+(defn try-sort-by-left
+  [blocks parent]
+  (let [result' (sort-by-left blocks parent {:check? false})]
+    (if (= (count result') (count blocks))
+      result'
+      blocks)))
+
+;; TODO: use the tree directly
+(defn flatten-tree
+  [blocks-tree]
+  (if-let [children (:block/_parent blocks-tree)]
+    (cons (dissoc blocks-tree :block/_parent) (mapcat flatten-tree children))
+    [blocks-tree]))
+
+;; TODO: performance enhance
+(defn get-block-and-children
+  [repo db block-uuid]
+  (some-> (d/q
+           '[:find [(pull ?block ?block-attrs) ...]
+             :in $ ?id ?block-attrs
+             :where
+             [?block :block/uuid ?id]]
+           db
+           block-uuid
+           block-attrs)
+          first
+          flatten-tree
+          (->> (map #(db-content/update-block-content repo db % (:db/id %))))))

deps/db/src/logseq/db/frontend/content.cljs

@@ -0,0 +1,45 @@
+(ns logseq.db.frontend.content
+  "fns to handle special ids"
+  (:require [clojure.string :as string]
+            [logseq.graph-parser.util.page-ref :as page-ref]
+            [datascript.core :as d]
+            [logseq.db.sqlite.util :as sqlite-util]))
+
+(defonce page-ref-special-chars "~^")
+
+(defn special-id->page
+  "Convert special id backs to page name."
+  [content refs]
+  (reduce
+   (fn [content ref]
+     (if (:block/name ref)
+       (string/replace content (str page-ref-special-chars (:block/uuid ref)) (:block/original-name ref))
+       content))
+   content
+   refs))
+
+(defn special-id-ref->page
+  "Convert special id ref backs to page name."
+  [content refs]
+  (reduce
+   (fn [content ref]
+     (if (:block/name ref)
+       (string/replace content
+                       (str page-ref/left-brackets
+                            page-ref-special-chars
+                            (:block/uuid ref)
+                            page-ref/right-brackets)
+                       (:block/original-name ref))
+       content))
+   content
+   refs))
+
+(defn update-block-content
+  "Replace `[[internal-id]]` with `[[page name]]`"
+  [repo db item eid]
+  (if (sqlite-util/db-based-graph? repo)
+    (if-let [content (:block/content item)]
+      (let [refs (:block/refs (d/entity db eid))]
+        (assoc item :block/content (special-id->page content refs)))
+      item)
+    item))

deps/db/src/logseq/db/frontend/property.cljs

@@ -1,6 +1,9 @@
 (ns logseq.db.frontend.property
   "Property related fns for DB graphs and frontend/datascript usage"
-  (:require [clojure.set :as set]))
+  (:require [clojure.set :as set]
+            [logseq.db.sqlite.util :as sqlite-util]
+            [datascript.core :as d]
+            [logseq.graph-parser.util :as gp-util]))
 
 ;; FIXME: no support for built-in-extended-properties
 (def ^:large-vars/data-var built-in-properties
@@ -101,3 +104,27 @@
   {:pre [(string? s)]}
   ;; Disallow tags or page refs as they would create unreferenceable page names
   (not (re-find #"^(#|\[\[)" s)))
+
+(defn lookup
+  "Get the value of coll's (a map) `key`. For file and db graphs"
+  [repo db coll key]
+  (when db
+    (let [property-name (if (keyword? key)
+                          (name key)
+                          key)]
+      (if (sqlite-util/db-based-graph? repo)
+        (when-let [property (d/entity db [:block/name (gp-util/page-name-sanity-lc property-name)])]
+          (get coll (:block/uuid property)))
+        (get coll key)))))
+
+(defn get-property
+  "Get the value of block's property `key`"
+  [repo db block key]
+  (when db
+    (let [block (or (d/entity db (:db/id block)) block)]
+      (when-let [properties (:block/properties block)]
+        (lookup repo db properties key)))))
+
+(defn shape-block?
+  [repo db block]
+  (= :whiteboard-shape (get-property repo db block :ls-type)))

deps/db/src/logseq/db/sqlite/util.cljs

@@ -3,7 +3,8 @@
   (:require [cljs-time.coerce :as tc]
             [cljs-time.core :as t]
             [clojure.string :as string]
-            [logseq.db.frontend.schema :as db-schema]))
+            [logseq.db.frontend.schema :as db-schema]
+            [logseq.graph-parser.util :as gp-util]))
 
 (defonce db-version-prefix "logseq_db_")
 (defonce file-version-prefix "logseq_local_")
@@ -34,12 +35,7 @@
                 (assoc :block/created-at updated-at))]
     block))
 
-(defn sanitize-page-name
-  "Prepares a string for insertion to :block/name. Not using
-  gp-util/page-name-sanity-lc yet because it's unclear if db graphs have all the
-  same naming constraints"
-  [s]
-  (string/lower-case s))
+(def sanitize-page-name gp-util/page-name-sanity-lc)
 
 (defn build-new-property
   "Build a standard new property so that it is is consistent across contexts"

deps/outliner/src/logseq/outliner/tree.cljs

@@ -0,0 +1,135 @@
+(ns logseq.outliner.tree
+  (:require [logseq.db :as ldb]
+            [clojure.string :as string]
+            [logseq.db.frontend.property :as db-property]
+            [datascript.core :as d]))
+
+(defprotocol INode
+  (-get-id [this])
+  (-get-parent-id [this])
+  (-get-left-id [this])
+  (-set-left-id [this left-id])
+  (-get-parent [this])
+  (-get-left [this])
+  (-get-right [this])
+  (-get-down [this])
+  (-save [this db])
+  (-del [this db children?])
+  (-get-children [this]))
+
+(defn satisfied-inode?
+  [node]
+  (satisfies? INode node))
+
+(defn- blocks->vec-tree-aux
+  [repo db blocks root]
+  (let [root-id (:db/id root)
+        blocks (remove #(db-property/shape-block? repo db %) blocks)
+        parent-blocks (group-by #(get-in % [:block/parent :db/id]) blocks) ;; exclude whiteboard shapes
+        sort-fn (fn [parent]
+                  (ldb/sort-by-left (get parent-blocks parent) {:db/id parent}))
+        block-children (fn block-children [parent level]
+                         (map (fn [m]
+                                (let [id (:db/id m)
+                                      children (-> (block-children id (inc level))
+                                                   (ldb/sort-by-left m))]
+                                  (assoc m
+                                         :block/level level
+                                         :block/children children)))
+                           (sort-fn parent)))]
+    (block-children root-id 1)))
+
+(defn- get-root-and-page
+  [db root-id]
+  (if (string? root-id)
+    (if-let [id (parse-uuid root-id)]
+      [false (d/entity db [:block/uuid id])]
+      [true (d/entity db [:block/name (string/lower-case root-id)])])
+    [false root-id]))
+
+(defn blocks->vec-tree
+  "`blocks` need to be in the same page."
+  [repo db blocks root-id]
+  (let [[page? root] (get-root-and-page db (str root-id))]
+    (if-not root ; custom query
+      blocks
+      (let [result (blocks->vec-tree-aux repo db blocks root)]
+        (if page?
+          result
+           ;; include root block
+          (let [root-block (some #(when (= (:db/id %) (:db/id root)) %) blocks)
+                root-block (assoc root-block :block/children result)]
+            [root-block]))))))
+
+(defn- tree [parent->children root default-level]
+  (let [root-id (:db/id root)
+        nodes (fn nodes [parent-id level]
+                (mapv (fn [b]
+                        (let [b' (assoc b :block/level (inc level))
+                              children (nodes (:db/id b) (inc level))]
+                          (if (seq children)
+                            (assoc b' :block/children children)
+                            b')))
+                      (let [parent {:db/id parent-id}]
+                        (-> (get parent->children parent)
+                            (ldb/try-sort-by-left parent)))))
+        children (nodes root-id 1)
+        root' (assoc root :block/level (or default-level 1))]
+    (if (seq children)
+      (assoc root' :block/children children)
+      root')))
+
+(defn block-entity->map
+  [e]
+  (cond-> {:db/id (:db/id e)
+           :block/uuid (:block/uuid e)
+           :block/parent {:db/id (:db/id (:block/parent e))}
+           :block/page (:block/page e)}
+    (:db/id (:block/left e))
+    (assoc :block/left {:db/id (:db/id (:block/left e))})
+    (:block/refs e)
+    (assoc :block/refs (:block/refs e))
+    (:block/children e)
+    (assoc :block/children (:block/children e))))
+
+(defn filter-top-level-blocks
+  [blocks]
+  (let [id->blocks (zipmap (map :db/id blocks) blocks)]
+    (filter #(nil?
+              (id->blocks
+               (:db/id (:block/parent (id->blocks (:db/id %)))))) blocks)))
+
+(defn non-consecutive-blocks->vec-tree
+  "`blocks` need to be in the same page."
+  ([blocks]
+   (non-consecutive-blocks->vec-tree blocks 1))
+  ([blocks default-level]
+   (let [blocks (map block-entity->map blocks)
+         top-level-blocks (filter-top-level-blocks blocks)
+         top-level-blocks' (ldb/try-sort-by-left top-level-blocks (:block/parent (first top-level-blocks)))
+         parent->children (group-by :block/parent blocks)]
+     (map #(tree parent->children % (or default-level 1)) top-level-blocks'))))
+
+(defn- sort-blocks-aux
+  [parents parent-groups]
+  (mapv (fn [parent]
+          (let [parent-id {:db/id (:db/id parent)}
+                children (ldb/sort-by-left (get @parent-groups parent-id) parent)
+                _ (swap! parent-groups #(dissoc % parent-id))
+                sorted-nested-children (when (not-empty children) (sort-blocks-aux children parent-groups))]
+                    (if sorted-nested-children [parent sorted-nested-children] [parent])))
+        parents))
+
+(defn sort-blocks
+  "sort blocks by parent & left"
+  [blocks-exclude-root root]
+  (let [parent-groups (atom (group-by :block/parent blocks-exclude-root))]
+    (flatten (concat (sort-blocks-aux [root] parent-groups) (vals @parent-groups)))))
+
+(defn get-sorted-block-and-children
+  [repo db db-id]
+  (when db-id
+    (when-let [root-block (d/pull db db-id)]
+      (let [blocks (ldb/get-block-and-children repo db (:block/uuid root-block))
+            blocks-exclude-root (remove (fn [b] (= (:db/id b) db-id)) blocks)]
+        (sort-blocks blocks-exclude-root root-block)))))

src/main/frontend/config.cljs

@@ -10,7 +10,8 @@
             [logseq.graph-parser.util :as gp-util]
             [shadow.resource :as rc]
             [goog.crypt.Md5]
-            [goog.crypt :as crypt]))
+            [goog.crypt :as crypt]
+            [logseq.db.frontend.content :as db-content]))
 
 (goog-define DEV-RELEASE false)
 (defonce dev-release? DEV-RELEASE)
@@ -525,5 +526,3 @@
 (defn get-block-hidden-properties
   []
   (:block-hidden-properties (state/get-config)))
-
-(defonce page-ref-special-chars "~^")

src/main/frontend/db/model.cljs

@@ -24,42 +24,13 @@
             [cljs-time.format :as tf]
             ;; add map ops to datascript Entity
             [frontend.db.datascript.entity-plus :as entity-plus]
-            [frontend.config :as config]))
+            [frontend.config :as config]
+            [logseq.db :as ldb]))
 
 ;; TODO: extract to specific models and move data transform logic to the
 ;; corresponding handlers.
 
-;; Use it as an input argument for datalog queries
-(def block-attrs
-  '[:db/id
-    :block/uuid
-    :block/parent
-    :block/left
-    :block/collapsed?
-    :block/collapsed-properties
-    :block/format
-    :block/refs
-    :block/_refs
-    :block/path-refs
-    :block/tags
-    :block/link
-    :block/content
-    :block/marker
-    :block/priority
-    :block/properties
-    :block/properties-order
-    :block/properties-text-values
-    :block/pre-block?
-    :block/scheduled
-    :block/deadline
-    :block/repeated?
-    :block/created-at
-    :block/updated-at
-    ;; TODO: remove this in later releases
-    :block/heading-level
-    :block/file
-    {:block/page [:db/id :block/name :block/original-name :block/journal-day]}
-    {:block/_parent ...}])
+(def block-attrs ldb/block-attrs)
 
 (def get-original-name util/get-page-original-name)
 
@@ -415,24 +386,8 @@ independent of format as format specific heading characters are stripped"
   (when-let [page (db-utils/entity [:block/name (util/safe-page-name-sanity-lc page)])]
     (:block/properties page)))
 
-(defn sort-by-left
-  ([blocks parent]
-   (sort-by-left blocks parent {:check? true}))
-  ([blocks parent {:keys [_check?]}]
-   (let [blocks (util/distinct-by :db/id blocks)
-         left->blocks (reduce (fn [acc b] (assoc acc (:db/id (:block/left b)) b)) {} blocks)]
-     (loop [block parent
-            result []]
-       (if-let [next (get left->blocks (:db/id block))]
-         (recur next (conj result next))
-         (vec result))))))
-
-(defn try-sort-by-left
-  [blocks parent]
-  (let [result' (sort-by-left blocks parent {:check? false})]
-    (if (= (count result') (count blocks))
-      result'
-      blocks)))
+(def sort-by-left ldb/sort-by-left)
+(def try-sort-by-left ldb/try-sort-by-left)
 
 (defn sub-block
   [id]
@@ -772,27 +727,10 @@ independent of format as format specific heading characters are stripped"
        (let [ids' (map (fn [id] [:block/uuid id]) ids)]
          (db-utils/pull-many repo '[*] ids'))))))
 
-;; TODO: use the tree directly
-(defn- flatten-tree
-  [blocks-tree]
-  (if-let [children (:block/_parent blocks-tree)]
-    (cons (dissoc blocks-tree :block/_parent) (mapcat flatten-tree children))
-    [blocks-tree]))
-
-;; TODO: performance enhance
 (defn get-block-and-children
   [repo block-uuid]
-  (some-> (d/q
-           '[:find [(pull ?block ?block-attrs) ...]
-             :in $ ?id ?block-attrs
-             :where
-             [?block :block/uuid ?id]]
-           (conn/get-db repo)
-           block-uuid
-           block-attrs)
-          first
-          flatten-tree
-          (->> (map #(db-utils/update-block-content % (:db/id %))))))
+  (let [db (conn/get-db repo)]
+    (ldb/get-block-and-children repo db block-uuid)))
 
 (defn get-file-page
   ([file-path]

src/main/frontend/db/utils.cljs

@@ -6,8 +6,7 @@
             [frontend.db.conn :as conn]
             [frontend.config :as config]
             [logseq.graph-parser.util :as gp-util]
-            [clojure.string :as string]
-            [logseq.graph-parser.util.page-ref :as page-ref]))
+            [logseq.db.frontend.content :as db-content]))
 
 ;; transit serialization
 
@@ -59,42 +58,16 @@
                    repo-or-db)]
      (d/entity db id-or-lookup-ref))))
 
-(defn special-id->page
-  "Convert special id backs to page name."
-  [content refs]
-  (reduce
-   (fn [content ref]
-     (if (:block/name ref)
-       (string/replace content (str config/page-ref-special-chars (:block/uuid ref)) (:block/original-name ref))
-       content))
-   content
-   refs))
-
-(defn special-id-ref->page
-  "Convert special id ref backs to page name."
-  [content refs]
-  (reduce
-   (fn [content ref]
-     (if (:block/name ref)
-       (string/replace content
-                       (str page-ref/left-brackets
-                            config/page-ref-special-chars
-                            (:block/uuid ref)
-                            page-ref/right-brackets)
-                       (:block/original-name ref))
-       content))
-   content
-   refs))
+(def special-id->page db-content/special-id->page)
+
+(def special-id-ref->page db-content/special-id-ref->page)
 
 (defn update-block-content
   "Replace `[[internal-id]]` with `[[page name]]`"
   [item eid]
-  (if (config/db-based-graph? (state/get-current-repo))
-    (if-let [content (:block/content item)]
-      (let [refs (:block/refs (entity eid))]
-        (assoc item :block/content (special-id->page content refs)))
-      item)
-    item))
+  (let [repo (state/get-current-repo)
+        db (conn/get-db repo)]
+    (db-content/update-block-content repo db item eid)))
 
 (defn pull
   ([eid]

src/main/frontend/handler/db_based/editor.cljs

@@ -17,7 +17,8 @@
             [frontend.modules.outliner.core :as outliner-core]
             [frontend.modules.outliner.transaction :as outliner-tx]
             [frontend.schema.handler.repo-config :as repo-config-schema]
-            [promesa.core :as p]))
+            [promesa.core :as p]
+            [logseq.db.frontend.content :as db-content]))
 
 (defn- remove-non-existed-refs!
   [refs]
@@ -29,7 +30,7 @@
 
 (defn- replace-tag-ref
   [content page-name id]
-  (let [id' (str config/page-ref-special-chars id)
+  (let [id' (str db-content/page-ref-special-chars id)
         [page wrapped-id] (if (string/includes? page-name " ")
                             (map page-ref/->page-ref [page-name id'])
                             [page-name id'])
@@ -46,7 +47,7 @@
 
 (defn- replace-page-ref
   [content page-name id]
-  (let [id' (str config/page-ref-special-chars id)
+  (let [id' (str db-content/page-ref-special-chars id)
         [page wrapped-id] (map page-ref/->page-ref [page-name id'])]
         (util/replace-ignore-case content page wrapped-id)))
 
@@ -153,4 +154,4 @@
    {:outliner-op :save-block}
    (doseq [block-tx (keep #(set-heading-aux! % heading) block-ids)]
      (outliner-core/save-block! block-tx))
-   (property-handler/batch-set-block-property! repo block-ids :heading heading)))
+   (property-handler/batch-set-block-property! repo block-ids :heading heading)))

src/main/frontend/handler/db_based/page.cljs

@@ -10,7 +10,7 @@
             [frontend.handler.notification :as notification]
             [frontend.handler.route :as route-handler]
             [frontend.modules.outliner.core :as outliner-core]
-            [frontend.modules.outliner.tree :as outliner-tree]
+            [logseq.outliner.tree :as otree]
             [frontend.handler.common.page :as page-common-handler]
             [datascript.core :as d]
             [medley.core :as medley]
@@ -71,7 +71,7 @@
           from-id (:db/id from-page)
           from-first-child (some->> (db/pull from-id)
                                     (outliner-core/block)
-                                    (outliner-tree/-get-down)
+                                    (otree/-get-down)
                                     (outliner-core/get-data))
           to-last-direct-child-id (model/get-block-last-direct-child-id (db/get-db) to-id)
           repo (state/get-current-repo)

src/main/frontend/handler/dnd.cljs

@@ -3,7 +3,7 @@
   (:require [frontend.handler.editor :as editor-handler]
             [frontend.handler.property :as property-handler]
             [frontend.modules.outliner.core :as outliner-core]
-            [frontend.modules.outliner.tree :as tree]
+            [logseq.outliner.tree :as otree]
             [frontend.modules.outliner.transaction :as outliner-tx]
             [logseq.graph-parser.util.block-ref :as block-ref]
             [frontend.state :as state]
@@ -45,12 +45,12 @@
          (editor-handler/save-current-block!)
          (if top?
            (let [first-child?
-                 (= (tree/-get-parent-id target-node)
-                    (tree/-get-left-id target-node))]
+                 (= (otree/-get-parent-id target-node)
+                    (otree/-get-left-id target-node))]
              (if first-child?
-               (when-let [parent (tree/-get-parent target-node)]
+               (when-let [parent (otree/-get-parent target-node)]
                  (outliner-core/move-blocks! blocks' (:data parent) false))
-               (when-let [before-node (tree/-get-left target-node)]
+               (when-let [before-node (otree/-get-left target-node)]
                  (outliner-core/move-blocks! blocks' (:data before-node) true))))
            (outliner-core/move-blocks! blocks' target-block (not nested?)))))
 

src/main/frontend/handler/editor.cljs

@@ -34,6 +34,7 @@
             [frontend.modules.outliner.core :as outliner-core]
             [frontend.modules.outliner.transaction :as outliner-tx]
             [frontend.modules.outliner.tree :as tree]
+            [logseq.outliner.tree :as otree]
             [frontend.search :as search]
             [frontend.state :as state]
             [frontend.template :as template]
@@ -770,7 +771,7 @@
             :else
             (let [has-children? (seq (:block/_parent block-e))
                   block (db/pull (:db/id block-e))
-                  left (tree/-get-left (outliner-core/block block))
+                  left (otree/-get-left (outliner-core/block block))
                   left-has-children? (and left
                                           (when-let [block-id (:block/uuid (:data left))]
                                             (let [block (db/entity [:block/uuid block-id])]
@@ -2255,7 +2256,7 @@
 (defn outdent-on-enter
   [node]
   (let [original-block (outliner-core/get-current-editing-original-block)
-        parent-node (tree/-get-parent node)
+        parent-node (otree/-get-parent node)
         target (or original-block (:data parent-node))
         pos (state/get-edit-pos)
         block (:data node)]
@@ -2274,7 +2275,7 @@
     (when-let [entity (if-let [id' (parse-uuid (str id))]
                         (db/entity [:block/uuid id'])
                         (db/entity [:block/name (util/page-name-sanity-lc id)]))]
-      (= (:block/uuid entity) (tree/-get-parent-id current-node)))))
+      (= (:block/uuid entity) (otree/-get-parent-id current-node)))))
 
 (defn insert
   ([insertion]
@@ -2471,7 +2472,7 @@
               content (gobj/get input "value")
               pos (cursor/pos input)
               current-node (outliner-core/block block)
-              has-right? (-> (tree/-get-right current-node)
+              has-right? (-> (otree/-get-right current-node)
                              (tree/satisfied-inode?))
               db-based? (config/db-based-graph? (state/get-current-repo))
               thing-at-point ;intern is not supported in cljs, need a more elegant solution

src/main/frontend/handler/file_based/page.cljs

@@ -18,7 +18,7 @@
             [frontend.util.fs :as fs-util]
             [frontend.modules.outliner.core :as outliner-core]
             [frontend.modules.outliner.file :as outliner-file]
-            [frontend.modules.outliner.tree :as outliner-tree]
+            [logseq.outliner.tree :as otree]
             [frontend.fs :as fs]
             [logseq.graph-parser.property :as gp-property]
             [logseq.graph-parser.util.page-ref :as page-ref]
@@ -202,7 +202,7 @@
       (let [old-original-name   (:block/original-name page)
             file                (:block/file page)
             journal?            (:block/journal? page)
-            properties-block    (:data (outliner-tree/-get-down (outliner-core/block page)))
+            properties-block    (:data (otree/-get-down (outliner-core/block page)))
             properties-content  (:block/content properties-block)
             properties-block-tx (when (and properties-block
                                            properties-content
@@ -320,7 +320,7 @@
           from-id (:db/id from-page)
           from-first-child (some->> (db/pull from-id)
                                     (outliner-core/block)
-                                    (outliner-tree/-get-down)
+                                    (otree/-get-down)
                                     (outliner-core/get-data))
           to-last-direct-child-id (model/get-block-last-direct-child-id (db/get-db) to-id)
           repo (state/get-current-repo)

src/main/frontend/handler/property/util.cljs

@@ -6,26 +6,22 @@
             [frontend.state :as state]
             [logseq.graph-parser.util :as gp-util]
             [frontend.db :as db]
-            [frontend.util :as util]))
+            [frontend.util :as util]
+            [logseq.db.frontend.property :as db-property]))
 
 (defn lookup
   "Get the value of coll's (a map) `key`. For file and db graphs"
   [coll key]
   (let [repo (state/get-current-repo)
-        property-name (if (keyword? key)
-                        (name key)
-                        key)]
-    (if (config/db-based-graph? repo)
-      (when-let [property (db/entity repo [:block/name (gp-util/page-name-sanity-lc property-name)])]
-        (get coll (:block/uuid property)))
-      (get coll key))))
+        db (db/get-db repo)]
+    (db-property/lookup repo db coll key)))
 
 (defn get-property
   "Get the value of block's property `key`"
   [block key]
-  (let [block (or (db/entity (:db/id block)) block)]
-    (when-let [properties (:block/properties block)]
-      (lookup properties key))))
+  (let [repo (state/get-current-repo)
+        db (db/get-db repo)]
+    (db-property/get-property repo db block key)))
 
 (defn get-page-uuid
   "Get a user property's uuid given its unsanitized name"
@@ -48,5 +44,8 @@
 (defn page-block->tldr-page [block]
   (get-property block :logseq.tldraw.page))
 
-(defn shape-block? [block]
-  (= :whiteboard-shape (get-property block :ls-type)))
+(defn shape-block?
+  [block]
+  (let [repo (state/get-current-repo)
+        db (db/get-db repo)]
+    (db-property/shape-block? repo db block)))

src/main/frontend/modules/outliner/core.cljs

@@ -8,7 +8,7 @@
             [frontend.db.conn :as conn]
             [frontend.db.outliner :as db-outliner]
             [frontend.modules.outliner.datascript :as ds]
-            [frontend.modules.outliner.tree :as tree]
+            [logseq.outliner.tree :as otree]
             [frontend.modules.outliner.utils :as outliner-u]
             [frontend.state :as state]
             [frontend.util :as util]
@@ -249,7 +249,7 @@
 ;; the :block/parent, :block/left should be datascript lookup ref
 
 (extend-type Block
-  tree/INode
+  otree/INode
   (-get-id [this]
     (or
      (when-let [block-id (get-in this [:data :block/uuid])]
@@ -276,20 +276,20 @@
     (update this :data assoc :block/left [:block/uuid left-id]))
 
   (-get-parent [this]
-    (when-let [parent-id (tree/-get-parent-id this)]
+    (when-let [parent-id (otree/-get-parent-id this)]
       (get-block-by-id parent-id)))
 
   (-get-left [this]
-    (let [left-id (tree/-get-left-id this)]
+    (let [left-id (otree/-get-left-id this)]
       (get-block-by-id left-id)))
 
   (-get-right [this]
-    (let [left-id (tree/-get-id this)
-          parent-id (tree/-get-parent-id this)]
+    (let [left-id (otree/-get-id this)
+          parent-id (otree/-get-parent-id this)]
       (get-by-parent-&-left parent-id left-id)))
 
   (-get-down [this]
-    (let [parent-id (tree/-get-id this)]
+    (let [parent-id (otree/-get-id this)]
       (get-by-parent-&-left parent-id parent-id)))
 
   (-save [this txs-state]
@@ -359,7 +359,7 @@
   (-del [this txs-state children?]
     (assert (ds/outliner-txs-state? txs-state)
             "db should be satisfied outliner-tx-state?")
-    (let [block-id (tree/-get-id this)
+    (let [block-id (otree/-get-id this)
           ids (set (if children?
                      (let [children (db/get-block-children (state/get-current-repo) block-id)
                            children-ids (map :block/uuid children)]
@@ -369,7 +369,7 @@
           txs (if-not children?
                 (let [immediate-children (db/get-block-immediate-children (state/get-current-repo) block-id)]
                   (if (seq immediate-children)
-                    (let [left-id (tree/-get-id (tree/-get-left this))]
+                    (let [left-id (otree/-get-id (otree/-get-left this))]
                       (concat txs
                               (map-indexed (fn [idx child]
                                              (let [parent [:block/uuid left-id]]
@@ -393,7 +393,7 @@
       block-id))
 
   (-get-children [this]
-    (let [parent-id (tree/-get-id this)
+    (let [parent-id (otree/-get-id this)
           children (db-model/get-block-immediate-children (state/get-current-repo) parent-id)]
       (map block children))))
 
@@ -496,15 +496,15 @@
 
 (defn- get-left-nodes
   [node limit]
-  (let [parent (tree/-get-parent node)]
+  (let [parent (otree/-get-parent node)]
     (loop [node node
            limit limit
            result []]
       (if (zero? limit)
         result
-        (if-let [left (tree/-get-left node)]
+        (if-let [left (otree/-get-left node)]
           (if-not (= left parent)
-            (recur left (dec limit) (conj result (tree/-get-id left)))
+            (recur left (dec limit) (conj result (otree/-get-id left)))
             result)
           result)))))
 
@@ -539,7 +539,7 @@
   [block']
   {:pre [(map? block')]}
   (let [txs-state (atom [])]
-    (tree/-save (block block') txs-state)
+    (otree/-save (block block') txs-state)
     {:tx-data @txs-state}))
 
 (defn blocks-with-level
@@ -613,16 +613,16 @@
 (defn- get-right-siblings
   "Get `node`'s right siblings."
   [node]
-  {:pre [(tree/satisfied-inode? node)]}
-  (when-let [parent (tree/-get-parent node)]
-    (let [children (tree/-get-children parent)]
-      (->> (split-with #(not= (tree/-get-id node) (tree/-get-id %)) children)
+  {:pre [(otree/satisfied-inode? node)]}
+  (when-let [parent (otree/-get-parent node)]
+    (let [children (otree/-get-children parent)]
+      (->> (split-with #(not= (otree/-get-id node) (otree/-get-id %)) children)
            last
            rest))))
 
 (defn- blocks-with-ordered-list-props
   [blocks target-block sibling?]
-  (let [target-block (if sibling? target-block (some-> target-block :db/id db/pull block tree/-get-down :data))
+  (let [target-block (if sibling? target-block (some-> target-block :db/id db/pull block otree/-get-down :data))
         list-type-fn (fn [block] (pu/get-property block :logseq.order-list-type))
         k (pu/get-pid :logseq.order-list-type)]
     (if-let [list-type (and target-block (list-type-fn target-block))]
@@ -821,12 +821,12 @@
                  (assign-temp-id tx replace-empty-target? target-block'))
             target-node (block target-block')
             next (if sibling?
-                   (tree/-get-right target-node)
-                   (tree/-get-down target-node))
+                   (otree/-get-right target-node)
+                   (otree/-get-down target-node))
             next-tx (when (and next
                                (if move? (not (contains? (set (map :db/id blocks)) (:db/id (:data next)))) true))
                       (when-let [left (last (filter (fn [b] (= 1 (:block/level b))) tx))]
-                        [{:block/uuid (tree/-get-id next)
+                        [{:block/uuid (otree/-get-id next)
                           :block/left (:db/id left)}]))
             full-tx (util/concat-without-nil (if (and keep-uuid? replace-empty-target?) (rest uuids-tx) uuids-tx) tx next-tx)]
         (when (and replace-empty-target? (state/editing?))
@@ -891,12 +891,12 @@
            (first (:block/_parent (db/entity [:block/uuid (:block/uuid (get-data node))]))))
     (throw (ex-info "Block can't be deleted because it still has children left, you can pass `children?` equals to `true`."
                     {:block (get-data node)}))
-    (let [right-node (tree/-get-right node)]
-      (tree/-del node txs-state children?)
-      (when (tree/satisfied-inode? right-node)
-        (let [left-node (tree/-get-left node)
-              new-right-node (tree/-set-left-id right-node (tree/-get-id left-node))]
-          (tree/-save new-right-node txs-state)))
+    (let [right-node (otree/-get-right node)]
+      (otree/-del node txs-state children?)
+      (when (otree/satisfied-inode? right-node)
+        (let [left-node (otree/-get-left node)
+              new-right-node (otree/-set-left-id right-node (otree/-get-id left-node))]
+          (otree/-save new-right-node txs-state)))
       @txs-state)))
 
 (defn- delete-blocks
@@ -917,28 +917,28 @@
         end-node-parents (->>
                           (db/get-block-parents
                            (state/get-current-repo)
-                           (tree/-get-id end-node)
+                           (otree/-get-id end-node)
                            {:depth 1000})
                           (map :block/uuid)
                           (set))
-        self-block? (contains? end-node-parents (tree/-get-id start-node))]
+        self-block? (contains? end-node-parents (otree/-get-id start-node))]
     (if (or
          (= 1 (count blocks))
          (= start-node end-node)
          self-block?)
       (delete-block txs-state start-node (assoc delete-opts :children? children?))
-      (let [sibling? (= (tree/-get-parent-id start-node)
-                        (tree/-get-parent-id end-node))
-            right-node (tree/-get-right end-node)]
-        (when (tree/satisfied-inode? right-node)
+      (let [sibling? (= (otree/-get-parent-id start-node)
+                        (otree/-get-parent-id end-node))
+            right-node (otree/-get-right end-node)]
+        (when (otree/satisfied-inode? right-node)
           (let [non-consecutive? (seq (db-model/get-non-consecutive-blocks blocks))
                 left-node-id (if sibling?
-                               (tree/-get-id (tree/-get-left start-node))
+                               (otree/-get-id (otree/-get-left start-node))
                                (let [end-node-left-nodes (get-left-nodes end-node (count block-ids))
                                      parents (->>
                                               (db/get-block-parents
                                                (state/get-current-repo)
-                                               (tree/-get-id start-node)
+                                               (otree/-get-id start-node)
                                                {:depth 1000})
                                               (map :block/uuid)
                                               (set))
@@ -950,15 +950,15 @@
             (when (and (nil? left-node-id) (not non-consecutive?))
               (assert left-node-id
                       (str "Can't find the left-node-id: "
-                           (pr-str {:start (db/entity [:block/uuid (tree/-get-id start-node)])
-                                    :end (db/entity [:block/uuid (tree/-get-id end-node)])
-                                    :right-node (db/entity [:block/uuid (tree/-get-id right-node)])}))))
+                           (pr-str {:start (db/entity [:block/uuid (otree/-get-id start-node)])
+                                    :end (db/entity [:block/uuid (otree/-get-id end-node)])
+                                    :right-node (db/entity [:block/uuid (otree/-get-id right-node)])}))))
             (when left-node-id
-              (let [new-right-node (tree/-set-left-id right-node left-node-id)]
-                (tree/-save new-right-node txs-state)))))
+              (let [new-right-node (otree/-set-left-id right-node left-node-id)]
+                (otree/-save new-right-node txs-state)))))
         (doseq [id block-ids]
           (let [node (block (db/pull id))]
-            (tree/-del node txs-state true)))
+            (otree/-del node txs-state true)))
         (let [fix-non-consecutive-tx (fix-non-consecutive-blocks blocks nil false)]
           (swap! txs-state concat fix-non-consecutive-tx))))
     {:tx-data @txs-state}))

src/main/frontend/modules/outliner/datascript.cljs

@@ -54,6 +54,8 @@
       (db/transact! (:repo opts) replace-tx-data (:replace-tx-meta opts)))
 
     (pipelines/invoke-hooks opts)
+
+    ;; TODO: move validate to worker
     ;; (when (and config/dev?
     ;;            (config/db-based-graph? (state/get-current-repo))
     ;;             ;; Skip tx with update-tx-ids? because they are immediately followed by the original block tx

src/main/frontend/modules/outliner/tree.cljs

@@ -1,138 +1,26 @@
 (ns frontend.modules.outliner.tree
   (:require [frontend.db :as db]
-            [frontend.db.model :as model]
-            [clojure.string :as string]
             [frontend.state :as state]
-            [frontend.handler.property.util :as pu]))
+            [logseq.outliner.tree :as otree]))
 
-(defprotocol INode
-  (-get-id [this])
-  (-get-parent-id [this])
-  (-get-left-id [this])
-  (-set-left-id [this left-id])
-  (-get-parent [this])
-  (-get-left [this])
-  (-get-right [this])
-  (-get-down [this])
-  (-save [this db])
-  (-del [this db children?])
-  (-get-children [this]))
-
-(defn satisfied-inode?
-  [node]
-  (satisfies? INode node))
-
-(defn- blocks->vec-tree-aux
-  [blocks root]
-  (let [root-id (:db/id root)
-        blocks (remove pu/shape-block? blocks)
-        parent-blocks (group-by #(get-in % [:block/parent :db/id]) blocks) ;; exclude whiteboard shapes
-        sort-fn (fn [parent]
-                  (db/sort-by-left (get parent-blocks parent) {:db/id parent}))
-        block-children (fn block-children [parent level]
-                         (map (fn [m]
-                                (let [id (:db/id m)
-                                      children (-> (block-children id (inc level))
-                                                   (db/sort-by-left m))]
-                                  (assoc m
-                                         :block/level level
-                                         :block/children children)))
-                           (sort-fn parent)))]
-    (block-children root-id 1)))
-
-(defn- get-root-and-page
-  [repo root-id]
-  (if (string? root-id)
-    (if-let [id (parse-uuid root-id)]
-      [false (db/entity repo [:block/uuid id])]
-      [true (db/entity repo [:block/name (string/lower-case root-id)])])
-    [false root-id]))
+(def satisfied-inode? otree/satisfied-inode?)
 
 (defn blocks->vec-tree
   "`blocks` need to be in the same page."
   ([blocks root-id]
    (blocks->vec-tree (state/get-current-repo) blocks root-id))
   ([repo blocks root-id]
-   (let [[page? root] (get-root-and-page repo (str root-id))]
-     (if-not root ; custom query
-       blocks
-       (let [result (blocks->vec-tree-aux blocks root)]
-         (if page?
-           result
-           ;; include root block
-           (let [root-block (some #(when (= (:db/id %) (:db/id root)) %) blocks)
-                 root-block (assoc root-block :block/children result)]
-             [root-block])))))))
-
-(defn- tree [parent->children root default-level]
-  (let [root-id (:db/id root)
-        nodes (fn nodes [parent-id level]
-                (mapv (fn [b]
-                        (let [b' (assoc b :block/level (inc level))
-                              children (nodes (:db/id b) (inc level))]
-                          (if (seq children)
-                            (assoc b' :block/children children)
-                            b')))
-                      (let [parent {:db/id parent-id}]
-                        (-> (get parent->children parent)
-                            (model/try-sort-by-left parent)))))
-        children (nodes root-id 1)
-        root' (assoc root :block/level (or default-level 1))]
-    (if (seq children)
-      (assoc root' :block/children children)
-      root')))
+   (let [db (db/get-db repo)]
+     (otree/blocks->vec-tree repo db blocks root-id))))
 
-(defn block-entity->map
-  [e]
-  (cond-> {:db/id (:db/id e)
-           :block/uuid (:block/uuid e)
-           :block/parent {:db/id (:db/id (:block/parent e))}
-           :block/page (:block/page e)}
-    (:db/id (:block/left e))
-    (assoc :block/left {:db/id (:db/id (:block/left e))})
-    (:block/refs e)
-    (assoc :block/refs (:block/refs e))
-    (:block/children e)
-    (assoc :block/children (:block/children e))))
-
-(defn filter-top-level-blocks
-  [blocks]
-  (let [id->blocks (zipmap (map :db/id blocks) blocks)]
-    (filter #(nil?
-              (id->blocks
-               (:db/id (:block/parent (id->blocks (:db/id %)))))) blocks)))
-
-(defn non-consecutive-blocks->vec-tree
-  "`blocks` need to be in the same page."
-  ([blocks]
-   (non-consecutive-blocks->vec-tree blocks 1))
-  ([blocks default-level]
-   (let [blocks (map block-entity->map blocks)
-         top-level-blocks (filter-top-level-blocks blocks)
-         top-level-blocks' (model/try-sort-by-left top-level-blocks (:block/parent (first top-level-blocks)))
-         parent->children (group-by :block/parent blocks)]
-     (map #(tree parent->children % (or default-level 1)) top-level-blocks'))))
+(def block-entity->map otree/block-entity->map)
 
-(defn- sort-blocks-aux
-  [parents parent-groups]
-  (mapv (fn [parent]
-          (let [parent-id {:db/id (:db/id parent)}
-                children (db/sort-by-left (get @parent-groups parent-id) parent)
-                _ (swap! parent-groups #(dissoc % parent-id))
-                sorted-nested-children (when (not-empty children) (sort-blocks-aux children parent-groups))]
-                    (if sorted-nested-children [parent sorted-nested-children] [parent])))
-        parents))
+(def filter-top-level-blocks otree/filter-top-level-blocks)
 
-(defn sort-blocks
-  "sort blocks by parent & left"
-  [blocks-exclude-root root]
-  (let [parent-groups (atom (group-by :block/parent blocks-exclude-root))]
-    (flatten (concat (sort-blocks-aux [root] parent-groups) (vals @parent-groups)))))
+(def non-consecutive-blocks->vec-tree otree/non-consecutive-blocks->vec-tree)
+(def sort-blocks otree/sort-blocks)
 
 (defn get-sorted-block-and-children
   [repo db-id]
-  (when db-id
-    (when-let [root-block (db/pull db-id)]
-      (let [blocks (db/get-block-and-children repo (:block/uuid root-block))
-            blocks-exclude-root (remove (fn [b] (= (:db/id b) db-id)) blocks)]
-        (sort-blocks blocks-exclude-root root-block)))))
+  (let [db (db/get-db repo)]
+    (otree/get-sorted-block-and-children repo db db-id)))

src/main/frontend/worker/file.cljs

@@ -0,0 +1,118 @@
+(ns frontend.worker.file
+  "Save pages to files for file-based graphs"
+  (:require [clojure.core.async :as async]
+            [clojure.string :as string]
+            [frontend.config :as config]
+            [frontend.db :as db]
+            [frontend.db.model :as model]
+            [frontend.handler.notification :as notification]
+            [frontend.modules.file.core :as file]
+            [frontend.modules.outliner.tree :as tree]
+            [frontend.util :as util]
+            [goog.object :as gobj]
+            [lambdaisland.glogi :as log]
+            [frontend.state :as state]
+            [cljs-time.core :as t]
+            [cljs-time.coerce :as tc]))
+
+(def batch-write-interval 1000)
+
+(def whiteboard-blocks-pull-keys-with-persisted-ids
+  '[:block/properties
+    :block/uuid
+    :block/content
+    :block/format
+    :block/created-at
+    :block/updated-at
+    :block/collapsed?
+    {:block/page      [:block/uuid]}
+    {:block/left      [:block/uuid]}
+    {:block/parent    [:block/uuid]}])
+
+(defn- cleanup-whiteboard-block
+  [block]
+  (if (get-in block [:block/properties :ls-type] false)
+    (dissoc block
+            :db/id
+            :block/uuid ;; shape block uuid is read from properties
+            :block/collapsed?
+            :block/content
+            :block/format
+            :block/left
+            :block/page
+            :block/parent) ;; these are auto-generated for whiteboard shapes
+    (dissoc block :db/id :block/page)))
+
+
+(defn do-write-file!
+  [repo page-db-id outliner-op]
+  (let [page-block (db/pull repo '[*] page-db-id)
+        page-db-id (:db/id page-block)
+        whiteboard? (contains? (:block/type page-block) "whiteboard")
+        blocks-count (model/get-page-blocks-count repo page-db-id)
+        blocks-just-deleted? (and (zero? blocks-count)
+                                  (contains? #{:delete-blocks :move-blocks} outliner-op))]
+    (when (or (>= blocks-count 1) blocks-just-deleted?)
+      (if (or (and (> blocks-count 500)
+                   (not (state/input-idle? repo {:diff 3000}))) ;; long page
+              ;; when this whiteboard page is just being updated
+              (and whiteboard? (not (state/whiteboard-idle? repo))))
+        (async/put! (state/get-file-write-chan) [repo page-db-id outliner-op (tc/to-long (t/now))])
+        (let [pull-keys (if whiteboard? whiteboard-blocks-pull-keys-with-persisted-ids '[*])
+              blocks (model/get-page-blocks-no-cache repo (:block/name page-block) {:pull-keys pull-keys})
+              blocks (if whiteboard? (map cleanup-whiteboard-block blocks) blocks)]
+          (when-not (and (= 1 (count blocks))
+                         (string/blank? (:block/content (first blocks)))
+                         (nil? (:block/file page-block)))
+            (let [tree-or-blocks (if whiteboard? blocks
+                                     (tree/blocks->vec-tree repo blocks (:block/name page-block)))]
+              (if page-block
+                (file/save-tree! page-block tree-or-blocks blocks-just-deleted?)
+                (js/console.error (str "can't find page id: " page-db-id))))))))))
+
+(defn write-files!
+  [pages]
+  (when (seq pages)
+    (when-not config/publishing?
+      (doseq [[repo page-id outliner-op] (set (map #(take 3 %) pages))] ; remove time to dedupe pages to write
+        (try (do-write-file! repo page-id outliner-op)
+             (catch :default e
+               (notification/show!
+                [:div
+                 [:p "Write file failed, please copy the changes to other editors in case of losing data."]
+                 "Error: " (str (gobj/get e "stack"))]
+                :error)
+               (log/error :file/write-file-error {:error e})))))))
+
+(defn sync-to-file
+  ([page]
+   (sync-to-file page nil))
+  ([{page-db-id :db/id} outliner-op]
+   (if (nil? page-db-id)
+     (notification/show!
+      "Write file failed, can't find the current page!"
+      :error)
+     (when-let [repo (state/get-current-repo)]
+       (if (:graph/importing @state/state) ; write immediately
+         (write-files! [[repo page-db-id outliner-op]])
+         (async/put! (state/get-file-write-chan) [repo page-db-id outliner-op (tc/to-long (t/now))]))))))
+
+(def *writes-finished? (atom {}))
+
+(defn <ratelimit-file-writes!
+  []
+  (util/<ratelimit (state/get-file-write-chan) batch-write-interval
+                   :filter-fn
+                   (fn [[repo _ _ time]]
+                     (swap! *writes-finished? assoc repo {:time time
+                                                          :value false})
+                     true)
+                   :flush-fn
+                   (fn [col]
+                     (let [start-time (tc/to-long (t/now))
+                           repos (distinct (map first col))]
+                       (write-files! col)
+                       (doseq [repo repos]
+                         (let [last-write-time (get-in @*writes-finished? [repo :time])]
+                           (when (> start-time last-write-time)
+                             (swap! *writes-finished? assoc repo {:value true}))))))))

src/main/frontend/worker/util.cljs

@@ -48,10 +48,10 @@
   gp-util/page-name-sanity-lc)
 
 (defn safe-page-name-sanity-lc
-     [s]
-     (if (string? s)
-       (page-name-sanity-lc s) s))
+  [s]
+  (if (string? s)
+    (page-name-sanity-lc s) s))
 
 (defn distinct-by
-     [f col]
-     (medley/distinct-by f (seq col)))
+  [f col]
+  (medley/distinct-by f (seq col)))

src/test/frontend/modules/outliner/core_test.cljs

@@ -4,6 +4,7 @@
             [frontend.test.fixtures :as fixtures]
             [frontend.modules.outliner.core :as outliner-core]
             [frontend.modules.outliner.tree :as tree]
+            [logseq.outliner.tree :as otree]
             [frontend.modules.outliner.transaction :as outliner-tx]
             [frontend.db :as db]
             [frontend.db.model :as db-model]
@@ -105,7 +106,7 @@
 (defn get-children
   [id]
   (->> (get-block id true)
-       (tree/-get-children)
+       (otree/-get-children)
        (mapv #(-> % :data :block/uuid))))
 
 (deftest test-delete-block