Move recalculate-block-path-refs to outliner dep

Moved dependent query fns also into the outliner dep. Removed
get-block-children-ids from model as there was only one use of it after
the refactor. Also setup testing for outliner and moved in a test for it

deps/outliner/nbb.edn

@@ -1,4 +1,6 @@
 {:paths ["src"]
  :deps
  {logseq/db
-  {:local/root "../db"}}}
+  {:local/root "../db"}
+  io.github.nextjournal/nbb-test-runner
+  {:git/sha "60ed57aa04bca8d604f5ba6b28848bd887109347"}}}

deps/outliner/package.json

@@ -0,0 +1,11 @@
+{
+  "name": "@logseq/outliner",
+  "version": "1.0.0",
+  "private": true,
+  "devDependencies": {
+    "@logseq/nbb-logseq": "^1.2.173"
+  },
+  "scripts": {
+    "test": "yarn nbb-logseq -cp test -m nextjournal.test-runner"
+  }
+}

deps/outliner/src/logseq/outliner/pipeline.cljs

@@ -2,7 +2,8 @@
   "Core fns for use with frontend.modules.outliner.pipeline"
   (:require [logseq.db.schema :as db-schema]
             [datascript.core :as d]
-            [cognitect.transit :as t]))
+            [cognitect.transit :as t]
+            [clojure.set :as set]))
 
 (defn filter-deleted-blocks
   [datoms]
@@ -47,4 +48,99 @@
                   b)))
          (map (fn [b]
                 (let [uuid (or (:block/uuid b) (random-uuid))]
-                  (assoc b :block/uuid uuid)))))))
+                  (assoc b :block/uuid uuid)))))))
+
+;; non recursive query
+(defn get-block-parents
+  [db block-id {:keys [depth] :or {depth 100}}]
+  (loop [block-id block-id
+         parents (list)
+         d 1]
+    (if (> d depth)
+      parents
+      (if-let [parent (:block/parent (d/entity db [:block/uuid block-id]))]
+        (recur (:block/uuid parent) (conj parents parent) (inc d))
+        parents))))
+
+(defn get-block-children-ids
+  [db block-uuid]
+  (when-let [eid (:db/id (d/entity db [:block/uuid block-uuid]))]
+    (let [seen   (volatile! [])]
+      (loop [steps          100      ;check result every 100 steps
+             eids-to-expand [eid]]
+        (when (seq eids-to-expand)
+          (let [eids-to-expand*
+                (mapcat (fn [eid] (map first (d/datoms db :avet :block/parent eid))) eids-to-expand)
+                uuids-to-add (remove nil? (map #(:block/uuid (d/entity db %)) eids-to-expand*))]
+            (when (and (zero? steps)
+                       (seq (set/intersection (set @seen) (set uuids-to-add))))
+              (throw (ex-info "bad outliner data, need to re-index to fix"
+                              {:seen @seen :eids-to-expand eids-to-expand})))
+            (vswap! seen (partial apply conj) uuids-to-add)
+            (recur (if (zero? steps) 100 (dec steps)) eids-to-expand*))))
+      @seen)))
+
+;; TODO: it'll be great if we can calculate the :block/path-refs before any
+;; outliner transaction, this way we can group together the real outliner tx
+;; and the new path-refs changes, which makes both undo/redo and
+;; react-query/refresh! easier.
+
+;; TODO: also need to consider whiteboard transactions
+
+;; Steps:
+;; 1. For each changed block, new-refs = its page + :block/refs + parents :block/refs
+;; 2. Its children' block/path-refs might need to be updated too.
+(defn computer-block-path-refs
+  [{:keys [db-before db-after]} blocks*]
+  (let [blocks (remove :block/name blocks*)
+        *computed-ids (atom #{})]
+    (mapcat (fn [block]
+              (when (and (not (@*computed-ids (:block/uuid block))) ; not computed yet
+                         (not (:block/name block)))
+                (let [parents (get-block-parents db-after (:block/uuid block) {})
+                      parents-refs (->> (mapcat :block/path-refs parents)
+                                        (map :db/id))
+                      old-refs (if db-before
+                                 (set (map :db/id (:block/path-refs (d/entity db-before (:db/id block)))))
+                                 #{})
+                      new-refs (set (concat
+                                     (some-> (:db/id (:block/page block)) vector)
+                                     (map :db/id (:block/refs block))
+                                     parents-refs))
+                      refs-changed? (not= old-refs new-refs)
+                      children (get-block-children-ids db-after (:block/uuid block))
+                            ;; Builds map of children ids to their parent id and :block/refs ids
+                      children-maps (into {}
+                                          (map (fn [id]
+                                                 (let [entity (d/entity db-after [:block/uuid id])]
+                                                   [(:db/id entity)
+                                                    {:parent-id (get-in entity [:block/parent :db/id])
+                                                     :block-ref-ids (map :db/id (:block/refs entity))}]))
+                                               children))
+                      children-refs (map (fn [[id {:keys [block-ref-ids] :as child-map}]]
+                                           {:db/id id
+                                                  ;; Recalculate :block/path-refs as db contains stale data for this attribute
+                                            :block/path-refs
+                                            (set/union
+                                                   ;; Refs from top-level parent
+                                             new-refs
+                                                   ;; Refs from current block
+                                             block-ref-ids
+                                                   ;; Refs from parents in between top-level
+                                                   ;; parent and current block
+                                             (loop [parent-refs #{}
+                                                    parent-id (:parent-id child-map)]
+                                               (if-let [parent (children-maps parent-id)]
+                                                 (recur (into parent-refs (:block-ref-ids parent))
+                                                        (:parent-id parent))
+                                                       ;; exits when top-level parent is reached
+                                                 parent-refs)))})
+                                         children-maps)]
+                  (swap! *computed-ids set/union (set (cons (:block/uuid block) children)))
+                  (concat
+                   (when (and (seq new-refs)
+                              refs-changed?)
+                     [{:db/id (:db/id block)
+                       :block/path-refs new-refs}])
+                   children-refs))))
+            blocks)))

deps/outliner/test/logseq/outliner/pipeline_test.cljs

@@ -0,0 +1,29 @@
+(ns logseq.outliner.pipeline-test
+  (:require [cljs.test :refer [deftest is]]
+            [logseq.db.schema :as db-schema]
+            [datascript.core :as d]
+            [logseq.outliner.pipeline :as outliner-pipeline]))
+
+
+;;; datoms
+;;; - 1 <----+
+;;;   - 2    |
+;;;     - 3 -+
+(def broken-outliner-data-with-cycle
+  [{:db/id 1
+    :block/uuid #uuid"e538d319-48d4-4a6d-ae70-c03bb55b6fe4"
+    :block/parent 3}
+   {:db/id 2
+    :block/uuid #uuid"c46664c0-ea45-4998-adf0-4c36486bb2e5"
+    :block/parent 1}
+   {:db/id 3
+    :block/uuid #uuid"2b736ac4-fd49-4e04-b00f-48997d2c61a2"
+    :block/parent 2}])
+
+(deftest get-block-children-ids-on-bad-outliner-data
+  (let [db (d/db-with (d/empty-db db-schema/schema)
+                      broken-outliner-data-with-cycle)]
+    (is (= "bad outliner data, need to re-index to fix"
+           (try (outliner-pipeline/get-block-children-ids db #uuid "e538d319-48d4-4a6d-ae70-c03bb55b6fe4")
+                (catch :default e
+                  (ex-message e)))))))

deps/outliner/yarn.lock

@@ -0,0 +1,15 @@
+# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+
+"@logseq/nbb-logseq@^1.2.173":
+  version "1.2.173"
+  resolved "https://registry.yarnpkg.com/@logseq/nbb-logseq/-/nbb-logseq-1.2.173.tgz#27a52c350f06ac9c337d73687738f6ea8b2fc3f3"
+  integrity sha512-ABKPtVnSOiS4Zpk9+UTaGcs5H6EUmRADr9FJ0aEAVpa0WfAyvUbX/NgkQGMe1kKRv3EbIuLwaxfy+txr31OtAg==
+  dependencies:
+    import-meta-resolve "^2.1.0"
+
+import-meta-resolve@^2.1.0:
+  version "2.2.2"
+  resolved "https://registry.yarnpkg.com/import-meta-resolve/-/import-meta-resolve-2.2.2.tgz#75237301e72d1f0fbd74dbc6cca9324b164c2cc9"
+  integrity sha512-f8KcQ1D80V7RnqVm+/lirO9zkOxjGxhaTC1IPrBGd3MEfNgmNG67tSUO9gTi2F3Blr2Az6g1vocaxzkVnWl9MA==

src/main/frontend/components/block.cljs

@@ -2590,7 +2590,7 @@
                               level-limit 3}
                          :as opts}]
   (when block-id
-    (let [parents (db/get-block-parents repo block-id (inc level-limit))
+    (let [parents (db/get-block-parents repo block-id {:depth (inc level-limit)})
           page (or (db/get-block-page repo block-id) ;; only return for block uuid
                    (model/query-block-by-uuid block-id)) ;; return page entity when received page uuid
           page-name (:block/name page)

src/main/frontend/db.cljs

@@ -33,7 +33,7 @@
   delete-files delete-pages-by-files get-all-block-contents get-all-tagged-pages
   get-all-templates get-block-and-children get-block-by-uuid get-block-children sort-by-left
   get-block-parent get-block-parents parents-collapsed? get-block-referenced-blocks get-all-referenced-blocks-uuid
-  get-block-children-ids get-block-immediate-children get-block-page
+  get-block-immediate-children get-block-page
   get-custom-css get-date-scheduled-or-deadlines
   get-file-last-modified-at get-file get-file-page get-file-page-id file-exists?
   get-files get-files-blocks get-files-full get-journals-length get-pages-with-file

src/main/frontend/db/model.cljs

@@ -20,6 +20,7 @@
             [logseq.graph-parser.util.page-ref :as page-ref]
             [logseq.graph-parser.util.db :as db-util]
             [logseq.graph-parser.util :as gp-util]
+            [logseq.outliner.pipeline :as outliner-pipeline]
             [cljs-time.core :as t]
             [cljs-time.format :as tf]
             ;; add map ops to datascript Entity
@@ -535,19 +536,10 @@ independent of format as format specific heading characters are stripped"
      (when-let [block (db-utils/entity db [:block/uuid block-id])]
        (:block/parent block)))))
 
-;; non recursive query
 (defn get-block-parents
-  ([repo block-id]
-   (get-block-parents repo block-id 100))
-  ([repo block-id depth]
-   (loop [block-id block-id
-          parents (list)
-          d 1]
-     (if (> d depth)
-       parents
-       (if-let [parent (get-block-parent repo block-id)]
-         (recur (:block/uuid parent) (conj parents parent) (inc d))
-         parents)))))
+  [repo block-id opts]
+  (when-let [db (conn/get-db repo)]
+    (outliner-pipeline/get-block-parents db block-id opts)))
 
 ;; Use built-in recursive
 (defn get-block-parents-v2
@@ -702,28 +694,6 @@ independent of format as format specific heading characters are stripped"
           (db-utils/pull-many repo
                               '[:db/id :block/name :block/original-name]
                               ids))))))
-(defn get-block-children-ids-in-db
-  [db block-uuid]
-  (when-let [eid (:db/id (db-utils/entity db [:block/uuid block-uuid]))]
-    (let [seen   (volatile! [])]
-      (loop [steps          100      ;check result every 100 steps
-             eids-to-expand [eid]]
-        (when (seq eids-to-expand)
-          (let [eids-to-expand*
-                (mapcat (fn [eid] (map first (d/datoms db :avet :block/parent eid))) eids-to-expand)
-                uuids-to-add (remove nil? (map #(:block/uuid (db-utils/entity db %)) eids-to-expand*))]
-            (when (and (zero? steps)
-                       (seq (set/intersection (set @seen) (set uuids-to-add))))
-              (throw (ex-info "bad outliner data, need to re-index to fix"
-                              {:seen @seen :eids-to-expand eids-to-expand})))
-            (vswap! seen (partial apply conj) uuids-to-add)
-            (recur (if (zero? steps) 100 (dec steps)) eids-to-expand*))))
-      @seen)))
-
-(defn get-block-children-ids
-  ([repo block-uuid]
-   (when-let [db (conn/get-db repo)]
-     (get-block-children-ids-in-db db block-uuid))))
 
 (defn get-block-immediate-children
   "Doesn't include nested children."
@@ -735,10 +705,11 @@ independent of format as format specific heading characters are stripped"
 (defn get-block-children
   "Including nested children."
   [repo block-uuid]
-  (let [ids (get-block-children-ids repo block-uuid)]
-    (when (seq ids)
-      (let [ids' (map (fn [id] [:block/uuid id]) ids)]
-        (db-utils/pull-many repo '[*] ids')))))
+  (when-let [db (conn/get-db repo)]
+    (let [ids (outliner-pipeline/get-block-children-ids db block-uuid)]
+     (when (seq ids)
+       (let [ids' (map (fn [id] [:block/uuid id]) ids)]
+         (db-utils/pull-many repo '[*] ids'))))))
 
 ;; TODO: use the tree directly
 (defn- flatten-tree

src/main/frontend/handler/editor.cljs

@@ -1600,7 +1600,7 @@
   (let [current-block (state/get-edit-block)
         block-parents (set (->> (db/get-block-parents (state/get-current-repo)
                                                       block-id
-                                                      99)
+                                                      {:depth 99})
                                 (map (comp str :block/uuid))))
         current-and-parents (set/union #{(str (:block/uuid current-block))} block-parents)]
     (p/let [result (search/block-search (state/get-current-repo) q {:limit 20})]

src/main/frontend/modules/outliner/core.cljs

@@ -18,10 +18,10 @@
             [frontend.format.block :as block]
             [frontend.handler.file-based.property.util :as property-util]
             [frontend.handler.property.util :as pu]
-            [frontend.db.rtc.op :as rtc-op]
             [frontend.format.mldoc :as mldoc]
             [dommy.core :as dom]
-            [goog.object :as gobj]))
+            [goog.object :as gobj]
+            [logseq.outliner.pipeline :as outliner-pipeline]))
 
 (s/def ::block-map (s/keys :opt [:db/id :block/uuid :block/page :block/left :block/parent]))
 
@@ -858,7 +858,7 @@
                           (db/get-block-parents
                            (state/get-current-repo)
                            (tree/-get-id end-node)
-                           1000)
+                           {:depth 1000})
                           (map :block/uuid)
                           (set))
         self-block? (contains? end-node-parents (tree/-get-id start-node))]
@@ -878,7 +878,7 @@
                                               (db/get-block-parents
                                                (state/get-current-repo)
                                                (tree/-get-id start-node)
-                                               1000)
+                                               {:depth 1000})
                                               (map :block/uuid)
                                               (set))
                                      result (first (set/intersection (set end-node-left-nodes) parents))]
@@ -917,7 +917,7 @@
         original-position? (move-to-original-position? blocks target-block sibling? non-consecutive-blocks?)]
     (when (and (not (contains? (set (map :db/id blocks)) (:db/id target-block)))
                (not original-position?))
-      (let [parents (->> (db/get-block-parents (state/get-current-repo) (:block/uuid target-block))
+      (let [parents (->> (db/get-block-parents (state/get-current-repo) (:block/uuid target-block) {})
                          (map :db/id)
                          (set))
             move-parents-to-child? (some parents (map :db/id blocks))]
@@ -933,7 +933,8 @@
                     move-blocks-next-tx [(build-move-blocks-next-tx target-block blocks {:sibling? sibling?
                                                                                          :non-consecutive-blocks? non-consecutive-blocks?})]
                     children-page-tx (when not-same-page?
-                                       (let [children-ids (mapcat #(db/get-block-children-ids (state/get-current-repo) (:block/uuid %)) blocks)]
+                                       (let [children-ids (mapcat #(outliner-pipeline/get-block-children-ids (db/get-db (state/get-current-repo)) (:block/uuid %))
+                                                                  blocks)]
                                          (map (fn [id] {:block/uuid id
                                                         :block/page target-page}) children-ids)))
                     fix-non-consecutive-tx (->> (fix-non-consecutive-blocks blocks target-block sibling?)

src/main/frontend/modules/outliner/pipeline.cljs

@@ -1,9 +1,6 @@
 (ns frontend.modules.outliner.pipeline
-  (:require [clojure.set :as set]
-            [datascript.core :as d]
-            [frontend.config :as config]
+  (:require [frontend.config :as config]
             [frontend.db :as db]
-            [frontend.db.model :as db-model]
             [frontend.db.react :as react]
             [frontend.modules.outliner.file :as file]
             [logseq.outliner.datascript-report :as ds-report]
@@ -20,73 +17,10 @@
          (not (get-in tx-report [:tx-meta :created-from-journal-template?])))
     (file/sync-to-file page (:outliner-op (:tx-meta tx-report)))))
 
-;; TODO: it'll be great if we can calculate the :block/path-refs before any
-;; outliner transaction, this way we can group together the real outliner tx
-;; and the new path-refs changes, which makes both undo/redo and
-;; react-query/refresh! easier.
-
-;; TODO: also need to consider whiteboard transactions
-
-;; Steps:
-;; 1. For each changed block, new-refs = its page + :block/refs + parents :block/refs
-;; 2. Its children' block/path-refs might need to be updated too.
 (defn compute-block-path-refs
-  [{:keys [tx-meta db-before]} blocks]
-  (let [repo (state/get-current-repo)
-        blocks (remove :block/name blocks)]
-    (when (:outliner-op tx-meta)
-      (when (react/path-refs-need-recalculated? tx-meta)
-        (let [*computed-ids (atom #{})]
-          (mapcat (fn [block]
-                    (when (and (not (@*computed-ids (:block/uuid block))) ; not computed yet
-                               (not (:block/name block)))
-                      (let [parents (db-model/get-block-parents repo (:block/uuid block))
-                            parents-refs (->> (mapcat :block/path-refs parents)
-                                              (map :db/id))
-                            old-refs (if db-before
-                                       (set (map :db/id (:block/path-refs (d/entity db-before (:db/id block)))))
-                                       #{})
-                            new-refs (set (util/concat-without-nil
-                                           [(:db/id (:block/page block))]
-                                           (map :db/id (:block/refs block))
-                                           parents-refs))
-                            refs-changed? (not= old-refs new-refs)
-                            children (db-model/get-block-children-ids repo (:block/uuid block))
-                            ;; Builds map of children ids to their parent id and :block/refs ids
-                            children-maps (into {}
-                                                (map (fn [id]
-                                                       (let [entity (db/entity [:block/uuid id])]
-                                                         [(:db/id entity)
-                                                          {:parent-id (get-in entity [:block/parent :db/id])
-                                                           :block-ref-ids (map :db/id (:block/refs entity))}]))
-                                                     children))
-                            children-refs (map (fn [[id {:keys [block-ref-ids] :as child-map}]]
-                                                 {:db/id id
-                                                  ;; Recalculate :block/path-refs as db contains stale data for this attribute
-                                                  :block/path-refs
-                                                  (set/union
-                                                   ;; Refs from top-level parent
-                                                   new-refs
-                                                   ;; Refs from current block
-                                                   block-ref-ids
-                                                   ;; Refs from parents in between top-level
-                                                   ;; parent and current block
-                                                   (loop [parent-refs #{}
-                                                          parent-id (:parent-id child-map)]
-                                                     (if-let [parent (children-maps parent-id)]
-                                                       (recur (into parent-refs (:block-ref-ids parent))
-                                                              (:parent-id parent))
-                                                       ;; exits when top-level parent is reached
-                                                       parent-refs)))})
-                                               children-maps)]
-                        (swap! *computed-ids set/union (set (cons (:block/uuid block) children)))
-                        (util/concat-without-nil
-                         [(when (and (seq new-refs)
-                                     refs-changed?)
-                            {:db/id (:db/id block)
-                             :block/path-refs new-refs})]
-                         children-refs))))
-                  blocks))))))
+  [{:keys [tx-meta] :as tx-report} blocks]
+  (when (and (:outliner-op tx-meta) (react/path-refs-need-recalculated? tx-meta))
+    (outliner-pipeline/computer-block-path-refs tx-report blocks)))
 
 (defn invoke-hooks
   [tx-report]

src/test/fixtures/broken-outliner-data-with-cycle.edn

@@ -1,14 +0,0 @@
-;;; datoms
-;;; - 1 <----+
-;;;   - 2    |
-;;;     - 3 -+
-[{:db/id 1
-  :block/uuid #uuid"e538d319-48d4-4a6d-ae70-c03bb55b6fe4"
-  :block/parent 3}
- {:db/id 2
-  :block/uuid #uuid"c46664c0-ea45-4998-adf0-4c36486bb2e5"
-  :block/parent 1}
- {:db/id 3
-  :block/uuid #uuid"2b736ac4-fd49-4e04-b00f-48997d2c61a2"
-  :block/parent 2}
- ]

src/test/frontend/db/model_test.cljs

@@ -3,12 +3,9 @@
             [frontend.db.model :as model]
             [frontend.db :as db]
             [frontend.db.conn :as conn]
-            [logseq.db.schema :as db-schema]
             [frontend.test.helper :as test-helper :refer [load-test-files]]
             [datascript.core :as d]
-            [shadow.resource :as rc]
-            [clojure.set :as set]
-            [clojure.edn :as edn]))
+            [clojure.set :as set]))
 
 (use-fixtures :each {:before test-helper/start-test-db!
                      :after test-helper/destroy-test-db!})
@@ -151,18 +148,6 @@ foo:: bar"}])
       "Non header block's content returns nil"))
 
 
-(def broken-outliner-data-with-cycle (-> (rc/inline "fixtures/broken-outliner-data-with-cycle.edn")
-                                         edn/read-string))
-
-(deftest get-block-children-ids-on-bad-outliner-data
-  (let [db (d/db-with (d/empty-db db-schema/schema)
-                      broken-outliner-data-with-cycle)]
-
-    (is (= "bad outliner data, need to re-index to fix"
-           (try (model/get-block-children-ids-in-db db #uuid"e538d319-48d4-4a6d-ae70-c03bb55b6fe4")
-                (catch :default e
-                  (ex-message e)))))))
-
 (deftest get-block-immediate-children
   (load-test-files [{:file/path "pages/page1.md"
                      :file/content "\n

src/test/frontend/modules/outliner/pipeline_test.cljs

@@ -14,6 +14,7 @@
             db)
        (map first)))
 
+;; TODO: Move this test to outliner dep when there is a load-test-files helper for deps
 (deftest compute-block-path-refs
   (load-test-files [{:file/path "pages/page1.md"
                      :file/content "prop:: #bar
@@ -31,7 +32,7 @@
                                          :block/path-refs [{:db/id new-tag-id}])
                                   %)
                                blocks)
-          refs-tx (pipeline/compute-block-path-refs {:tx-meta {:outliner-op :save-block}} modified-blocks)
+          refs-tx (pipeline/compute-block-path-refs {:tx-meta {:outliner-op :save-block} :db-after @conn} modified-blocks)
           _ (d/transact! conn (concat (map (fn [m] [:db/retract (:db/id m) :block/path-refs]) refs-tx)
                                       refs-tx))
           updated-blocks (->> (get-blocks @conn)