fix: move :block/path-res calculation to dep graph-parser

deps/graph-parser/src/logseq/graph_parser/cli.cljs

@@ -7,7 +7,8 @@
             [logseq.common.config :as common-config]
             [logseq.graph-parser :as graph-parser]
             [logseq.common.util :as common-util]
-            [logseq.graph-parser.db :as gp-db]))
+            [logseq.graph-parser.db :as gp-db]
+            [logseq.graph-parser.db-pipeline :as db-pipeline]))
 
 (defn- slurp
   "Return file contents like clojure.core/slurp"
@@ -75,6 +76,7 @@
    (let [config (read-config dir)
          files (or (:files options) (build-graph-files dir config))
          conn (or (:conn options) (gp-db/start-conn))
+         _ (db-pipeline/add-listener conn)
          _ (when-not (:files options) (println "Parsing" (count files) "files..."))
          asts (parse-files conn files (merge options {:config config}))]
      {:conn conn

deps/outliner/src/logseq/outliner/datascript_report.cljs → deps/graph-parser/src/logseq/graph_parser/datascript_report.cljs

@@ -1,4 +1,4 @@
-(ns logseq.outliner.datascript-report
+(ns logseq.graph-parser.datascript-report
   "Datascript fns related to getting data from a connection listener's tx-report"
   (:require [clojure.set :as set]
             [datascript.core :as d]))

deps/graph-parser/src/logseq/graph_parser/db_pipeline.cljs

@@ -0,0 +1,102 @@
+(ns logseq.graph-parser.db-pipeline
+  "This ns provides a datascript listener for DB graphs to add additional changes
+   that the frontend also adds per transact.
+   Known limitations:
+   * Deleted blocks don't update effected :block/tx-id"
+  (:require [datascript.core :as d]
+            [clojure.set :as set]
+            [logseq.db :as ldb]
+            [logseq.graph-parser.datascript-report :as ds-report]))
+
+(defn filter-deleted-blocks
+  [datoms]
+  (keep
+   (fn [d]
+     (when (and (= :block/uuid (:a d)) (false? (:added d)))
+       (:v d)))
+   datoms))
+
+;; TODO: it'll be great if we can calculate the :block/path-refs before any
+;; outliner transaction, this way we can group together the real outliner tx
+;; and the new path-refs changes, which makes both undo/redo and
+;; react-query/refresh! easier.
+
+;; TODO: also need to consider whiteboard transactions
+
+;; Steps:
+;; 1. For each changed block, new-refs = its page + :block/refs + parents :block/refs
+;; 2. Its children' block/path-refs might need to be updated too.
+(defn- compute-block-path-refs
+  [{:keys [db-before db-after]} blocks*]
+  (let [blocks (remove :block/name blocks*)
+        *computed-ids (atom #{})]
+    (mapcat (fn [block]
+              (when (and (not (@*computed-ids (:block/uuid block))) ; not computed yet
+                         (not (:block/name block)))
+                (let [parents (ldb/get-block-parents db-after (:block/uuid block) {})
+                      parents-refs (->> (mapcat :block/path-refs parents)
+                                        (map :db/id))
+                      old-refs (if db-before
+                                 (set (map :db/id (:block/path-refs (d/entity db-before (:db/id block)))))
+                                 #{})
+                      new-refs (set (concat
+                                     (some-> (:db/id (:block/page block)) vector)
+                                     (map :db/id (:block/refs block))
+                                     parents-refs))
+                      refs-changed? (not= old-refs new-refs)
+                      children (ldb/get-block-children-ids db-after (:block/uuid block))
+                            ;; Builds map of children ids to their parent id and :block/refs ids
+                      children-maps (into {}
+                                          (map (fn [id]
+                                                 (let [entity (d/entity db-after [:block/uuid id])]
+                                                   [(:db/id entity)
+                                                    {:parent-id (get-in entity [:block/parent :db/id])
+                                                     :block-ref-ids (map :db/id (:block/refs entity))}]))
+                                               children))
+                      children-refs (map (fn [[id {:keys [block-ref-ids] :as child-map}]]
+                                           {:db/id id
+                                                  ;; Recalculate :block/path-refs as db contains stale data for this attribute
+                                            :block/path-refs
+                                            (set/union
+                                                   ;; Refs from top-level parent
+                                             new-refs
+                                                   ;; Refs from current block
+                                             block-ref-ids
+                                                   ;; Refs from parents in between top-level
+                                                   ;; parent and current block
+                                             (loop [parent-refs #{}
+                                                    parent-id (:parent-id child-map)]
+                                               (if-let [parent (children-maps parent-id)]
+                                                 (recur (into parent-refs (:block-ref-ids parent))
+                                                        (:parent-id parent))
+                                                       ;; exits when top-level parent is reached
+                                                 parent-refs)))})
+                                         children-maps)]
+                  (swap! *computed-ids set/union (set (cons (:block/uuid block) children)))
+                  (concat
+                   (when (and (seq new-refs) refs-changed?)
+                     [{:db/id (:db/id block)
+                       :block/path-refs new-refs}])
+                   children-refs))))
+            blocks)))
+
+(defn compute-block-path-refs-tx
+  [tx-report blocks]
+  (let [refs-tx (compute-block-path-refs tx-report blocks)
+        truncate-refs-tx (map (fn [m] [:db/retract (:db/id m) :block/path-refs]) refs-tx)]
+    (concat truncate-refs-tx refs-tx)))
+
+(defn- invoke-hooks
+  "Modified copy of frontend.modules.outliner.pipeline/invoke-hooks that doesn't
+  handle :block/tx-id"
+  [conn tx-report]
+  (when (not (get-in tx-report [:tx-meta :pipeline-replace?]))
+    (let [{:keys [blocks]} (ds-report/get-blocks-and-pages tx-report)
+          block-path-refs-tx (compute-block-path-refs-tx tx-report blocks)]
+      (d/transact! conn block-path-refs-tx {:pipeline-replace? true}))))
+
+(defn add-listener
+  "Adds a listener to the datascript connection to add additional changes from outliner.pipeline"
+  [conn]
+  (d/listen! conn :pipeline-updates (fn pipeline-updates [tx-report]
+                                      (invoke-hooks conn tx-report))))

deps/graph-parser/src/logseq/graph_parser/test/docs_graph_helper.cljs

@@ -162,16 +162,13 @@
   ;; only increase over time as the docs graph rarely has deletions
   (testing "Counts"
     (is (= 303 (count files)) "Correct file count")
-    ;; async compute :block/path-refs
-    (comment
-     (is (= 57913 (count (d/datoms db :eavt))) "Correct datoms count"))
-
-    (comment
-      (is (= 5866
-             (ffirst
-              (d/q '[:find (count ?b)
-                     :where [?b :block/path-refs ?bp] [?bp :block/name]] db)))
-          "Correct referenced blocks count"))
+    (is (= 63926 (count (d/datoms db :eavt))) "Correct datoms count")
+
+    (is (= 5946
+           (ffirst
+            (d/q '[:find (count ?b)
+                   :where [?b :block/path-refs ?bp] [?bp :block/name]] db)))
+        "Correct referenced blocks count")
     (is (= 23
            (ffirst
             (d/q '[:find (count ?b)

deps/graph-parser/test/logseq/graph_parser/cli_test.cljs

@@ -158,7 +158,7 @@
                        datoms->entity-maps
                        (map #(assoc (or (not-empty (select-keys % [:block/content :block/name]))
                                         %)
-                                    :attributes (disj (set (keys %)) :block/file :block/format)))
+                                    :attributes (disj (set (keys %)) :block/file :block/format :block/path-refs)))
                        set)
         db-ents (->> (d/datoms graph-db :eavt)
                      datoms->entity-maps
@@ -167,7 +167,8 @@
                                   :attributes (cond-> (disj (set (keys %))
                                                             ;; Don't compare :block/format as db graphs
                                                             ;; are purposely different
-                                                            :block/format)
+                                                            :block/format
+                                                            :block/path-refs)
                                                 (seq (:block/content %))
                                                 (set/difference #{:block/created-at :block/updated-at}))))
                      set)]

deps/outliner/script/transact.cljs

@@ -1,6 +1,6 @@
 (ns transact
   "This script generically runs transactions against the queried blocks"
-  (:require [logseq.outliner.cli.pipeline :as cli-pipeline]
+  (:require [logseq.graph-parser.db-pipeline :as db-pipeline]
             [logseq.db.sqlite.db :as sqlite-db]
             [logseq.db.frontend.rules :as rules]
             [datascript.core :as d]
@@ -33,9 +33,9 @@
           (println "With the following blocks updated:")
           (prn (map #(select-keys (d/entity @conn %) [:block/name :block/content]) blocks-to-update)))
       (do
-        (cli-pipeline/add-listener conn)
+        (db-pipeline/add-listener conn)
         (d/transact! conn update-tx)
         (println "Updated" (count update-tx) "block(s) for graph" (str db-name "!"))))))
 
 (when (= nbb/*file* (:file (meta #'-main)))
-  (-main *command-line-args*))
+  (-main *command-line-args*))

deps/outliner/src/logseq/outliner/cli/pipeline.cljs

@@ -1,23 +0,0 @@
-(ns ^:node-only logseq.outliner.cli.pipeline
-  "This ns provides a datascript listener for DB graphs to add additional changes
-   that the frontend also adds per transact.
-   Known limitations:
-   * Deleted blocks don't update effected :block/tx-id"
-  (:require [datascript.core :as d]
-            [logseq.outliner.datascript-report :as ds-report]
-            [logseq.outliner.pipeline :as outliner-pipeline]))
-
-(defn- invoke-hooks
-  "Modified copy of frontend.modules.outliner.pipeline/invoke-hooks that doesn't
-  handle :block/tx-id"
-  [conn tx-report]
-  (when (not (get-in tx-report [:tx-meta :replace?]))
-    (let [{:keys [blocks]} (ds-report/get-blocks-and-pages tx-report)
-          block-path-refs-tx (outliner-pipeline/compute-block-path-refs-tx tx-report blocks)]
-      (d/transact! conn block-path-refs-tx {:replace? true}))))
-
-(defn add-listener
-  "Adds a listener to the datascript connection to add additional changes from outliner.pipeline"
-  [conn]
-  (d/listen! conn :pipeline-updates (fn pipeline-updates [tx-report]
-                                      (invoke-hooks conn tx-report))))

scripts/README.md

@@ -65,4 +65,4 @@ Created graph schema!
 #### Update graph scripts
 
 For database graphs, it is recommended to use
-`logseq.outliner.cli.pipeline/add-listener!` when updating graphs.  TODO
+`logseq.graph-parser.db-pipeline/add-listener!` when updating graphs.  TODO

scripts/src/logseq/tasks/db_graph/create_graph.cljs

@@ -7,7 +7,7 @@
             [logseq.db.sqlite.util :as sqlite-util]
             [logseq.db.sqlite.create-graph :as sqlite-create-graph]
             [logseq.db.frontend.property.build :as db-property-build]
-            [logseq.outliner.cli.pipeline :as cli-pipeline]
+            [logseq.graph-parser.db-pipeline :as db-pipeline]
             [logseq.common.util :as common-util]
             [clojure.string :as string]
             [clojure.set :as set]
@@ -44,7 +44,7 @@
   (fs/mkdirSync (node-path/join dir db-name) #js {:recursive true})
   ;; Same order as frontend.db.conn/start!
   (let [conn (sqlite-db/open-db! dir db-name)]
-    (cli-pipeline/add-listener conn)
+    (db-pipeline/add-listener conn)
     (setup-init-data conn additional-config)
     conn))
 

src/main/frontend/handler/whiteboard.cljs

@@ -142,7 +142,7 @@
        :deleted-shapes deleted-shapes
        :new-shapes created-shapes
        :metadata {:whiteboard/transact? true
-                  :replace? replace?}})))
+                  :pipeline-replace? replace?}})))
 
 (defonce *last-shapes-nonce (atom {}))
 

src/main/frontend/worker/db/fix.cljs

@@ -182,7 +182,7 @@
         page (d/entity db page-id)]
     (when-not (or (ldb/whiteboard-page? page)
                   (ldb/hidden-page? page))
-      (let [transact-opts (if replace-tx? {:replace? true} {})
+      (let [transact-opts (if replace-tx? {:pipeline-replace? true} {})
             *fix-tx-data (atom [])]
         (when fix-parent-left?
           (loop-fix-conflicts conn page-id transact-opts *fix-tx-data tx-report from-fix-test?))

src/main/frontend/worker/pipeline.cljs

@@ -8,8 +8,8 @@
             [logseq.db :as ldb]
             [logseq.db.frontend.validate :as db-validate]
             [logseq.db.sqlite.util :as sqlite-util]
-            [logseq.outliner.datascript-report :as ds-report]
-            [logseq.outliner.pipeline :as outliner-pipeline]
+            [logseq.graph-parser.datascript-report :as ds-report]
+            [logseq.graph-parser.db-pipeline :as outliner-pipeline]
             [logseq.db.frontend.property :as db-property]
             [logseq.outliner.core :as outliner-core]))
 
@@ -94,8 +94,7 @@
               path-refs (set (compute-block-path-refs-tx tx-report blocks))
               tx-report' (or
                           (when (seq path-refs)
-                            (ldb/transact! conn path-refs {:replace? true
-                                                           :pipeline-replace? true}))
+                            (ldb/transact! conn path-refs {:pipeline-replace? true}))
                           (do
                             (when-not (exists? js/process) (d/store @conn))
                             tx-report))
@@ -134,8 +133,7 @@
               tx-report' (or
                           (when (seq replace-tx)
                           ;; TODO: remove this since transact! is really slow
-                            (ldb/transact! conn replace-tx {:replace? true
-                                                            :pipeline-replace? true}))
+                            (ldb/transact! conn replace-tx {:pipeline-replace? true}))
                           (do
                             (when-not (exists? js/process) (d/store @conn))
                             tx-report))