Move db fix to worker

deps/db/src/logseq/db.cljs

@@ -168,3 +168,18 @@
 (defn get-page-blocks-count
   [db page-id]
   (count (d/datoms db :avet :block/page page-id)))
+
+(defn get-by-parent-&-left
+  [db parent-id left-id]
+  (when (and parent-id left-id)
+    (let [lefts (:block/_left (d/entity db left-id))]
+      (some (fn [node] (when (and (= parent-id (:db/id (:block/parent node)))
+                                  (not= parent-id (:db/id node)))
+                         node)) lefts))))
+
+(defn get-right-sibling
+  [db db-id]
+  (when-let [block (d/entity db db-id)]
+    (get-by-parent-&-left db
+                          (:db/id (:block/parent block))
+                          db-id)))

src/main/frontend/db/model.cljs

@@ -488,13 +488,7 @@ independent of format as format specific heading characters are stripped"
   [db block]
   (and (:block/collapsed? block) (has-children? db (:block/uuid block))))
 
-(defn get-by-parent-&-left
-  [db parent-id left-id]
-  (when (and parent-id left-id)
-    (let [lefts (:block/_left (db-utils/entity db left-id))]
-      (some (fn [node] (when (and (= parent-id (:db/id (:block/parent node)))
-                                  (not= parent-id (:db/id node)))
-                         node)) lefts))))
+(def get-by-parent-&-left ldb/get-by-parent-&-left)
 
 (defn top-block?
   [block]
@@ -552,12 +546,7 @@ independent of format as format specific heading characters are stripped"
       (when (not= (:db/id left) (:db/id (:block/parent e)))
         left))))
 
-(defn get-right-sibling
-  [db db-id]
-  (when-let [block (db-utils/entity db db-id)]
-    (get-by-parent-&-left db
-                          (:db/id (:block/parent block))
-                          db-id)))
+(def get-right-sibling ldb/get-right-sibling)
 
 (defn get-next
   "Get next block, either its right sibling, or loop to find its next block."

src/main/frontend/modules/outliner/datascript.cljs

@@ -1,6 +1,5 @@
 (ns frontend.modules.outliner.datascript
-  (:require [datascript.core :as d]
-            [frontend.db :as db]
+  (:require [frontend.db :as db]
             [frontend.modules.outliner.pipeline :as pipelines]
             [frontend.modules.editor.undo-redo :as undo-redo]
             [frontend.state :as state]
@@ -10,7 +9,6 @@
             [clojure.string :as string]
             [frontend.util :as util]
             [logseq.graph-parser.util.block-ref :as block-ref]
-            [frontend.db.fix :as db-fix]
             [frontend.handler.file-based.property.util :as property-util]))
 
 (defn new-outliner-txs-state [] (atom []))
@@ -97,21 +95,6 @@
       (concat txs retracted-tx' macros-tx))
     txs))
 
-(defn fix-db!
-  [{:keys [db-before db-after tx-data]}]
-  (let [changed-pages (->> (filter (fn [d] (contains? #{:block/left :block/parent} (:a d))) tx-data)
-                           (map :e)
-                           distinct
-                           (map (fn [id]
-                                  (-> (or (d/entity db-after id)
-                                          (d/entity db-before id))
-                                      :block/page
-                                      :db/id)))
-                           (remove nil?)
-                           (distinct))]
-    (doseq [changed-page-id changed-pages]
-      (db-fix/fix-page-if-broken! db-after changed-page-id {}))))
-
 (defn transact!
   [txs opts before-editor-cursor]
   (let [repo (state/get-current-repo)
@@ -141,8 +124,7 @@
         (let [repo (get opts :repo (state/get-current-repo))
               rs (db/transact! repo txs (assoc opts :outliner/transact? true))
               tx-id (get-tx-id rs)]
-          ;; TODO: disable this when db is stable
-          (when (and config/dev? (not util/node-test?)) (fix-db! rs))
+
           (state/update-state! :history/tx->editor-cursor
                                (fn [m] (assoc m tx-id before-editor-cursor)))
 

src/main/frontend/persist_db/browser.cljs

@@ -86,6 +86,7 @@
         (let [tx-meta' (pr-str tx-meta)
               tx-data' (pr-str tx-data)
               context {:dev? config/dev?
+                       :node-test? util/node-test?
                        :validate-db-options (:dev/validate-db-options (state/get-config))
                        :importing? (:graph/importing @state/state)
                        :date-formatter (state/get-date-formatter)

src/main/frontend/db/fix.cljs → src/main/frontend/worker/db/fix.cljs

@@ -1,23 +1,21 @@
-(ns frontend.db.fix
+(ns frontend.worker.db.fix
   "DB validation and fix.
   For pages:
   1. Each block should has a unique [:block/parent :block/left] position.
   2. For any block, its children should be connected by :block/left (no broken chain, no circle, no left to self)."
   (:require [datascript.core :as d]
-            [frontend.db :as db]
-            [frontend.db.model :as db-model]
             [frontend.util :as util]
-            [frontend.state :as state]
-            [frontend.handler.notification :as notification]))
+            [frontend.handler.notification :as notification]
+            [logseq.db :as ldb]))
 
 (defn- fix-parent-broken-chain
   [db parent-id]
-  (let [parent (db/entity parent-id)
+  (let [parent (d/entity db parent-id)
         parent-id (:db/id parent)
         blocks (:block/_parent parent)]
     (when (seq blocks)
       (let [children-ids (set (map :db/id blocks))
-            sorted (db-model/sort-by-left blocks parent)
+            sorted (ldb/sort-by-left blocks parent)
             broken-chain? (not= (count sorted) (count blocks))]
         (when broken-chain?
           (let [error-data {:parent {:db/id parent-id
@@ -34,7 +32,7 @@
               (str "Broken chain detected:\n" error-data)]
              :error
              false))
-          (let [first-child-id (:db/id (db-model/get-by-parent-&-left db parent-id parent-id))
+          (let [first-child-id (:db/id (ldb/get-by-parent-&-left db parent-id parent-id))
                 *ids (atom children-ids)
                 sections (loop [sections []]
                            (if (seq @*ids)
@@ -49,7 +47,7 @@
                                                          (swap! *ids disj id)
                                                          [id])))
                                    section-with-left (or
-                                                      (when-let [left-id (:db/id (:block/left (db/entity (first current-section))))]
+                                                      (when-let [left-id (:db/id (:block/left (d/entity db (first current-section))))]
                                                         (swap! *ids disj left-id)
                                                         (when (and
                                                                (not (contains? (set current-section) left-id)) ; circle
@@ -57,7 +55,7 @@
                                                           (vec (cons left-id current-section))))
                                                       current-section)
                                    section-with-right (or
-                                                       (when-let [right-id (:db/id (db-model/get-right-sibling db (last section-with-left)))]
+                                                       (when-let [right-id (:db/id (ldb/get-right-sibling db (last section-with-left)))]
                                                          (swap! *ids disj right-id)
                                                          (when (and (not (contains? (set section-with-left) right-id)) ; circle
                                                                     (contains? children-ids right-id))
@@ -111,7 +109,7 @@
          (into {}))))
 
 (defn- fix-parent-left-conflicts
-  [conflicts]
+  [db conflicts]
   (when (seq conflicts)
     (prn :debug "Parent left id conflicts:")
     (notification/show!
@@ -130,7 +128,7 @@
                   :block/left (:db/id (nth items (if (zero? idx) idx (dec idx))))
                   :block/parent (:db/id (:block/parent first-item))})
                others)
-           right-tx (when-let [right (db-model/get-right-sibling (db/get-db) (:db/id first-item))]
+           right-tx (when-let [right (ldb/get-right-sibling db (:db/id first-item))]
                       [{:db/id (:db/id right)
                         :block/left (:db/id (last items))}])]
        (concat tx right-tx)))
@@ -141,33 +139,37 @@
   (let [parent-left->es (build-parent-left->es db page-id)]
     (filter #(> (count (second %)) 1) parent-left->es)))
 
-(defn loop-fix-conflicts
-  [repo db page-id transact-opts]
-  (let [conflicts (get-conflicts db page-id)
+(defn- loop-fix-conflicts
+  [conn page-id transact-opts *fix-tx-data]
+  (let [db @conn
+        conflicts (get-conflicts db page-id)
         fix-conflicts-tx (when (seq conflicts)
-                           (fix-parent-left-conflicts conflicts))]
+                           (fix-parent-left-conflicts db conflicts))]
     (when (seq fix-conflicts-tx)
       (prn :debug :conflicts-tx)
       (util/pprint fix-conflicts-tx)
-      (db/transact! repo fix-conflicts-tx transact-opts)
-      (let [db (db/get-db repo)]
-        (when (seq (get-conflicts db page-id))
-          (loop-fix-conflicts repo db page-id transact-opts))))))
+      (let [tx-data (:tx-data (d/transact! conn fix-conflicts-tx transact-opts))]
+        (swap! *fix-tx-data (fn [old-data] (concat old-data tx-data))))
+      (when (seq (get-conflicts @conn page-id))
+        (loop-fix-conflicts conn page-id transact-opts *fix-tx-data)))))
 
 (defn fix-page-if-broken!
   "Fix the page if it has either parent-left conflicts or broken chains."
-  [db page-id {:keys [fix-parent-left? fix-broken-chain? replace-tx?]
-            :or {fix-parent-left? true
-                 fix-broken-chain? true
-                 replace-tx? false}
-            :as _opts}]
-  (let [repo (state/get-current-repo)
-        transact-opts (if replace-tx? {:replace? true} {})]
+  [conn page-id {:keys [fix-parent-left? fix-broken-chain? replace-tx?]
+                 :or {fix-parent-left? true
+                      fix-broken-chain? true
+                      replace-tx? false}
+                 :as _opts}]
+  (let [db @conn
+        transact-opts (if replace-tx? {:replace? true} {})
+        *fix-tx-data (atom [])]
     (when fix-parent-left?
-      (loop-fix-conflicts repo db page-id transact-opts))
+      (loop-fix-conflicts conn page-id transact-opts *fix-tx-data))
     (when fix-broken-chain?
-      (let [db' (db/get-db)
-            parent-left->es' (build-parent-left->es (db/get-db) page-id)
+      (let [db' @conn
+            parent-left->es' (build-parent-left->es db page-id)
             fix-broken-chain-tx (fix-broken-chain db' parent-left->es')]
         (when (seq fix-broken-chain-tx)
-          (db/transact! repo fix-broken-chain-tx transact-opts))))))
+          (let [tx-data (:tx-data (d/transact! conn fix-broken-chain-tx transact-opts))]
+            (swap! *fix-tx-data (fn [old-data] (concat old-data tx-data)))))))
+    @*fix-tx-data))

src/main/frontend/worker/pipeline.cljs

@@ -6,7 +6,8 @@
             [frontend.worker.react :as worker-react]
             [frontend.worker.file :as file]
             [logseq.db.frontend.validate :as validate]
-            [logseq.db.sqlite.util :as sqlite-util]))
+            [logseq.db.sqlite.util :as sqlite-util]
+            [frontend.worker.db.fix :as db-fix]))
 
 (defn- path-refs-need-recalculated?
   [tx-meta]
@@ -42,14 +43,34 @@
                empty-property-parents)
        (remove nil?)))))
 
-(defn invoke-hooks
-  [repo conn tx-report context]
+(defn fix-db!
+  [conn {:keys [db-before db-after tx-data]}]
+  (let [changed-pages (->> (filter (fn [d] (contains? #{:block/left :block/parent} (:a d))) tx-data)
+                           (map :e)
+                           distinct
+                           (map (fn [id]
+                                  (-> (or (d/entity db-after id)
+                                          (d/entity db-before id))
+                                      :block/page
+                                      :db/id)))
+                           (remove nil?)
+                           (distinct))]
+    (doseq [changed-page-id changed-pages]
+      (db-fix/fix-page-if-broken! conn changed-page-id {}))))
 
+(defn validate-and-fix-db!
+  [repo conn tx-report context]
   (when (and (:dev? context) (sqlite-util/db-based-graph? repo))
     (validate/validate-db! tx-report (:validate-db-options context)))
+  (when (and (:dev? context)
+             (not (:node-test? context)))
+    (fix-db! conn tx-report)))
 
+(defn invoke-hooks
+  [repo conn tx-report context]
   (let [tx-meta (:tx-meta tx-report)
-        {:keys [from-disk? new-graph?]} tx-meta]
+        {:keys [from-disk? new-graph?]} tx-meta
+        fix-tx-data (validate-and-fix-db! repo conn tx-report context)]
     (if (or from-disk? new-graph?)
       {:tx-report tx-report}
       (let [{:keys [pages blocks]} (ds-report/get-blocks-and-pages tx-report)
@@ -73,7 +94,7 @@
                            (remove nil?))))
             tx-report' (d/transact! conn replace-tx {:replace? true
                                                      :pipeline-replace? true})
-            full-tx-data (concat (:tx-data tx-report) (:tx-data tx-report'))
+            full-tx-data (concat (:tx-data tx-report) fix-tx-data (:tx-data tx-report'))
             final-tx-report (assoc tx-report' :tx-data full-tx-data)
             affected-query-keys (when-not (:importing? context)
                                   (worker-react/get-affected-queries-keys final-tx-report context))]

src/test/frontend/db/fix_test.cljs

@@ -3,7 +3,7 @@
             [datascript.core :as d]
             [frontend.core-test :as core-test]
             [frontend.test.fixtures :as fixtures]
-            [frontend.db.fix :as db-fix]))
+            [frontend.worker.db.fix :as db-fix]))
 
 (use-fixtures :each fixtures/reset-db)
 
@@ -22,7 +22,7 @@
   (let [conn (core-test/get-current-conn)
         _ (d/transact! conn init-conflicts)
         page-id (:db/id (d/entity @conn 1))
-        _ (db-fix/fix-page-if-broken! @conn page-id {})]
+        _ (db-fix/fix-page-if-broken! conn page-id {})]
     (is (= 2 (:db/id (:block/left (d/entity @conn 3)))))))
 
 (deftest test-conflicts-with-right
@@ -34,7 +34,7 @@
                        :block/left [:block/uuid "2"]}])
         _ (d/transact! conn data)
         page-id (:db/id (d/entity @conn 1))
-        _ (db-fix/fix-page-if-broken! @conn page-id {})]
+        _ (db-fix/fix-page-if-broken! conn page-id {})]
     (is (= 3 (:db/id (:block/left (d/entity @conn 4)))))))
 
 (def init-broken-chain
@@ -58,7 +58,7 @@
         data init-broken-chain
         _ (d/transact! conn data)
         page-id (:db/id (d/entity @conn 1))
-        _ (db-fix/fix-page-if-broken! @conn page-id {})]
+        _ (db-fix/fix-page-if-broken! conn page-id {})]
     (is
      (=
       (set [{:db/id 2, :block/left 1}
@@ -84,7 +84,7 @@
                :block/left [:block/uuid "2"]}]
         _ (d/transact! conn data)
         page-id (:db/id (d/entity @conn 1))
-        _ (db-fix/fix-page-if-broken! @conn page-id {})]
+        _ (db-fix/fix-page-if-broken! conn page-id {})]
     (is
      (=
       (set [{:db/id 3, :block/left 1}
@@ -112,7 +112,7 @@
                :block/left [:block/uuid "3"]}]
         _ (d/transact! conn data)
         page-id (:db/id (d/entity @conn 1))
-        _ (db-fix/fix-page-if-broken! @conn page-id {})]
+        _ (db-fix/fix-page-if-broken! conn page-id {})]
     (is
      (=
       (set [{:db/id 2, :block/left 1}
@@ -150,7 +150,7 @@
                :block/left [:block/uuid "2"]}]
         _ (d/transact! conn data)
         page-id (:db/id (d/entity @conn 1))
-        _ (db-fix/fix-page-if-broken! @conn page-id {})]
+        _ (db-fix/fix-page-if-broken! conn page-id {})]
     (is
      (=
       #{{:db/id 3, :block/left 1}
@@ -190,7 +190,7 @@
                :block/left [:block/uuid "5"]}]
         _ (d/transact! conn data)
         page-id (:db/id (d/entity @conn 1))
-        _ (db-fix/fix-page-if-broken! @conn page-id {})]
+        _ (db-fix/fix-page-if-broken! conn page-id {})]
     (is
      (=
       #{{:db/id 2, :block/left 1}

src/test/frontend/handler/db_based/page_test.cljs

@@ -5,7 +5,7 @@
             [datascript.core :as d]
             [frontend.handler.page :as page-handler]
             [frontend.db :as db]
-            [frontend.db.fix :as db-fix]
+            [frontend.worker.db.fix :as db-fix]
             [frontend.handler.editor :as editor-handler]
             [goog.dom :as gdom]))