Move db validate to worker

deps/db/deps.edn

@@ -4,7 +4,7 @@
                          :sha     "21fc7880c7042fb1d9086135d162ea7a91681f89"}
   com.cognitect/transit-cljs   {:mvn/version "0.8.280"}
   cljs-bean/cljs-bean          {:mvn/version "1.5.0"}
-  logseq/graph-parser          {:local/root "deps/graph-parser"}}
+  logseq/graph-parser          {:local/root "../../deps/graph-parser"}}
 
  :aliases
  {:clj-kondo

deps/db/src/logseq/db/frontend/validate.cljs

@@ -0,0 +1,29 @@
+(ns logseq.db.frontend.validate
+  "Validate db"
+  (:require [datascript.core :as d]
+            [logseq.db.frontend.malli-schema :as db-malli-schema]
+            [malli.util :as mu]
+            [malli.core :as m]
+            [cljs.pprint :as pprint]))
+
+(defn validate-db!
+  "Validates the entities that have changed in the given datascript tx-report.
+   Validation is only for DB graphs"
+  [{:keys [db-after tx-data tx-meta]} validate-options]
+  (let [{:keys [known-schema? closed-schema? fail-invalid?]} validate-options
+        changed-ids (->> tx-data (map :e) distinct)
+        ent-maps* (->> changed-ids (mapcat #(d/datoms db-after :eavt %)) db-malli-schema/datoms->entity-maps vals)
+        ent-maps (vec (db-malli-schema/update-properties-in-ents ent-maps*))
+        db-schema (cond-> (if known-schema? db-malli-schema/DB-known db-malli-schema/DB)
+                    true
+                    (db-malli-schema/update-properties-in-schema db-after)
+                    closed-schema?
+                    mu/closed-schema)]
+    (js/console.log "changed eids:" changed-ids tx-meta)
+    (when-let [errors (->> ent-maps
+                           (m/explain db-schema)
+                           :errors)]
+      (js/console.error "Invalid datascript entities detected amongst changed entity ids:" changed-ids)
+      (pprint/pprint {:errors errors})
+      (pprint/pprint {:entity-maps ent-maps})
+      (when fail-invalid? (js/alert "Invalid DB!")))))

scripts/src/logseq/tasks/dev/db_and_file_graphs.clj

@@ -43,8 +43,7 @@
   ["src/main/frontend/handler/file_based" "src/main/frontend/handler/conversion.cljs" "src/main/frontend/handler/file_sync.cljs"
    "src/main/frontend/fs"
    "src/main/frontend/components/conversion.cljs" "src/main/frontend/components/file_sync.cljs"
-   "src/main/frontend/util/fs.cljs"
-   "src/main/frontend/modules/outliner/file.cljs"])
+   "src/main/frontend/util/fs.cljs"])
 
 (defn- validate-db-ns-not-in-file
   []

src/main/frontend/date.cljs

@@ -154,10 +154,6 @@
                     default-journal-filename-formatter)]
     (journal-title-> journal-title #(tf/unparse formatter %))))
 
-(defn date->file-name
-  [date]
-  (worker-date/date->file-name date (state/get-date-formatter)))
-
 (defn journal-title->custom-format
   [journal-title]
   (journal-title-> journal-title #(date-time-util/format % (state/get-date-formatter))))

src/main/frontend/db/model.cljs

@@ -386,7 +386,6 @@ independent of format as format specific heading characters are stripped"
     (:block/properties page)))
 
 (def sort-by-left ldb/sort-by-left)
-(def try-sort-by-left ldb/try-sort-by-left)
 
 (defn sub-block
   [id]

src/main/frontend/handler/file_based/page.cljs

@@ -126,9 +126,6 @@
   (let [repo (state/get-current-repo)
         to-page (db/entity [:block/name (util/page-name-sanity-lc new-name)])
         blocks (:block/_refs (db/entity (:db/id page)))
-        page-ids (->> (map (fn [b]
-                             {:db/id (:db/id (:block/page b))}) blocks)
-                      (set))
         tx       (->> (map (fn [{:block/keys [uuid content properties format] :as block}]
                              (let [content    (let [content' (replace-old-page! content old-original-name new-name format)]
                                                 (when-not (= content' content)

src/main/frontend/handler/whiteboard.cljs

@@ -199,9 +199,6 @@
              :updated-at (util/time-ms),
              :created-at (util/time-ms)}]))
 
-(defn get-whiteboard-entity [page-name]
-  (db-utils/entity [:block/name (util/page-name-sanity-lc page-name)]))
-
 (defn create-new-whiteboard-page!
   ([]
    (create-new-whiteboard-page! nil))

src/main/frontend/modules/editor/undo_redo.cljs

@@ -1,7 +1,6 @@
 (ns frontend.modules.editor.undo-redo
   (:require [frontend.db :as db]
             [frontend.handler.notification :as notification]
-            [logseq.outliner.datascript-report :as ds-report]
             [frontend.util.page :as page-util]
             [frontend.state :as state]
             [clojure.set :as set]

src/main/frontend/modules/file/core.cljs

@@ -1,16 +1,8 @@
 (ns frontend.modules.file.core
   (:require [clojure.string :as string]
-            [frontend.config :as config]
-            [frontend.date :as date]
             [frontend.db :as db]
-            [frontend.db.model :as model]
-            [frontend.db.utils :as db-utils]
-            [frontend.handler.file :as file-handler]
             [frontend.state :as state]
-            [frontend.util.fs :as fs-util]
-            [frontend.handler.file-based.property.util :as property-util]
-            [logseq.common.path :as path]
-            [frontend.worker.file.util :as wfu]))
+            [frontend.handler.file-based.property.util :as property-util]))
 
 (defn- indented-block-content
   [content spaces-tabs]
@@ -109,70 +101,3 @@
 (defn tree->file-content
   [tree opts]
   (->> (tree->file-content-aux tree opts) (string/join "\n")))
-
-
-(def init-level 1)
-
-(defn- transact-file-tx-if-not-exists!
-  [page-block ok-handler]
-  (when (and (state/get-current-repo)
-             (:block/name page-block))
-    (let [format (name (get page-block :block/format
-                            (state/get-preferred-format)))
-          title (string/capitalize (:block/name page-block))
-          whiteboard-page? (model/whiteboard-page? page-block)
-          format (if whiteboard-page? "edn" format)
-          journal-page? (date/valid-journal-title? title)
-          journal-title (date/normalize-journal-title title)
-          journal-page? (and journal-page? (not (string/blank? journal-title)))
-          filename (if journal-page?
-                     (date/date->file-name journal-title)
-                     (-> (or (:block/original-name page-block) (:block/name page-block))
-                         (fs-util/file-name-sanity)))
-          sub-dir (cond
-                    journal-page?    (config/get-journals-directory)
-                    whiteboard-page? (config/get-whiteboards-directory)
-                    :else            (config/get-pages-directory))
-          ext (if (= format "markdown") "md" format)
-          file-rpath (path/path-join sub-dir (str filename "." ext))
-          file {:file/path file-rpath}
-          tx [{:file/path file-rpath}
-              {:block/name (:block/name page-block)
-               :block/file file}]]
-      (db/transact! tx)
-      (when ok-handler (ok-handler)))))
-
-(defn- remove-transit-ids [block] (dissoc block :db/id :block/file))
-
-(defn save-tree-aux!
-  [page-block tree blocks-just-deleted?]
-  (let [page-block (db/pull (:db/id page-block))
-        file-db-id (-> page-block :block/file :db/id)
-        file-path (-> (db-utils/entity file-db-id) :file/path)]
-    (if (and (string? file-path) (not-empty file-path))
-      (let [new-content (if (contains? (:block/type page-block) "whiteboard")
-                          (->
-                           (wfu/ugly-pr-str {:blocks tree
-                                            :pages (list (remove-transit-ids page-block))})
-                           (string/triml))
-                          (tree->file-content tree {:init-level init-level}))]
-        (if (and (string/blank? new-content)
-                 (not blocks-just-deleted?))
-          (state/pub-event! [:capture-error {:error (js/Error. "Empty content")
-                                             :payload {}}])
-          (let [files [[file-path new-content]]
-                repo (state/get-current-repo)]
-            (file-handler/alter-files-handler! repo files {} {}))))
-      ;; In e2e tests, "card" page in db has no :file/path
-      (js/console.error "File path from page-block is not valid" page-block tree))))
-
-(defn save-tree!
-  [page-block tree blocks-just-deleted?]
-  {:pre [(map? page-block)]}
-  (let [ok-handler #(save-tree-aux! page-block tree blocks-just-deleted?)
-        file (or (:block/file page-block)
-                 (when-let [page (:db/id (:block/page page-block))]
-                   (:block/file (db-utils/entity page))))]
-    (if file
-      (ok-handler)
-      (transact-file-tx-if-not-exists! page-block ok-handler))))

src/main/frontend/modules/outliner/datascript.cljs

@@ -10,12 +10,8 @@
             [clojure.string :as string]
             [frontend.util :as util]
             [logseq.graph-parser.util.block-ref :as block-ref]
-            [logseq.db.frontend.malli-schema :as db-malli-schema]
             [frontend.db.fix :as db-fix]
-            [frontend.handler.file-based.property.util :as property-util]
-            [cljs.pprint :as pprint]
-            [malli.core :as m]
-            [malli.util :as mu]))
+            [frontend.handler.file-based.property.util :as property-util]))
 
 (defn new-outliner-txs-state [] (atom []))
 
@@ -25,28 +21,6 @@
    (instance? cljs.core/Atom state)
    (coll? @state)))
 
-(defn- validate-db!
-  "Validates the entities that have changed in the given datascript tx-report.
-   Validation is only for DB graphs"
-  [{:keys [db-after tx-data tx-meta]}]
-  (let [{:keys [known-schema? closed-schema? fail-invalid?]} (:dev/validate-db-options (state/get-config))
-        changed-ids (->> tx-data (map :e) distinct)
-        ent-maps* (->> changed-ids (mapcat #(d/datoms db-after :eavt %)) db-malli-schema/datoms->entity-maps vals)
-        ent-maps (vec (db-malli-schema/update-properties-in-ents ent-maps*))
-        db-schema (cond-> (if known-schema? db-malli-schema/DB-known db-malli-schema/DB)
-                    true
-                    (db-malli-schema/update-properties-in-schema db-after)
-                    closed-schema?
-                    mu/closed-schema)]
-    (js/console.log "changed eids:" changed-ids tx-meta)
-    (when-let [errors (->> ent-maps
-                           (m/explain db-schema)
-                           :errors)]
-      (js/console.error "Invalid datascript entities detected amongst changed entity ids:" changed-ids)
-      (pprint/pprint {:errors errors})
-      (pprint/pprint {:entity-maps ent-maps})
-      (when fail-invalid? (js/alert "Invalid DB!")))))
-
 (defn after-transact-pipelines
   [{:keys [tx-meta] :as opts}]
   (when-not config/test?
@@ -55,13 +29,6 @@
 
     (pipelines/invoke-hooks opts)
 
-    ;; TODO: move validate to worker
-    ;; (when (and config/dev?
-    ;;            (config/db-based-graph? (state/get-current-repo))
-    ;;             ;; Skip tx with update-tx-ids? because they are immediately followed by the original block tx
-    ;;            (not (:update-tx-ids? tx-meta)))
-    ;;   (validate-db! tx-report))
-
     (when (or (:outliner/transact? tx-meta)
               (:outliner-op tx-meta)
               (:whiteboard/transact? tx-meta))

src/main/frontend/persist_db/browser.cljs

@@ -86,7 +86,8 @@
         (let [tx-meta' (pr-str tx-meta)
               tx-data' (pr-str tx-data)
               context (if (config/db-based-graph? repo)
-                        {}
+                        {:dev? config/dev?
+                         :validate-db-options (:dev/validate-db-options (state/get-config))}
                         {:importing? (:graph/importing @state/state)
                          :date-formatter (state/get-date-formatter)
                          :export-bullet-indentation (state/get-export-bullet-indentation)

src/main/frontend/state.cljs

@@ -16,7 +16,6 @@
             [goog.dom :as gdom]
             [goog.object :as gobj]
             [logseq.graph-parser.config :as gp-config]
-            [malli.core :as m]
             [medley.core :as medley]
             [promesa.core :as p]
             [rum.core :as rum]))
@@ -37,12 +36,6 @@
      {:route-match                           nil
       :today                                 nil
       :system/events                         (async/chan 1000)
-      :file/writes                           (let [coercer (m/coercer [:catn
-                                                                       [:repo :string]
-                                                                       [:page-id :any]
-                                                                       [:outliner-op :any]
-                                                                       [:epoch :int]])]
-                                               (async/chan 10000 (map coercer)))
       :file/unlinked-dirs                    #{}
       :reactive/custom-queries               (async/chan 1000)
       :notification/show?                    false
@@ -1551,10 +1544,6 @@ Similar to re-frame subscriptions"
              :modal/dropdowns {}
              :ui/open-select nil))))
 
-(defn get-file-write-chan
-  []
-  (:file/writes @state))
-
 (defn get-reactive-custom-queries-chan
   []
   (:reactive/custom-queries @state))

src/main/frontend/util/fs.cljs

@@ -3,7 +3,6 @@
 (ns frontend.util.fs
   "Misc util fns built on top of frontend.fs"
   (:require ["path" :as node-path]
-            [frontend.util :as util]
             [logseq.graph-parser.util :as gp-util]
             [clojure.string :as string]
             [frontend.state :as state]

src/main/frontend/worker/file.cljs

@@ -12,8 +12,7 @@
             [logseq.db :as ldb]
             [malli.core :as m]
             [frontend.worker.state :as state]
-            [goog.object :as gobj]
-            [frontend.worker.util :as util]))
+            [goog.object :as gobj]))
 
 (def *writes file/*writes)
 
@@ -94,7 +93,7 @@
 
 (defn sync-to-file
   [repo page-id tx-meta]
-  (when (and (util/local-file-based-graph? repo)
+  (when (and (worker-util/local-file-based-graph? repo)
              page-id
              (not (:created-from-journal-template? tx-meta))
              (not (:delete-files? tx-meta)))

src/main/frontend/worker/pipeline.cljs

@@ -4,7 +4,9 @@
             [logseq.outliner.datascript-report :as ds-report]
             [logseq.outliner.pipeline :as outliner-pipeline]
             [frontend.worker.react :as worker-react]
-            [frontend.worker.file :as file]))
+            [frontend.worker.file :as file]
+            [logseq.db.frontend.validate :as validate]
+            [logseq.db.sqlite.util :as sqlite-util]))
 
 (defn- path-refs-need-recalculated?
   [tx-meta]
@@ -42,6 +44,10 @@
 
 (defn invoke-hooks
   [repo conn tx-report context]
+
+  (when (and (:dev? context) (sqlite-util/db-based-graph? repo))
+    (validate/validate-db! tx-report (:validate-db-options context)))
+
   (let [tx-meta (:tx-meta tx-report)
         {:keys [from-disk? new-graph?]} tx-meta]
     (if (or from-disk? new-graph?)

src/test/frontend/db/name_sanity_test.cljs

@@ -4,13 +4,14 @@
             [logseq.graph-parser.util :as gp-util]
             [frontend.handler.file-based.page :as file-page-handler]
             [frontend.handler.conversion :as conversion-handler]
-            [frontend.util.fs :as fs-util]))
+            [frontend.util.fs :as fs-util]
+            [frontend.worker.file.util :as wfu]))
 
 (defn- test-page-name
   "Check if page name can be preserved after escaping"
   [page-name]
   (testing (str "Test sanitization page-name: " page-name)
-    (let [file-name   (#'fs-util/tri-lb-file-name-sanity page-name)
+    (let [file-name   (#'wfu/tri-lb-file-name-sanity page-name)
           page-name'  (#'gp-util/tri-lb-title-parsing file-name)
           url-single  (js/encodeURIComponent file-name)
           url-double  (js/encodeURIComponent url-single)

src/test/frontend/modules/outliner/pipeline_test.cljs

@@ -2,7 +2,7 @@
   (:require [cljs.test :refer [deftest is use-fixtures testing]]
             [datascript.core :as d]
             [frontend.db :as db]
-            [frontend.modules.outliner.pipeline :as pipeline]
+            [frontend.worker.pipeline :as pipeline]
             [frontend.test.helper :as test-helper :refer [load-test-files]]))
 
 (use-fixtures :each test-helper/start-and-destroy-db)
@@ -44,4 +44,4 @@
                :path-ref-names ["page1" "bar" "baz"]}
               {:block/content "grandchild #bing"
                :path-ref-names ["page1" "bar" "baz" "bing"]}]
-             updated-blocks)))))
+             updated-blocks)))))

src/test/frontend/util/property_test.cljs

@@ -1,6 +1,7 @@
 (ns frontend.util.property-test
   (:require [cljs.test :refer [are deftest testing]]
-            [frontend.handler.file-based.property.util :as property-util]))
+            [frontend.handler.file-based.property.util :as property-util]
+            [frontend.worker.file.property-util :as wfp]))
 
 (deftest remove-id-property
   (testing "org"
@@ -167,14 +168,14 @@ SCHEDULED: <2021-10-25 Mon>\n:PROPERTIES:\n:a: b\n:END:\nworld\n" "c" "d")
     "abcd\nempty::\nanother-empty::\nid:: 123"))
 
 (deftest test-build-properties-str
-  (are [x y] (= (#'property-util/build-properties-str :mardown x) y)
+  (are [x y] (= (#'wfp/build-properties-str :mardown x) y)
     {:title "a"}
     "title:: a\n"
     {:title "a/b/c"}
     "title:: a/b/c\n"
     {:title "a/b/c" :tags "d,e"}
     "title:: a/b/c\ntags:: d,e\n")
-  (are [x y] (= (#'property-util/build-properties-str :org x) y)
+  (are [x y] (= (#'wfp/build-properties-str :org x) y)
     {:title "a"}
     ":PROPERTIES:\n:title: a\n:END:"
     {:title "a/b/c"}