enhance(dev): add deps/db/src/logseq/{frontend,sqlite} to

separate linter. Will catch mistakes like
e92b107b5b47d93d98ef2520dd420bbc33d95bb1 next time. Also fixed
a few more ldb/* calls and expanded /db-based-graph? check

deps/db/README.md

@@ -4,16 +4,19 @@ This library provides an API to the
 frontend([datascript](https://github.com/tonsky/datascript)) and
 backend([SQLite](https://www.sqlite.org/index.html)) databases from the Logseq
 app and the CLI. The majority of this library is focused on supporting DB graphs
-but there are a few older namespaces that support file graphs. This library is
+but there are a few namespaces that support file graphs. This library is
 compatible with ClojureScript and with
 [nbb-logseq](https://github.com/logseq/nbb-logseq) to respectively provide
 frontend and commandline functionality.
 
 ## API
 
-This library is under the parent namespace `logseq.db`. While `logseq.db` is the
-main entry point, this library also provides frontend namespaces under
-`logseq.db.frontend` and backend/sqlite namespaces under `logseq.db.sqlite`.
+This library is under the parent namespace `logseq.db`. It provides the following namespaces:
+* `logseq.db` - main entry point serving both file and DB graphs
+* `logseq.db.frontend.*` - frontend namespaces for DB graphs
+* `logseq.db.sqlite.*` - backend/sqlite namespaces for DB graphs
+* `logseq.db.file-based.*` - namespaces for file graphs, mostly old namespaces
+* `logseq.db.common.*` - namespaces for both file and DB graphs
 
 ## Usage
 

deps/db/src/logseq/db/frontend/class.cljs

@@ -102,7 +102,7 @@
 
 (def private-tags
   "Built-in classes that are private and should not be used by a user directly.
-  These used to be in :block/type"
+  These used to be in block/type"
   (set/union internal-tags
              #{:logseq.class/Journal :logseq.class/Whiteboard}))
 

deps/db/src/logseq/db/frontend/content.cljs

@@ -1,11 +1,10 @@
 (ns logseq.db.frontend.content
   "Fns to handle block content e.g. internal ids"
   (:require [clojure.string :as string]
-            [logseq.common.util.page-ref :as page-ref]
             [datascript.core :as d]
             [logseq.common.util :as common-util]
-            [logseq.db.frontend.entity-util :as entity-util]
-            [logseq.db.frontend.entity-plus :as entity-plus]))
+            [logseq.common.util.page-ref :as page-ref]
+            [logseq.db.frontend.entity-util :as entity-util]))
 
 ;; [[uuid]]
 (def id-ref-pattern
@@ -122,11 +121,9 @@
 (defn update-block-content
   "Replace `[[internal-id]]` with `[[page name]]`"
   [db item eid]
-  (if (entity-plus/db-based-graph? db)
-    (if-let [content (:block/title item)]
-      (let [refs (:block/refs (d/entity db eid))]
-        (assoc item :block/title (id-ref->title-ref content refs false)))
-      item)
+  (if-let [content (:block/title item)]
+    (let [refs (:block/refs (d/entity db eid))]
+      (assoc item :block/title (id-ref->title-ref content refs false)))
     item))
 
 (defn replace-tags-with-id-refs

deps/db/src/logseq/db/frontend/entity_plus.cljc

@@ -16,9 +16,9 @@
 
 (def nil-db-ident-entities
   "No such entities with these :db/ident, but `(d/entity <db> <ident>)` has been called somewhere."
-  #{:block/tx-id :block/warning :block/pre-block? :block/uuid :block/scheduled
-    :block/deadline :block/journal-day :block/level :block/heading-level
-    :block/type :block/name :block/marker :block/_refs
+  #{:block/tx-id :block/uuid :block/journal-day :block/_refs :block/level :block/heading-level :block/warning
+    ;; File graph only attributes. Can these be removed if this is only called in db graphs?
+    :block/pre-block? :block/scheduled :block/deadline :block/type :block/name :block/marker
 
     :block.temp/ast-title :block.temp/top? :block.temp/bottom? :block.temp/search?
     :block.temp/fully-loaded? :block.temp/ast-body

deps/db/src/logseq/db/frontend/malli_schema.cljs

@@ -536,9 +536,8 @@
 (let [malli-one-ref-attrs (->> (concat property-attrs page-attrs block-attrs page-or-block-attrs (rest normal-page))
                                (filter #(= (last %) :int))
                                (map first)
-                               set)
-      attrs-to-ignore #{:block/file}]
-  (when-let [undeclared-ref-attrs (seq (remove (some-fn malli-one-ref-attrs attrs-to-ignore) db-schema/card-one-ref-type-attributes))]
+                               set)]
+  (when-let [undeclared-ref-attrs (seq (remove malli-one-ref-attrs db-schema/card-one-ref-type-attributes))]
     (throw (ex-info (str "The malli DB schema is missing the following cardinality-one ref attributes from datascript's schema: "
                          (string/join ", " undeclared-ref-attrs))
                     {}))))

deps/db/src/logseq/db/sqlite/export.cljs

@@ -7,6 +7,7 @@
             [logseq.db :as ldb]
             [logseq.db.frontend.class :as db-class]
             [logseq.db.frontend.content :as db-content]
+            [logseq.db.frontend.entity-util :as entity-util]
             [logseq.db.frontend.entity-plus :as entity-plus]
             [logseq.db.frontend.property :as db-property]
             [logseq.db.sqlite.build :as sqlite-build]))
@@ -30,7 +31,7 @@
    Pages that are shallow copied are at the edges of export and help keep the export size reasonable and
    avoid exporting unexpected info"
   [page-entity]
-  (if (ldb/journal? page-entity)
+  (if (entity-util/journal? page-entity)
     {:build/journal (:block/journal-day page-entity)}
     {:block/title (block-title page-entity)}))
 
@@ -42,7 +43,7 @@
         [:build/page (cond-> (shallow-copy-page pvalue)
                        (seq (:block/tags pvalue))
                        (assoc :build/tags (->build-tags (:block/tags pvalue))))]
-        (ldb/journal? pvalue)
+        (entity-util/journal? pvalue)
         [:build/page {:build/journal (:block/journal-day pvalue)}]
         :else
         (if (= :node (:logseq.property/type property-ent))
@@ -223,7 +224,7 @@
   [db page-blocks]
   (let [content-ref-uuids (set (mapcat (comp db-content/get-matched-ids block-title) page-blocks))
         content-ref-ents (map #(d/entity db [:block/uuid %]) content-ref-uuids)
-        content-ref-pages (filter #(or (ldb/internal-page? %) (ldb/journal? %)) content-ref-ents)
+        content-ref-pages (filter #(or (ldb/internal-page? %) (entity-util/journal? %)) content-ref-ents)
         content-ref-properties (when-let [prop-ids (seq (map :db/ident (filter ldb/property? content-ref-ents)))]
                                  (update-vals (build-export-properties db prop-ids {:include-uuid? true :shallow-copy? true})
                                               #(merge % {:build/keep-uuid? true})))
@@ -288,7 +289,7 @@
      :pvalue-uuids @*pvalue-uuids}))
 
 (defn- build-uuid-block-export [db pvalue-uuids content-ref-ents {:keys [page-entity]}]
-  (let [content-ref-blocks (set (remove ldb/page? content-ref-ents))
+  (let [content-ref-blocks (set (remove entity-util/page? content-ref-ents))
         uuid-block-ents-to-export (concat (map #(d/entity db [:block/uuid %]) pvalue-uuids)
                                           content-ref-blocks)
         uuid-block-pages

scripts/src/logseq/tasks/dev/db_and_file_graphs.clj

@@ -1,5 +1,6 @@
 (ns logseq.tasks.dev.db-and-file-graphs
   (:require [babashka.process :refer [shell]]
+            [clojure.set :as set]
             [clojure.string :as string]))
 
 (defn- escape-shell-regex
@@ -33,18 +34,23 @@
          "frontend.components.file-based"
          "frontend.util.fs"]))
 
+(def block-name-db-graph-paths
+  "DB graph paths with :block/name"
+  ["deps/db/src/logseq/db/frontend"
+   "deps/db/src/logseq/db/sqlite"
+   "src/main/frontend/worker/handler/page/db_based"])
+
 (def db-graph-paths
   "Paths _only_ for DB graphs"
-  ["src/main/frontend/handler/db_based"
-   ;; TODO: Re-enable if db namespace support is moved elsewhere
-  ;;  "src/main/frontend/worker/handler/page/db_based"
-   "src/main/frontend/components/class.cljs"
-   "src/main/frontend/components/property.cljs"
-   "src/main/frontend/components/property"
-   "src/main/frontend/components/objects.cljs"
-   "src/main/frontend/components/db_based"
-   "src/main/frontend/components/query/view.cljs"
-   "src/electron/electron/db.cljs"])
+  (into block-name-db-graph-paths
+        ["src/main/frontend/handler/db_based"
+         "src/main/frontend/components/class.cljs"
+         "src/main/frontend/components/property.cljs"
+         "src/main/frontend/components/property"
+         "src/main/frontend/components/objects.cljs"
+         "src/main/frontend/components/db_based"
+         "src/main/frontend/components/query/view.cljs"
+         "src/electron/electron/db.cljs"]))
 
 (def file-graph-paths
   "Paths _only_ for file graphs"
@@ -55,11 +61,16 @@
    "src/main/frontend/components/file_based"
    "src/main/frontend/util/fs.cljs"])
 
+(defn- grep-many
+  "Git greps a coll of patterns for given paths. Returns result from process/shell"
+  [patterns paths]
+  (apply shell {:out :string :continue true}
+         "git grep -E" (str "(" (string/join "|" patterns) ")")
+         paths))
+
 (defn- validate-db-ns-not-in-file
   []
-  (let [res (apply shell {:out :string :continue true}
-                   "git grep -E" (str "(" (string/join "|" db-graph-ns) ")")
-                   file-graph-paths)]
+  (let [res (grep-many db-graph-ns file-graph-paths)]
     (when-not (and (= 1 (:exit res)) (= "" (:out res)))
       (println "The following db graph namespaces should not be in file graph files:")
       (println (:out res))
@@ -67,9 +78,7 @@
 
 (defn- validate-file-ns-not-in-db
   []
-  (let [res (apply shell {:out :string :continue true}
-                   "git grep -E" (str "(" (string/join "|" file-graph-ns) ")")
-                   db-graph-paths)]
+  (let [res (grep-many file-graph-ns db-graph-paths)]
     (when-not (and (= 1 (:exit res)) (= "" (:out res)))
       (println "The following file graph namespaces should not be in db graph files:")
       (println (:out res))
@@ -77,12 +86,11 @@
 
 (defn- validate-multi-graph-fns-not-in-file-or-db
   []
-  (let [multi-graph-fns ["config/db-based-graph\\?" "sqlite-util/db-based-graph\\?"
+  ;; TODO: Lint `(db-based-graph?` when db.frontend.entity-plus is split into separate graph contexts
+  (let [multi-graph-fns ["/db-based-graph\\?"
                          ;; Use file-entity-util and entity-util when in a single graph context
                          "ldb/whiteboard\\?" "ldb/journal\\?" "ldb/page\\?"]
-        res (apply shell {:out :string :continue true}
-                   "git grep -E" (str "(" (string/join "|" multi-graph-fns) ")")
-                   (into file-graph-paths db-graph-paths))]
+        res (grep-many multi-graph-fns (into file-graph-paths db-graph-paths))]
     (when-not (and (= 1 (:exit res)) (= "" (:out res)))
       (println "The following files should not have fns meant to be used in multi-graph contexts:")
       (println (:out res))
@@ -91,37 +99,47 @@
 (defn- validate-file-concepts-not-in-db
   []
   (let [file-concepts (->>
-                       ;; from logseq.db.frontend.schema
+                       ;; from logseq.db.file-based.schema
                        [:block/namespace :block/properties-text-values :block/pre-block :recent/pages :block/file :block/properties-order
                         :block/repeated :block/deadline :block/scheduled :block/priority :block/marker :block/macros
                         :block/type :block/format]
                        (map str)
                        (into [;; e.g. block/properties :title
                               "block/properties :"
-                              "block/name"
-                              ;; anything org mode
-                              "org"
+                              ;; anything org mode except for org.babashka
+                              "org[^\\.]"
                               "#+BEGIN_"
                               "#+END_"
-                              "pre-block"
-                              "db/get-page"
-                              "/page-name-sanity-lc"]))
+                              "pre-block"]))
         ;; For now use the whole code line. If this is too brittle can make this smaller
-        allowed-exceptions #{"{:block/name page-title})))"
+        allowed-exceptions #{":block/pre-block? :block/scheduled :block/deadline :block/type :block/name :block/marker"
+                             "(dissoc :block/format))]"
+                             ;; The next 3 are from components.property.value
                              "{:block/name page-title})"
                              "(when-not (db/get-page journal)"
-                             "(let [value (if datetime? (tc/to-long d) (db/get-page journal))]"
-                             "(dissoc :block/format))]"}
-        res (apply shell {:out :string :continue true}
-                   "git grep -E" (str "(" (string/join "|" file-concepts) ")")
-                   db-graph-paths)
+                             "(let [value (if datetime? (tc/to-long d) (db/get-page journal))]"}
+        res (grep-many file-concepts db-graph-paths)
         invalid-lines (when (= 0 (:exit res))
                         (remove #(some->> (string/split % #":\s+") second string/trim (contains? allowed-exceptions))
-                                (string/split-lines (:out res))))]
-    (when (> (:exit res) 1) (System/exit 1))
-    (when (and (= 0 (:exit res)) (seq invalid-lines))
+                                (string/split-lines (:out res))))
+        _ (when (> (:exit res) 1) (System/exit 1))
+        _ (when (and (= 0 (:exit res)) (seq invalid-lines))
+            (println "The following files should not have contained file specific concepts:")
+            (println (string/join "\n" invalid-lines))
+            (System/exit 1))
+
+        ;; :block/name isn't used in db graphs except for fns with journal or internal-page
+        block-name-file-concepts #{"block/name" "/page-name-sanity-lc" "db/get-page"}
+        no-block-name-db-graph-paths (set/difference (set db-graph-paths) (set block-name-db-graph-paths))
+        block-name-res (grep-many block-name-file-concepts no-block-name-db-graph-paths)
+        block-name-invalid-lines (when (= 0 (:exit block-name-res))
+                                   (remove #(some->> (string/split % #":\s+") second string/trim (contains? allowed-exceptions))
+                                           (string/split-lines (:out block-name-res))))]
+
+    (when (> (:exit block-name-res) 1) (System/exit 1))
+    (when (and (= 0 (:exit block-name-res)) (seq block-name-invalid-lines))
       (println "The following files should not have contained file specific concepts:")
-      (println (string/join "\n" invalid-lines))
+      (println (string/join "\n" block-name-invalid-lines))
       (System/exit 1))))
 
 (defn- validate-db-concepts-not-in-file
@@ -130,9 +148,7 @@
         ;; from logseq.db.frontend.schema
         ["closed-value" "class/properties" "classes" "property/parent"
          "logseq.property" "logseq.class"]
-        res (apply shell {:out :string :continue true}
-                   "git grep -E" (str "(" (string/join "|" db-concepts) ")")
-                   file-graph-paths)]
+        res (grep-many db-concepts file-graph-paths)]
     (when-not (and (= 1 (:exit res)) (= "" (:out res)))
       (println "The following files should not have contained db specific concepts:")
       (println (:out res))

src/main/frontend/common/file/core.cljs

@@ -1,12 +1,13 @@
 (ns frontend.common.file.core
-  "Save file to disk. Used by both file and DB graphs and shared
-   by worker and frontend namespaces"
+  "Convert blocks to file content. Used for exports and saving file to disk. Shared
+  by worker and frontend namespaces"
   (:require [clojure.string :as string]
-            [logseq.graph-parser.property :as gp-property]
             [datascript.core :as d]
             [logseq.db :as ldb]
             [logseq.db.frontend.content :as db-content]
+            [logseq.db.frontend.entity-plus :as entity-plus]
             [logseq.db.sqlite.util :as sqlite-util]
+            [logseq.graph-parser.property :as gp-property]
             [logseq.outliner.tree :as otree]))
 
 (defn- indented-block-content
@@ -107,6 +108,12 @@
   [repo db tree opts context]
   (->> (tree->file-content-aux repo db tree opts context) (string/join "\n")))
 
+(defn- update-block-content
+  [db item eid]
+  ;; This may not be needed if this becomes a file-graph only context
+  (if (entity-plus/db-based-graph? db)
+    (db-content/update-block-content db item eid)
+    item))
 
 (defn block->content
   "Converts a block including its children (recursively) to plain-text."
@@ -117,7 +124,7 @@
                          0
                          1))
         blocks (->> (d/pull-many db '[*] (keep :db/id (ldb/get-block-and-children db root-block-uuid)))
-                    (map #(db-content/update-block-content db % (:db/id %))))
+                    (map #(update-block-content db % (:db/id %))))
         tree (otree/blocks->vec-tree repo db blocks (str root-block-uuid))]
     (tree->file-content repo db tree
                         (assoc tree->file-opts :init-level init-level)

src/main/frontend/db/utils.cljs

@@ -3,7 +3,8 @@
   (:require [datascript.core :as d]
             [frontend.state :as state]
             [frontend.db.conn :as conn]
-            [logseq.db.frontend.content :as db-content]))
+            [logseq.db.frontend.content :as db-content]
+            [logseq.db.frontend.entity-plus :as entity-plus]))
 
 ;; transit serialization
 
@@ -46,7 +47,9 @@
   "Replace `[[internal-id]]` with `[[page name]]`"
   [item eid]
   (if-let [db (conn/get-db)]
-    (db-content/update-block-content db item eid)
+    (if (entity-plus/db-based-graph? db)
+      (db-content/update-block-content db item eid)
+      item)
     item))
 
 (defn pull

src/main/frontend/worker/db_worker.cljs

@@ -33,8 +33,8 @@
             [logseq.common.util :as common-util]
             [logseq.db :as ldb]
             [logseq.db.common.order :as db-order]
-            [logseq.db.frontend.schema :as db-schema]
             [logseq.db.common.sqlite :as sqlite-common-db]
+            [logseq.db.frontend.schema :as db-schema]
             [logseq.db.sqlite.create-graph :as sqlite-create-graph]
             [logseq.db.sqlite.export :as sqlite-export]
             [logseq.db.sqlite.util :as sqlite-util]
@@ -959,6 +959,18 @@
     :delete-page (fn [repo conn [page-uuid]]
                    (delete-page! repo conn page-uuid))}))
 
+(defn- <ratelimit-file-writes!
+  []
+  (file/<ratelimit-file-writes!
+   (fn [col]
+     (when (seq col)
+       (let [repo (ffirst col)
+             conn (worker-state/get-datascript-conn repo)]
+         (if conn
+           (when-not (ldb/db-based-graph? @conn)
+             (file/write-files! conn col (worker-state/get-context)))
+           (js/console.error (str "DB is not found for " repo))))))))
+
 (defn init
   "web worker entry"
   []
@@ -967,7 +979,7 @@
   (let [^js obj (DBWorker.)]
     (outliner-register-op-handlers!)
     (worker-state/set-worker-object! obj)
-    (file/<ratelimit-file-writes!)
+    (<ratelimit-file-writes!)
     (js/setInterval #(.postMessage js/self "keepAliveResponse") (* 1000 25))
     (Comlink/expose obj)
     (reset! worker-state/*main-thread (Comlink/wrap js/self))))

src/main/frontend/worker/file.cljs

@@ -203,15 +203,7 @@
       (async/put! file-writes-chan [repo page-id (:outliner-op tx-meta) (tc/to-long (t/now)) request-id]))))
 
 (defn <ratelimit-file-writes!
-  []
+  [flush-fn]
   (async-util/<ratelimit file-writes-chan batch-write-interval
-                          :filter-fn (fn [_] true)
-                          :flush-fn
-                          (fn [col]
-                            (when (seq col)
-                              (let [repo (ffirst col)
-                                    conn (worker-state/get-datascript-conn repo)]
-                                (if conn
-                                  (when-not (ldb/db-based-graph? @conn)
-                                    (write-files! conn col (worker-state/get-context)))
-                                  (js/console.error (str "DB is not found for ") repo)))))))
+                         :filter-fn (fn [_] true)
+                         :flush-fn flush-fn))