perf: db reload

Load as less pages && closed values as we can to avoid loading too
much data.

We already know that transacting db attributes is slow, going to
investigate it later.

deps/db/src/logseq/db/sqlite/common_db.cljs

@@ -27,10 +27,11 @@
       (map #(d/pull db '[*] [:file/path %]) files))))
 
 (defn get-all-pages
-  [db]
+  [db exclude-page-ids]
   (->> (d/datoms db :avet :block/name)
-       (map (fn [e]
-              (d/pull db '[*] (:e e))))))
+       (keep (fn [e]
+               (when-not (contains? exclude-page-ids (:e e))
+                 (d/pull db '[:db/id :block/uuid :block/name :block/original-name :block/alias :block/type] (:e e)))))))
 
 (defn get-all-files
   [db]
@@ -71,33 +72,47 @@
   [b]
   (assoc b :block.temp/fully-loaded? true))
 
+(comment
+  (defn- property-without-db-attrs
+    [property]
+    (dissoc property :db/index :db/valueType :db/cardinality)))
+
 (defn- property-with-values
   [db block]
-  (when (seq (:block/raw-properties block))
-    (let [pairs (d/pull-many db '[*] (map :db/id (:block/raw-properties block)))]
-      (mapcat
-       (fn [pair]
-         (let [property (d/entity db (:db/id (:property/pair-property pair)))
-               property-values (get pair (:db/ident property))
-               values (if (set? property-values) property-values #{property-values})
-               value-ids (when (every? map? values)
-                           (->> (map :db/id values)
-                                (filter (fn [id] (or (int? id) (keyword? id))))))
-               value-blocks (when (seq value-ids)
-                              (mapcat
-                               (fn [id]
-                                 (let [b (d/entity db id)]
-                                   (cons (d/pull db '[*] id)
-                                         (let [ids (map :db/id (:block/raw-properties b))]
-                                           (when (seq ids)
-                                             (d/pull-many db '[*] ids))))))
-                               value-ids))
-               page (when (seq values)
-                      (when-let [page-id (:db/id (:block/page (d/entity db (:db/id (first values)))))]
-                        (d/pull db '[*] page-id)))]
-
-           (remove nil? (concat [page] value-blocks [pair]))))
-       pairs))))
+  (let [block (d/entity db (:db/id block))
+        class-properties (when (contains? (:block/type block) "class")
+                           (let [property-ids (map :db/id (:class/schema.properties block))]
+                             (when (seq property-ids)
+                               (d/pull-many db '[*] property-ids))))
+        block-properties (when (seq (:block/raw-properties block))
+                           (let [pairs (d/pull-many db '[*] (map :db/id (:block/raw-properties block)))]
+                             (mapcat
+                              (fn [pair]
+                                (let [property (d/entity db (:db/id (:property/pair-property pair)))
+                                      property-values (get pair (:db/ident property))
+                                      values (if (coll? property-values) property-values #{property-values})
+                                      value-ids (when (every? map? values)
+                                                  (->> (map :db/id values)
+                                                       (filter (fn [id] (or (int? id) (keyword? id))))))
+                                      value-blocks (when (seq value-ids)
+                                                     (mapcat
+                                                      (fn [id]
+                                                        (let [b (d/entity db id)]
+                                                          (cons (d/pull db '[*] id)
+                                                                (let [ids (map :db/id (:block/raw-properties b))]
+                                                                  (when (seq ids)
+                                                                    (d/pull-many db '[*] ids))))))
+                                                      value-ids))
+                                      page (when (seq values)
+                                             (when-let [page-id (:db/id (:block/page (d/entity db (:db/id (first values)))))]
+                                               (d/pull db '[*] page-id)))
+                                      property' (d/pull db '[*] (:db/id property))]
+                                  (remove nil? (concat [page]
+                                                       [property']
+                                                       value-blocks
+                                                       [pair]))))
+                              pairs)))]
+    (concat class-properties block-properties)))
 
 (defn get-block-and-children
   [db id children?]
@@ -127,17 +142,15 @@
                           (with-block-refs db)
                           mark-block-fully-loaded)]
           (cond->
-           {:block block'}
-            (seq (:block/raw-properties block))
-            (assoc :property-pairs (property-with-values db block))
+           {:block block'
+            :properties (property-with-values db block)}
             children?
             (assoc :children (get-children (:block/_parent block)))))
         (cond->
          {:block (->> (d/pull db '[*] (:db/id block))
                       (with-tags db)
-                      mark-block-fully-loaded)}
-          (seq (:block/raw-properties block))
-          (assoc :property-pairs (property-with-values db block))
+                      mark-block-fully-loaded)
+          :properties (property-with-values db block)}
           children?
           (assoc :children
                  (if (contains? (:block/type block) "whiteboard")
@@ -163,32 +176,26 @@
      (reverse)
      (take n))))
 
-(defn- block-with-properties
-  [db e]
-  (cons (d/pull db '[*] (:db/id e))
-        (property-with-values db e)))
-
 (defn get-structured-blocks
   [db]
-  (let [blocks (->> (d/datoms db :avet :block/type)
-                    (keep (fn [datom]
-                            (when (contains? #{"closed value" "property" "class"} (:v datom))
-                              (d/entity db (:e datom))))))
-        blocks-data (mapcat (fn [block] (block-with-properties db block)) blocks)
-        default-block-values (mapcat (fn [block]
-                                       (when (and (contains? (:block/type block) "property")
-                                                  (= :default (get-in block [:block/schema :type]))
-                                                  (not (:logseq.property/built-in? block)))
-                                         (let [values (get-in block [:block/schema :values])]
-                                           (mapcat (fn [id]
-                                                     (when (uuid? id)
-                                                       (let [e (d/entity db [:block/uuid id])
-                                                             children (:block/_parent e)]
-                                                         (concat
-                                                          (block-with-properties db e)
-                                                          (mapcat #(block-with-properties db %) children))))) values))))
-                                     blocks)]
-    (concat blocks-data default-block-values)))
+  (let [classes (d/pull-many db '[*] [:logseq.class/task :logseq.class/card])
+        property-ids (->> classes
+                          (mapcat :class/schema.properties)
+                          (map :db/id)
+                          (into #{:block/tags :block/alias}))
+        properties (d/pull-many db '[*] property-ids)
+        closed-values (->> properties
+                           (mapcat (fn [block]
+                                     (let [values (get-in block [:block/schema :values])]
+                                       (when (seq values)
+                                         (d/pull-many db '[*] (map #(vector :block/uuid %) values))))))
+                           (mapcat (fn [block] (cons block (property-with-values db block))))
+                           (map (fn [block]
+                                  (let [val (:logseq.property/created-from-property block)]
+                                    (if (keyword? (:db/id val))
+                                      (assoc block :logseq.property/created-from-property {:db/id (:db/id (d/entity db (:db/id val)))})
+                                      block)))))]
+    (concat closed-values properties classes)))
 
 (defn get-favorites
   "Favorites page and its blocks"
@@ -224,10 +231,7 @@
   [db]
   (let [schema (->> (:schema db)
                     (remove (fn [[k _v]]
-                              (or (integer? k)
-                                  (and (keyword? k)
-                                       ;; filters out all logseq-ident-namespaces
-                                       (string/starts-with? (namespace k) "logseq.")))))
+                              (integer? k)))
                     (into {}))
         db-type-ident (when (d/entity db :logseq.kv/db-type)
                         (d/pull db '[*] :logseq.kv/db-type))

src/main/frontend/components/block.cljs

@@ -707,9 +707,8 @@
    - `:preview?`: Is this component under preview mode? (If true, `page-preview-trigger` won't be registered to this `page-cp`)"
   [state {:keys [html-export? label children contents-page? preview? disable-preview?] :as config} page]
   (let [page-entity (::page-entity state)]
-    (when (:db/id page-entity)
-      (let [page-entity (when page-entity (db/sub-block (:db/id page-entity)))
-            page-name (or (:block/name page-entity)
+    (when-let [page-entity (when page-entity (db/sub-block (:db/id page-entity)))]
+      (let [page-name (or (:block/name page-entity)
                           (:block/name page))
             whiteboard-page? (model/whiteboard-page? page-name)
             inner (page-inner config page-entity contents-page? children html-export? label whiteboard-page?)

src/main/frontend/components/page.cljs

@@ -17,6 +17,7 @@
             [frontend.context.i18n :refer [t]]
             [frontend.date :as date]
             [frontend.db :as db]
+            [frontend.db.async :as db-async]
             [frontend.db-mixins :as db-mixins]
             [frontend.db.model :as model]
             [frontend.extensions.graph :as graph]
@@ -47,7 +48,6 @@
             [reitit.frontend.easy :as rfe]
             [rum.core :as rum]
             [frontend.extensions.graph.pixi :as pixi]
-            [frontend.db.async :as db-async]
             [logseq.db :as ldb]
             [frontend.handler.property.util :as pu]
             [frontend.components.hierarchy :as hierarchy]))
@@ -221,19 +221,24 @@
                  query))
              (str repo "-custom-query-" (:query query))))]))))
 
-(defn tagged-pages
+(rum/defc tagged-pages
   [repo tag tag-original-name]
-  (let [pages (db/get-tag-pages repo tag)]
+  (let [[pages set-pages!] (rum/use-state nil)]
+    (rum/use-effect!
+     (fn []
+       (p/let [result (db-async/<get-tag-pages repo (:db/id tag))]
+         (set-pages! result)))
+     [tag])
     (when (seq pages)
       [:div.references.page-tags.mt-6.flex-1.flex-row
        [:div.content
         (ui/foldable
-          [:h2.font-bold.opacity-50 (util/format "Pages tagged with \"%s\"" tag-original-name)]
-          [:ul.mt-2
-           (for [[original-name name] (sort-by last pages)]
-             [:li {:key (str "tagged-page-" name)}
-              (component-block/page-cp {} {:block/name name :block/original-name original-name})])]
-          {:default-collapsed? false})]])))
+         [:h2.font-bold.opacity-50 (util/format "Pages tagged with \"%s\"" tag-original-name)]
+         [:ul.mt-2
+          (for [page (sort-by :block/original-name pages)]
+            [:li {:key (str "tagged-page-" (:db/id page))}
+             (component-block/page-cp {} page)])]
+         {:default-collapsed? false})]])))
 
 (rum/defc page-title-editor < rum/reactive
   [page {:keys [*input-value *title-value *edit? untitled? page-name old-name whiteboard-page?]}]
@@ -498,8 +503,9 @@
             (when (or page-name block-or-whiteboard?)
               [:div.flex-1.page.relative
                (merge (if (seq (:block/tags page))
-                        (let [page-names (model/get-page-names-by-ids (map :db/id (:block/tags page)))]
-                          {:data-page-tags (text-util/build-data-value page-names)})
+                        (let [page-names (map :block/original-name (:block/tags page))]
+                          (when (seq page-names)
+                            {:data-page-tags (text-util/build-data-value page-names)}))
                         {})
 
                       {:key path-page-name
@@ -566,7 +572,7 @@
                  (scheduled/scheduled-and-deadlines page-name))
 
                (when-not block?
-                 (tagged-pages repo page-name page-original-name))
+                 (tagged-pages repo page page-original-name))
 
                ;; referenced blocks
                (when-not block-or-whiteboard?

src/main/frontend/components/property.cljs

@@ -387,10 +387,13 @@
 (defn- add-property-from-dropdown
   "Adds an existing or new property from dropdown. Used from a block or page context.
    For pages, used to add both schema properties or properties for a page"
-  [entity property-name {:keys [class-schema? page-configure?]}]
-  (let [repo (state/get-current-repo)]
+  [entity property-id {:keys [class-schema? page-configure?]}]
+  (p/let [repo (state/get-current-repo)
+          result (when (uuid? property-id)
+                   (db-async/<get-block repo property-id {:children? false}))
+          property (some-> (:db/id (:block result)) db/entity)]
     ;; existing property selected or entered
-    (if-let [property (db/entity [:block/original-name property-name])]
+    (if property
       (cond
         (and (not (get-in property [:block/schema :public?]))
              (ldb/built-in? property))
@@ -406,16 +409,16 @@
         :else
         ;; Both conditions necessary so that a class can add its own page properties
         (when (and (contains? (:block/type entity) "class") class-schema?)
-          (pv/<add-property! entity property-name "" {:class-schema? class-schema?
+          (pv/<add-property! entity (:db/ident property) "" {:class-schema? class-schema?
                                                      ;; Only enter property names from sub-modal as inputting
                                                      ;; property values is buggy in sub-modal
-                                                      :exit-edit? page-configure?})))
+                                                             :exit-edit? page-configure?})))
       ;; new property entered
-      (if (db-property/valid-property-name? property-name)
+      (if (db-property/valid-property-name? property-id)
         (if (and (contains? (:block/type entity) "class") page-configure?)
-          (pv/<add-property! entity property-name "" {:class-schema? class-schema? :exit-edit? page-configure?})
+          (pv/<add-property! entity property-id "" {:class-schema? class-schema? :exit-edit? page-configure?})
           (p/do!
-           (db-property-handler/upsert-property! repo nil {} {:property-name property-name})
+           (db-property-handler/upsert-property! repo nil {} {:property-name property-id})
            true))
         (do (notification/show! "This is an invalid property name. A property name cannot start with page reference characters '#' or '[['." :error)
             (pv/exit-edit-property))))))
@@ -427,7 +430,7 @@
     (rum/use-effect!
      (fn []
        (p/let [properties (db-async/<db-based-get-all-properties (state/get-current-repo))]
-         (set-properties! (map :block/original-name (remove exclude-properties properties)))
+         (set-properties! (remove exclude-properties properties))
          (set-excluded-properties! (->> properties
                                         (filter exclude-properties)
                                         ;; lower case b/c of case insensitive name lookups
@@ -435,17 +438,20 @@
                                         set))))
      [])
     [:div.ls-property-add.flex.flex-row.items-center
-    [:span.bullet-container.cursor [:span.bullet]]
-    [:div.ls-property-key {:style {:padding-left 9
-                                   :height "1.5em"}} ; TODO: ugly
-     (select/select {:items (map (fn [x] {:value x}) properties)
-                     :dropdown? true
-                     :close-modal? false
-                     :show-new-when-not-exact-match? true
-                     :exact-match-exclude-items (fn [s] (contains? excluded-properties (string/lower-case s)))
-                     :input-default-placeholder "Add property"
-                     :on-chosen on-chosen
-                     :input-opts input-opts})]]))
+     [:span.bullet-container.cursor [:span.bullet]]
+     [:div.ls-property-key {:style {:padding-left 9
+                                    :height "1.5em"}} ; TODO: ugly
+      (select/select {:items (map (fn [x]
+                                    {:label (:block/original-name x)
+                                     :value (:block/uuid x)}) properties)
+                      :extract-fn :label
+                      :dropdown? true
+                      :close-modal? false
+                      :show-new-when-not-exact-match? true
+                      :exact-match-exclude-items (fn [s] (contains? excluded-properties (string/lower-case s)))
+                      :input-default-placeholder "Add property"
+                      :on-chosen on-chosen
+                      :input-opts input-opts})]]))
 
 (rum/defcs property-input < rum/reactive
   (rum/local false ::show-new-property-config?)
@@ -489,8 +495,8 @@
                                         :*show-new-property-config? *show-new-property-config?})
                  (pv/property-value entity property @*property-value (assoc opts :editing? true))))])])
 
-       (let [on-chosen (fn [{:keys [value]}]
-                         (reset! *property-key value)
+       (let [on-chosen (fn [{:keys [value label]}]
+                         (reset! *property-key (if (uuid? value) label value))
                          (p/let [result (add-property-from-dropdown entity value opts)]
                            (when (and (true? result) *show-new-property-config?)
                              (reset! *show-new-property-config? true))))
@@ -719,12 +725,21 @@
           (property-cp block k v opts))))))
 
 ;; TODO: Remove :page-configure? as it only ever seems to be set to true
-(rum/defcs ^:large-vars/cleanup-todo properties-area < rum/reactive
+(rum/defcs ^:large-vars/cleanup-todo properties-area < rum/reactive db-mixins/query
   {:init (fn [state]
-           (assoc state ::id (str (random-uuid))))}
+           (assoc state
+                  ::id (str (random-uuid))
+                  ::tags-requested? (atom false)))}
   [state target-block edit-input-id {:keys [in-block-container? page-configure? class-schema?] :as opts}]
   (let [id (::id state)
+        *tags-requested? (::tags-requested? state)
+        repo (state/get-current-repo)
         block (resolve-linked-block-if-exists target-block)
+        _ (when (and (not @*tags-requested?) (:block/tags block))
+            (doseq [tag (:block/tags block)]
+              (db-async/<get-block repo (:block/uuid tag) {:children? false})
+              (reset! *tags-requested? true)
+              (db/sub-block (:db/id tag))))
         page? (db/page? block)
         block-properties (:block/properties block)
         properties (if (and class-schema? page-configure?)

src/main/frontend/components/query/builder.cljs

@@ -150,6 +150,21 @@
                   (reset! *property nil)
                   (append-tree! *tree opts loc x)))))))
 
+(rum/defc tags
+  [repo *tree opts loc]
+  (let [[values set-values!] (rum/use-state nil)]
+    (rum/use-effect!
+     (fn []
+       (p/let [result (db-async/<get-tags repo)]
+         (set-values! result)))
+     [])
+    (let [items (->> values
+                     (map :block/original-name)
+                     sort)]
+      (select items
+              (fn [{:keys [value]}]
+                (append-tree! *tree opts loc [:page-tags value]))))))
+
 (defn- query-filter-picker
   [state *find *tree loc clause opts]
   (let [*mode (::mode state)
@@ -164,12 +179,7 @@
                    (append-tree! *tree opts loc [:namespace value]))))
 
        "tags"
-       (let [items (->> (db-model/get-all-tagged-pages repo)
-                        (map second)
-                        sort)]
-         (select items
-                 (fn [{:keys [value]}]
-                   (append-tree! *tree opts loc [:page-tags value]))))
+       (tags repo *tree opts loc)
 
        "property"
        (property-select *mode *property)

src/main/frontend/db.cljs

@@ -27,17 +27,17 @@
 
  [frontend.db.model
   delete-blocks get-pre-block
-  delete-files delete-pages-by-files get-all-tagged-pages
+  delete-files delete-pages-by-files
   get-block-and-children get-block-by-uuid get-block-children sort-by-left
   get-block-parent get-block-parents parents-collapsed? get-block-referenced-blocks
   get-block-immediate-children get-block-page
   get-custom-css
   get-file-last-modified-at get-file get-file-page get-file-page-id file-exists?
-  get-files-blocks get-files-full get-journals-length get-pages-with-file
+  get-files-blocks get-files-full get-journals-length
   get-latest-journals get-page get-page-alias get-page-alias-names
   get-page-blocks-count get-page-blocks-no-cache get-page-file get-page-format
   get-page-referenced-blocks get-page-referenced-blocks-full get-page-referenced-pages
-  get-all-pages get-pages-relation get-pages-that-mentioned-page get-tag-pages
+  get-all-pages get-pages-relation get-pages-that-mentioned-page
   journal-page? page? page-alias-set sub-block
   set-file-last-modified-at! page-empty? page-exists? get-alias-source-page
   set-file-content! has-children? whiteboard-page?

src/main/frontend/db/async.cljs

@@ -25,16 +25,17 @@
 
 (defn <get-files
   [graph]
-  (p/let [result (<q
-                  graph
-                  '[:find [(pull ?file [:file/path :file/last-modified-at]) ...]
-                    :where
-                    [?file :file/path ?path]])]
+  (p/let [result (<q graph
+                     {:transact-db? false}
+                     '[:find [(pull ?file [:file/path :file/last-modified-at]) ...]
+                       :where
+                       [?file :file/path ?path]])]
     (->> result seq reverse (map #(vector (:file/path %) (or (:file/last-modified-at %) 0))))))
 
 (defn <get-all-templates
   [graph]
   (p/let [result (<q graph
+                     {:transact-db? true}
                      '[:find ?t (pull ?b [*])
                        :where
                        [?b :block/properties ?p]
@@ -51,7 +52,8 @@
   "Return seq of all property names except for private built-in properties."
   [graph]
   (p/let [result (<q graph
-                     '[:find [(pull ?e [:block/original-name :block/schema :db/ident]) ...]
+                     {:transact-db? false}
+                     '[:find [(pull ?e [:block/uuid :db/ident :block/original-name :block/schema]) ...]
                        :where
                        [?e :block/type "property"]
                        [?e :block/original-name]])]
@@ -77,7 +79,7 @@
 
 (defn <get-block-property-values
   [graph property-id]
-  (<q graph
+  (<q graph {:transact-db? false}
       '[:find ?b ?v
         :in $ ?property-id
         :where
@@ -97,15 +99,17 @@
             (db/entity [:block/uuid (uuid name')])
             :else
             (db/get-page name'))
-        uuid-str (or (and (:block/uuid e) (str (:block/uuid e))) name')]
+        id (or (and (:block/uuid e) (str (:block/uuid e)))
+               (and (util/uuid-string? name') name')
+               name-or-uuid)]
     (if (:block.temp/fully-loaded? e)
       e
       (when-let [^Object sqlite @db-browser/*worker]
         (state/update-state! :db/async-queries (fn [s] (conj s name')))
-        (p/let [result (.get-block-and-children sqlite graph uuid-str children?)
-                {:keys [property-pairs block children] :as result'} (ldb/read-transit-str result)
+        (p/let [result (.get-block-and-children sqlite graph id children?)
+                {:keys [properties block children] :as result'} (ldb/read-transit-str result)
                 conn (db/get-db graph false)
-                block-and-children (concat property-pairs [block] children)
+                block-and-children (concat properties [block] children)
                 _ (d/transact! conn block-and-children)]
           (state/update-state! :db/async-queries (fn [s] (disj s name')))
           (react/refresh-affected-queries!
@@ -159,7 +163,7 @@
 (defn <get-all-referenced-blocks-uuid
   "Get all uuids of blocks with any back link exists."
   [graph]
-  (<q graph
+  (<q graph {:transact-db? false}
       '[:find [?refed-uuid ...]
         :where
            ;; ?referee-b is block with ref towards ?refed-b
@@ -183,7 +187,7 @@
                               (parse-long))]
       (when future-day
         (when-let [repo (state/get-current-repo)]
-          (p/let [result (<q repo
+          (p/let [result (<q repo {}
                              '[:find [(pull ?block ?block-attrs) ...]
                                :in $ ?day ?future ?block-attrs
                                :where
@@ -206,7 +210,28 @@
                  db-model/sort-by-left-recursive
                  db-utils/group-by-page)))))))
 
+(defn <get-tag-pages
+  [graph tag-id]
+  (<q graph {:transact-db? true}
+      '[:find [(pull ?page [:db/id :block/uuid :block/name :block/original-name :block/created-at :block/updated-at])]
+        :in $ ?tag-id
+        :where
+        [?page :block/tags ?tag-id]]
+      tag-id))
+
+(defn <get-tags
+  [graph]
+  (<q graph {:transact-db? false}
+      '[:find [(pull ?tag-id [:db/id :block/original-name])]
+        :in $ ?tag-id
+        :where
+        [?page :block/tags ?tag-id]]))
+
 (defn <fetch-all-pages
   [graph]
   (when-let [^Object worker @db-browser/*worker]
-    (.fetch-all-pages worker graph)))
+    (let [db (db/get-db graph)
+          exclude-ids (->> (d/datoms db :avet :block/name)
+                           (map :db/id)
+                           (ldb/write-transit-str))]
+      (.fetch-all-pages worker graph exclude-ids))))

src/main/frontend/db/async/util.cljs

@@ -7,13 +7,14 @@
             [logseq.db :as ldb]))
 
 (defn <q
-  [graph & inputs]
+  [graph {:keys [transact-db?]
+          :or {transact-db? true}} & inputs]
   (assert (not-any? fn? inputs) "Async query inputs can't include fns because fn can't be serialized")
   (when-let [^Object sqlite @state/*db-worker]
     (p/let [result (.q sqlite graph (ldb/write-transit-str inputs))]
       (when result
         (let [result' (ldb/read-transit-str result)]
-          (when (and (seq result') (coll? result'))
+          (when (and transact-db? (seq result') (coll? result'))
             (when-let [conn (db-conn/get-db graph false)]
               (let [tx-data (->>
                              (if (and (coll? (first result'))

src/main/frontend/db/file_based/async.cljs

@@ -9,7 +9,7 @@
 
 (defn <file-based-get-all-properties
   [graph]
-  (p/let [properties (<q graph
+  (p/let [properties (<q graph {:transact-db? false}
                          '[:find [?p ...]
                            :where
                            [_ :block/properties ?p]])
@@ -37,7 +37,7 @@
 
 (defn <get-file-based-property-values
   [graph property]
-  (p/let [result (<q graph
+  (p/let [result (<q graph {:transact-db? false}
                      '[:find ?property-val ?text-property-val
                        :in $ ?property
                        :where

src/main/frontend/db/model.cljs

@@ -27,22 +27,7 @@
 
 (def block-attrs ldb/block-attrs)
 
-(defn get-tag-pages
-  [repo tag-name]
-  (when tag-name
-    (d/q '[:find ?original-name ?name
-           :in $ ?tag
-           :where
-           [?e :block/name ?tag]
-           [?page :block/tags ?e]
-           [?page :block/original-name ?original-name]
-           [?page :block/name ?name]]
-         (conn/get-db repo)
-         (util/page-name-sanity-lc tag-name))))
-
-(defn get-tag-blocks
-  [repo tag-name]
-  (ldb/get-tag-blocks (conn/get-db repo) tag-name))
+(def hidden-page? ldb/hidden-page?)
 
 (defn get-all-tagged-pages
   [repo]
@@ -53,8 +38,6 @@
          [?page :block/name ?page-name]]
        (conn/get-db repo)))
 
-(def hidden-page? ldb/hidden-page?)
-
 (defn get-all-pages
   [repo]
   (->>
@@ -74,16 +57,6 @@
      (remove hidden-page?)
      (remove nil?))))
 
-(defn get-pages-with-file
-  "Return full file entity for calling file renaming"
-  [repo]
-  (d/q
-   '[:find (pull ?page [:block/name :block/properties :block/journal?]) (pull ?file [*])
-     :where
-     [?page :block/name ?page-name]
-     [?page :block/file ?file]]
-   (conn/get-db repo)))
-
 (defn get-page-alias
   [repo page-name]
   (when-let [db (and repo (conn/get-db repo))]
@@ -253,15 +226,6 @@ independent of format as format specific heading characters are stripped"
    (set)
    (set/union #{page-id})))
 
-(defn get-page-names-by-ids
-  ([ids]
-   (get-page-names-by-ids (state/get-current-repo) ids))
-  ([repo ids]
-   (let [ids (remove nil? ids)]
-     (when repo
-       (->> (db-utils/pull-many repo '[:block/name] ids)
-            (map :block/name))))))
-
 (defn get-page-alias-names
   [repo page-id]
   (let [page (db-utils/entity page-id)

src/main/frontend/db/react.cljs

@@ -96,7 +96,7 @@
         journals? (and (vector? k) (= :frontend.worker.react/journals (last k)))
         q (if (or journals? util/node-test?)
             (fn [query inputs] (apply d/q query db inputs))
-            (fn [query inputs] (apply db-async-util/<q repo (cons query inputs))))]
+            (fn [query inputs] (apply db-async-util/<q repo {} (cons query inputs))))]
     (when (or query-fn query kv?)
       (cond
         query-fn

src/main/frontend/db_worker.cljs

@@ -406,10 +406,10 @@
      (ldb/write-transit-str (sqlite-common-db/get-initial-data @conn))))
 
   (fetch-all-pages
-   [_this repo]
+   [_this repo exclude-page-ids-str]
    (when-let [conn (worker-state/get-datascript-conn repo)]
      (async/go
-       (let [all-pages (sqlite-common-db/get-all-pages @conn)
+       (let [all-pages (sqlite-common-db/get-all-pages @conn (ldb/read-transit-str exclude-page-ids-str))
              partitioned-data (map-indexed (fn [idx p] [idx p]) (partition-all 2000 all-pages))]
          (doseq [[idx tx-data] partitioned-data]
            (worker-util/post-message :sync-db-changes {:repo repo

src/main/frontend/external/roam_export.cljs

@@ -22,16 +22,16 @@
 
 (defn <uuid->uid-map []
   (let [repo (state/get-current-repo)]
-    (p/let [result (db-async/<q repo
+    (p/let [result (db-async/<q repo {:transact-db? false}
                                 '[:find (pull ?r [:block/uuid])
                                   :in $
                                   :where
                                   [?b :block/refs ?r]])]
       (->> result
-       (map (comp :block/uuid first))
-       (distinct)
-       (map (fn [uuid] [uuid (nano-id)]))
-       (into {})))))
+           (map (comp :block/uuid first))
+           (distinct)
+           (map (fn [uuid] [uuid (nano-id)]))
+           (into {})))))
 
 (defn update-content [content uuid->uid-map]
   (when content                         ; page block doesn't have content

src/main/frontend/handler/graph.cljs

@@ -3,6 +3,7 @@
   (:require [clojure.set :as set]
             [clojure.string :as string]
             [frontend.db :as db]
+            [frontend.db.model :as db-model]
             [frontend.state :as state]
             [frontend.util :as util]
             [frontend.handler.property.util :as pu]
@@ -91,7 +92,7 @@
         current-page (or (:block/name (db/get-current-page)) "")]
     (when-let [repo (state/get-current-repo)]
       (let [relation (db/get-pages-relation repo journal?)
-            tagged-pages (map (fn [[x y]] [x (common-util/page-name-sanity-lc y)]) (db/get-all-tagged-pages repo))
+            tagged-pages (map (fn [[x y]] [x (common-util/page-name-sanity-lc y)]) (db-model/get-all-tagged-pages repo))
             namespaces (map (fn [[x y]] [x (common-util/page-name-sanity-lc y)]) (db/get-all-namespace-relation repo))
             tags (set (map second tagged-pages))
             full-pages (db/get-all-pages repo)

src/main/frontend/worker/handler/page.cljs

@@ -192,10 +192,6 @@
   [conn page]
   (try
     (cond
-      (and (contains? (:block/type page) "class")
-           (seq (ldb/get-tag-blocks @conn (:block/name page))))
-      {:msg "Page content deleted but unable to delete this page because blocks are tagged with this page"}
-
       ;; TODO: allow users to delete not built-in properties
       (contains? (:block/type page) "property")
       (cond (seq (ldb/get-classes-with-property @conn (:block/uuid page)))

src/main/logseq/api.cljs

@@ -935,7 +935,8 @@
 
                                       :else %)
                                    inputs)
-              result          (apply db-async/<q repo (cons query resolved-inputs))]
+              result          (apply db-async/<q repo {:transact-db? false}
+                                (cons query resolved-inputs))]
         (bean/->js (sdk-utils/normalize-keyword-for-json result false))))))
 
 (defn ^:export custom_query

src/test/frontend/db/db_based_model_test.cljs

@@ -5,7 +5,8 @@
             [frontend.test.helper :as test-helper]
             [datascript.core :as d]
             [frontend.handler.db-based.property :as db-property-handler]
-            [frontend.handler.page :as page-handler]))
+            [frontend.handler.page :as page-handler]
+            [logseq.db :as ldb]))
 
 (def repo test-helper/test-db-name-db-version)
 
@@ -88,7 +89,7 @@
         _ (test-helper/save-block! repo fbid "Block 1" {:tags ["class1"]})
         _ (test-helper/save-block! repo sbid "Block 2" {:tags ["class1"]})]
     (is
-     (= (model/get-tag-blocks repo "class1")
+     (= (ldb/get-tag-blocks (db/get-db) "class1")
         [(:db/id (db/entity [:block/uuid fbid]))
          (:db/id (db/entity [:block/uuid sbid]))]))))
 

src/test/frontend/handler/db_based/property_async_test.cljs

@@ -45,19 +45,18 @@
     (let [repo (state/get-current-repo)
           fb (db/entity [:block/uuid fbid])
           k :user.property/property-1]
-      (let []
-        (p/do!
-         ;; add property
-         (db-property-handler/upsert-property! repo k {:type :default} {})
-         (let [property (db/entity k)]
-           (p/do!
-            (db-property-handler/create-property-text-block! fb property "Block content" editor-handler/wrap-parse-block {})
+      (p/do!
+       ;; add property
+       (db-property-handler/upsert-property! repo k {:type :default} {})
+       (let [property (db/entity k)]
+         (p/do!
+          (db-property-handler/create-property-text-block! fb property "Block content" editor-handler/wrap-parse-block {})
             ;; collapse property-1
-            (db-property-handler/collapse-expand-property! repo fb property true)
-            (is (=
-                 [(:db/id property)]
-                 (map :db/id (:block/collapsed-properties (db/entity [:block/uuid fbid])))))
+          (db-property-handler/collapse-expand-property! repo fb property true)
+          (is (=
+               [(:db/id property)]
+               (map :db/id (:block/collapsed-properties (db/entity [:block/uuid fbid])))))
 
             ;; expand property-1
-            (db-property-handler/collapse-expand-property! repo fb property false)
-            (is (nil? (:block/collapsed-properties (db/entity [:block/uuid fbid])))))))))))
+          (db-property-handler/collapse-expand-property! repo fb property false)
+          (is (nil? (:block/collapsed-properties (db/entity [:block/uuid fbid]))))))))))

src/test/frontend/test/helper.cljs

@@ -12,7 +12,7 @@
             [frontend.handler.page :as page-handler]
             [datascript.core :as d]
             [logseq.graph-parser.text :as text]
-            [logseq.db.sqlite.create-graph :as sqlite-create-graph :refer [kv]]
+            [logseq.db.sqlite.create-graph :as sqlite-create-graph]
             [frontend.config :as config]
             [frontend.worker.pipeline :as worker-pipeline]))