Merge branch 'feat/db' into enhance/reference-view

.clj-kondo/config.edn

@@ -166,23 +166,27 @@
              logseq.common.util.namespace ns-util
              logseq.common.util.page-ref page-ref
              logseq.db ldb
+             logseq.db.common.entity-util common-entity-util
+             logseq.db.common.order db-order
+             logseq.db.common.property-util db-property-util
+             logseq.db.common.sqlite sqlite-common-db
+             logseq.db.file-based.rules file-rules
+             logseq.db.file-based.schema file-schema
+             logseq.db.file-based.entity-util file-entity-util
              logseq.db.frontend.class db-class
              logseq.db.frontend.content db-content
              logseq.db.frontend.db-ident db-ident
              logseq.db.frontend.entity-plus entity-plus
              logseq.db.frontend.entity-util entity-util
              logseq.db.frontend.inputs db-inputs
-             logseq.db.frontend.order db-order
              logseq.db.frontend.property db-property
              logseq.db.frontend.property.build db-property-build
              logseq.db.frontend.property.type db-property-type
-             logseq.db.frontend.property.util db-property-util
              logseq.db.frontend.rules rules
              logseq.db.frontend.schema db-schema
              logseq.db.frontend.validate db-validate
              logseq.db.sqlite.build sqlite-build
              logseq.db.sqlite.cli sqlite-cli
-             logseq.db.sqlite.common-db sqlite-common-db
              logseq.db.sqlite.create-graph sqlite-create-graph
              logseq.db.sqlite.export sqlite-export
              logseq.db.sqlite.util sqlite-util

deps/db/.carve/config.edn

@@ -1,12 +1,12 @@
 {:paths ["src"]
- :api-namespaces [logseq.db.sqlite.common-db
+ :api-namespaces [logseq.db.common.sqlite
                   logseq.db.sqlite.util
                   logseq.db.sqlite.cli
                   logseq.db.frontend.property
                   logseq.db.frontend.property.build
-                  logseq.db.frontend.property.util
+                  logseq.db.common.property-util
                   logseq.db.frontend.content
-                  logseq.db.frontend.order
+                  logseq.db.common.order
                   logseq.db.sqlite.create-graph
                   logseq.db.frontend.malli-schema
                   ;; Some fns are used by frontend but not worth moving over yet

deps/db/.carve/ignore

@@ -1,5 +1,5 @@
 ;; API
-logseq.db.frontend.rules/query-dsl-rules
+logseq.db.file-based.schema/retract-attributes
 ;; API
 logseq.db.frontend.rules/db-query-dsl-rules
 ;; API

deps/db/.clj-kondo/config.edn

@@ -9,22 +9,26 @@
   :consistent-alias
   {:aliases {clojure.string string
              logseq.db ldb
+             logseq.db.common.entity-util common-entity-util
+             logseq.db.common.order db-order
+             logseq.db.common.property-util db-property-util
+             logseq.db.common.sqlite sqlite-common-db
              logseq.db.frontend.content db-content
              logseq.db.frontend.class db-class
              logseq.db.frontend.db-ident db-ident
              logseq.db.frontend.inputs db-inputs
-             logseq.db.frontend.order db-order
              logseq.db.frontend.property db-property
              logseq.db.frontend.property.build db-property-build
              logseq.db.frontend.property.type db-property-type
-             logseq.db.frontend.property.util db-property-util
+             logseq.db.file-based.rules file-rules
+             logseq.db.file-based.schema file-schema
+             logseq.db.file-based.entity-util file-entity-util
              logseq.db.frontend.entity-plus entity-plus
              logseq.db.frontend.rules rules
              logseq.db.frontend.schema db-schema
              logseq.db.frontend.validate db-validate
              logseq.db.sqlite.build sqlite-build
              logseq.db.sqlite.cli sqlite-cli
-             logseq.db.sqlite.common-db sqlite-common-db
              logseq.db.sqlite.create-graph sqlite-create-graph
              logseq.db.sqlite.export sqlite-export
              logseq.db.sqlite.util sqlite-util}}}

deps/db/README.md

@@ -4,16 +4,19 @@ This library provides an API to the
 frontend([datascript](https://github.com/tonsky/datascript)) and
 backend([SQLite](https://www.sqlite.org/index.html)) databases from the Logseq
 app and the CLI. The majority of this library is focused on supporting DB graphs
-but there are a few older namespaces that support file graphs. This library is
+but there are a few namespaces that support file graphs. This library is
 compatible with ClojureScript and with
 [nbb-logseq](https://github.com/logseq/nbb-logseq) to respectively provide
 frontend and commandline functionality.
 
 ## API
 
-This library is under the parent namespace `logseq.db`. While `logseq.db` is the
-main entry point, this library also provides frontend namespaces under
-`logseq.db.frontend` and backend/sqlite namespaces under `logseq.db.sqlite`.
+This library is under the parent namespace `logseq.db`. It provides the following namespaces:
+* `logseq.db` - main entry point serving both file and DB graphs
+* `logseq.db.frontend.*` - frontend namespaces for DB graphs
+* `logseq.db.sqlite.*` - backend/sqlite namespaces for DB graphs
+* `logseq.db.file-based.*` - namespaces for file graphs, mostly old namespaces
+* `logseq.db.common.*` - namespaces for both file and DB graphs
 
 ## Usage
 

deps/db/bb.edn

@@ -24,13 +24,14 @@
 
   lint:rules
   {:requires ([logseq.bb-tasks.lint.datalog :as datalog]
+              [logseq.db.file-based.rules :as file-rules]
               [logseq.db.frontend.rules :as rules])
    :doc "Lint datalog rules for parsability and unbound variables"
    :task (datalog/lint-rules
           (set
-           (concat (mapcat val rules/rules)
+           (concat (mapcat val (merge file-rules/rules rules/rules))
                    ;; TODO: Update linter to handle false positive on ?str-val for :property
-                   (rules/extract-rules (dissoc rules/query-dsl-rules :property))
+                   (rules/extract-rules (dissoc file-rules/query-dsl-rules :property))
                    ;; TODO: Update linter to handle false positive on :task, :priority, :*property* rules
                    (rules/extract-rules (dissoc rules/db-query-dsl-rules
                                                 :task :priority

deps/db/src/logseq/db.cljs

@@ -10,13 +10,15 @@
             [logseq.common.util.namespace :as ns-util]
             [logseq.common.util.page-ref :as page-ref]
             [logseq.common.uuid :as common-uuid]
+            [logseq.db.common.delete-blocks :as delete-blocks] ;; Load entity extensions
+            [logseq.db.common.entity-util :as common-entity-util]
+            [logseq.db.common.sqlite :as sqlite-common-db]
+            [logseq.db.file-based.rules :as file-rules]
             [logseq.db.frontend.class :as db-class]
-            [logseq.db.frontend.delete-blocks :as delete-blocks] ;; Load entity extensions
             [logseq.db.frontend.entity-plus :as entity-plus]
             [logseq.db.frontend.entity-util :as entity-util]
             [logseq.db.frontend.property :as db-property]
             [logseq.db.frontend.rules :as rules]
-            [logseq.db.sqlite.common-db :as sqlite-common-db]
             [logseq.db.sqlite.util :as sqlite-util])
   (:refer-clojure :exclude [object?]))
 
@@ -88,13 +90,13 @@
              (prn :debug-tx-data tx-data)
              (throw e))))))))
 
-(def page? entity-util/page?)
+(def page? common-entity-util/page?)
 (def internal-page? entity-util/internal-page?)
 (def class? entity-util/class?)
 (def property? entity-util/property?)
 (def closed-value? entity-util/closed-value?)
-(def whiteboard? entity-util/whiteboard?)
-(def journal? entity-util/journal?)
+(def whiteboard? common-entity-util/whiteboard?)
+(def journal? common-entity-util/journal?)
 (def hidden? entity-util/hidden?)
 (def object? entity-util/object?)
 (def asset? entity-util/asset?)
@@ -547,7 +549,7 @@
       ['?p :block/name '?namespace]
       (list 'namespace '?p '?c)]
      db
-     (:namespace rules/rules)
+     (:namespace file-rules/rules)
      namespace'')))
 
 (defn get-pages-by-name-partition

deps/db/src/logseq/db/frontend/delete_blocks.cljs → deps/db/src/logseq/db/common/delete_blocks.cljs

@@ -1,10 +1,11 @@
-(ns logseq.db.frontend.delete-blocks
-  "Delete refs/macros when deleting blocks"
+(ns logseq.db.common.delete-blocks
+  "For file and DB graphs, provides fn to handle any deletion to occur per ldb/transact!"
   (:require [clojure.string :as string]
             [datascript.core :as d]
             [logseq.common.util :as common-util]
             [logseq.common.util.block-ref :as block-ref]
             [logseq.common.util.page-ref :as page-ref]
+            [logseq.db.common.entity-util :as common-entity-util]
             [logseq.db.frontend.entity-plus :as entity-plus]
             [logseq.db.frontend.entity-util :as entity-util]))
 
@@ -54,7 +55,7 @@
                                                     (contains? #{:db.fn/retractEntity :db/retractEntity} (first tx)))
                                            (second tx))) txs)
                                  (filter (fn [id]
-                                           (not (entity-util/page? (d/entity db id))))))]
+                                           (not (common-entity-util/page? (d/entity db id))))))]
     (when (seq retracted-block-ids)
       (let [retracted-blocks (map #(d/entity db %) retracted-block-ids)
             retracted-tx (build-retracted-tx retracted-blocks)

deps/db/src/logseq/db/common/entity_util.cljs

@@ -0,0 +1,19 @@
+(ns logseq.db.common.entity-util
+  "Lower level entity util fns for DB and file graphs"
+  (:require [logseq.db.file-based.entity-util :as file-entity-util]
+            [logseq.db.frontend.entity-util :as entity-util]))
+
+(defn whiteboard?
+  [entity]
+  (or (entity-util/whiteboard? entity)
+      (file-entity-util/whiteboard? entity)))
+
+(defn journal?
+  [entity]
+  (or (entity-util/journal? entity)
+      (file-entity-util/journal? entity)))
+
+(defn page?
+  [entity]
+  (or (entity-util/page? entity)
+      (file-entity-util/page? entity)))

deps/db/src/logseq/db/frontend/order.cljs → deps/db/src/logseq/db/common/order.cljs

@@ -1,5 +1,6 @@
-(ns logseq.db.frontend.order
-  "Use fractional-indexing order for blocks/properties/closed values/etc."
+(ns logseq.db.common.order
+  "Use fractional-indexing order for blocks/properties/closed values/etc.
+   Used by DB and file graphs"
   (:require [logseq.clj-fractional-indexing :as index]
             [datascript.core :as d]))
 

deps/db/src/logseq/db/frontend/property/util.cljs → deps/db/src/logseq/db/common/property_util.cljs

@@ -1,4 +1,4 @@
-(ns logseq.db.frontend.property.util
+(ns logseq.db.common.property-util
   "Property related util fns. Fns used in both DB and file graphs should go here"
   (:require [datascript.core :as d]
             [logseq.db.frontend.property :as db-property]

deps/db/src/logseq/db/sqlite/common_db.cljs → deps/db/src/logseq/db/common/sqlite.cljs

@@ -1,5 +1,6 @@
-(ns logseq.db.sqlite.common-db
-  "Common sqlite db fns for browser and node"
+(ns logseq.db.common.sqlite
+  "Provides common sqlite db fns for file and DB graphs. These fns work on
+  browser and node"
   (:require ["path" :as node-path]
             [clojure.set :as set]
             [clojure.string :as string]
@@ -9,7 +10,8 @@
             [logseq.common.util.date-time :as date-time-util]
             [logseq.db.frontend.entity-plus :as entity-plus]
             [logseq.db.frontend.entity-util :as entity-util]
-            [logseq.db.frontend.order :as db-order]
+            [logseq.db.common.entity-util :as common-entity-util]
+            [logseq.db.common.order :as db-order]
             [logseq.db.sqlite.util :as sqlite-util]))
 
 (defn- get-pages-by-name
@@ -29,7 +31,7 @@
   (->> (d/datoms db :avet :block/title page-name)
        (filter (fn [d]
                  (let [e (d/entity db (:e d))]
-                   (entity-util/page? e))))
+                   (common-entity-util/page? e))))
        (map :e)
        sort
        first))
@@ -142,9 +144,9 @@
   (let [block (d/entity db (if (uuid? id)
                              [:block/uuid id]
                              id))
-        page? (entity-util/page? block)
+        page? (common-entity-util/page? block)
         get-children (fn [block children]
-                       (let [long-page? (and (> (count children) 500) (not (entity-util/whiteboard? block)))]
+                       (let [long-page? (and (> (count children) 500) (not (common-entity-util/whiteboard? block)))]
                          (if long-page?
                            (->> (map (fn [e]
                                        (select-keys e [:db/id :block/uuid :block/page :block/order :block/parent :block/collapsed? :block/link]))

deps/db/src/logseq/db/file_based/entity_util.cljs

@@ -0,0 +1,14 @@
+(ns logseq.db.file-based.entity-util
+  "Lower level entity util fns for file graphs")
+
+(defn whiteboard?
+  [entity]
+  (identical? "whiteboard" (:block/type entity)))
+
+(defn journal?
+  [entity]
+  (identical? "journal" (:block/type entity)))
+
+(defn page?
+  [entity]
+  (contains? #{"page" "journal" "whiteboard"} (:block/type entity)))

deps/db/src/logseq/db/file_based/rules.cljc

@@ -0,0 +1,93 @@
+(ns ^:bb-compatible logseq.db.file-based.rules
+  "Datalog rules for file graphs")
+
+(def rules
+  "File graph rules used in db.model queries"
+  {:namespace
+   '[[(namespace ?p ?c)
+      [?c :block/namespace ?p]]
+     [(namespace ?p ?c)
+      [?t :block/namespace ?p]
+      (namespace ?t ?c)]]})
+
+(def ^:large-vars/data-var query-dsl-rules
+  "Rules used by frontend.db.query-dsl for file graphs. The symbols ?b and ?p
+  respectively refer to block and page. Do not alter them as they are
+  programmatically built by the query-dsl ns"
+  {:page-property
+   '[(page-property ?p ?key ?val)
+     [?p :block/name]
+     [?p :block/properties ?prop]
+     [(get ?prop ?key) ?v]
+     (or [(= ?v ?val)] [(contains? ?v ?val)])]
+
+   :has-page-property
+   '[(has-page-property ?p ?key)
+     [?p :block/name]
+     [?p :block/properties ?prop]
+     [(get ?prop ?key)]]
+
+   :task
+   '[(task ?b ?markers)
+     [?b :block/marker ?marker]
+     [(contains? ?markers ?marker)]]
+
+   :priority
+   '[(priority ?b ?priorities)
+     [?b :block/priority ?priority]
+     [(contains? ?priorities ?priority)]]
+
+   :page-tags
+   '[(page-tags ?p ?tags)
+     [?p :block/tags ?t]
+     [?t :block/name ?tag]
+     [(contains? ?tags ?tag)]]
+
+   :all-page-tags
+   '[(all-page-tags ?p)
+     [_ :block/tags ?p]]
+
+   :between
+   '[(between ?b ?start ?end)
+     [?b :block/page ?p]
+     [?p :block/type "journal"]
+     [?p :block/journal-day ?d]
+     [(>= ?d ?start)]
+     [(<= ?d ?end)]]
+
+   :has-property
+   '[(has-property ?b ?prop)
+     [?b :block/properties ?bp]
+     [(missing? $ ?b :block/name)]
+     [(get ?bp ?prop)]]
+
+   :block-content
+   '[(block-content ?b ?query)
+     [?b :block/title ?content]
+     [(clojure.string/includes? ?content ?query)]]
+
+   :page
+   '[(page ?b ?page-name)
+     [?b :block/page ?bp]
+     [?bp :block/name ?page-name]]
+
+   :namespace
+   '[(namespace ?p ?namespace)
+     [?p :block/namespace ?parent]
+     [?parent :block/name ?namespace]]
+
+   :property
+   '[(property ?b ?key ?val)
+     [?b :block/properties ?prop]
+     [(missing? $ ?b :block/name)]
+     [(get ?prop ?key) ?v]
+     [(str ?val) ?str-val]
+     (or [(= ?v ?val)]
+         [(contains? ?v ?val)]
+         ;; For integer pages that aren't strings
+         [(contains? ?v ?str-val)])]
+
+   :page-ref
+   '[(page-ref ?b ?page-name)
+     [?b :block/path-refs ?br]
+     [?br :block/name ?page-name]]})

deps/db/src/logseq/db/file_based/schema.cljs

@@ -0,0 +1,125 @@
+(ns logseq.db.file-based.schema
+  "Schema related vars for file graphs")
+
+;; A page is a special block, a page can corresponds to multiple files with the same ":block/name".
+(def ^:large-vars/data-var schema
+  "Schema for file graphs"
+  {:db/ident        {:db/unique :db.unique/identity}
+   :kv/value       {}
+   :recent/pages {}
+
+   ;; :block/type is a string type of the current block
+   ;; "whiteboard" for whiteboards
+   ;; "property" for property blocks
+   ;; "class" for structured page
+   :block/type {:db/index true}
+   :block/uuid {:db/unique :db.unique/identity}
+   :block/parent {:db/valueType :db.type/ref
+                  :db/index true}
+   :block/order {:db/index true}
+   :block/collapsed? {}
+
+   ;; :markdown, :org
+   :block/format {}
+
+   ;; belongs to which page
+   :block/page {:db/valueType :db.type/ref
+                :db/index true}
+   ;; reference blocks
+   :block/refs {:db/valueType :db.type/ref
+                :db/cardinality :db.cardinality/many}
+   ;; referenced pages inherited from the parents
+   :block/path-refs {:db/valueType   :db.type/ref
+                     :db/cardinality :db.cardinality/many}
+
+   ;; tags are structured classes
+   :block/tags {:db/valueType :db.type/ref
+                :db/cardinality :db.cardinality/many}
+
+   ;; which block this block links to, used for tag, embeds
+   :block/link {:db/valueType :db.type/ref
+                :db/index true}
+
+   ;; page's namespace
+   :block/namespace {:db/valueType :db.type/ref}
+
+   ;; for pages
+   :block/alias {:db/valueType :db.type/ref
+                 :db/cardinality :db.cardinality/many}
+
+   ;; todo keywords, e.g. "TODO", "DOING", "DONE"
+   :block/marker {}
+
+   ;; "A", "B", "C"
+   :block/priority {}
+
+   ;; map, key -> set of refs in property value or full text if none are found
+   :block/properties {}
+   ;; vector
+   :block/properties-order {}
+   ;; map, key -> original property value's content
+   :block/properties-text-values {}
+
+   ;; first block that's not a heading or unordered list
+   :block/pre-block? {}
+
+   ;; scheduled day
+   :block/scheduled {}
+
+   ;; deadline day
+   :block/deadline {}
+
+   ;; whether blocks is a repeated block (usually a task)
+   :block/repeated? {}
+
+   :block/created-at {:db/index true}
+   :block/updated-at {:db/index true}
+
+   ;; page additional attributes
+   ;; page's name, lowercase
+   :block/name {:db/unique :db.unique/identity}
+
+   ;; page's original name
+   :block/title {:db/index true}
+
+   ;; page's journal day
+   :block/journal-day {:db/index true}
+
+   ;; macros in block
+   :block/macros {:db/valueType :db.type/ref
+                  :db/cardinality :db.cardinality/many}
+
+   ;; block's file
+   :block/file {:db/valueType :db.type/ref}
+
+   ;; latest tx that affected the block
+   :block/tx-id {}
+
+   ;; file
+   :file/path {:db/unique :db.unique/identity}
+   :file/content {}
+   :file/created-at {}
+   :file/last-modified-at {}
+   :file/size {}})
+
+(def file-only-attributes
+  [:block/namespace :block/properties-text-values :block/pre-block? :recent/pages :block/file
+   :block/properties :block/properties-order :block/repeated? :block/deadline :block/scheduled :block/priority
+   :block/marker :block/macros :block/type :block/format])
+
+(def retract-attributes
+  #{:block/refs
+    :block/tags
+    :block/alias
+    :block/marker
+    :block/priority
+    :block/scheduled
+    :block/deadline
+    :block/repeated?
+    :block/pre-block?
+    :block/properties
+    :block/properties-order
+    :block/properties-text-values
+    :block/macros
+    :block/invalid-properties
+    :block/warning})

deps/db/src/logseq/db/frontend/class.cljs

@@ -102,7 +102,7 @@
 
 (def private-tags
   "Built-in classes that are private and should not be used by a user directly.
-  These used to be in :block/type"
+  These used to be in block/type"
   (set/union internal-tags
              #{:logseq.class/Journal :logseq.class/Whiteboard}))
 

deps/db/src/logseq/db/frontend/content.cljs

@@ -1,11 +1,10 @@
 (ns logseq.db.frontend.content
   "Fns to handle block content e.g. internal ids"
   (:require [clojure.string :as string]
-            [logseq.common.util.page-ref :as page-ref]
             [datascript.core :as d]
             [logseq.common.util :as common-util]
-            [logseq.db.frontend.entity-util :as entity-util]
-            [logseq.db.frontend.entity-plus :as entity-plus]))
+            [logseq.common.util.page-ref :as page-ref]
+            [logseq.db.frontend.entity-util :as entity-util]))
 
 ;; [[uuid]]
 (def id-ref-pattern
@@ -122,11 +121,9 @@
 (defn update-block-content
   "Replace `[[internal-id]]` with `[[page name]]`"
   [db item eid]
-  (if (entity-plus/db-based-graph? db)
-    (if-let [content (:block/title item)]
-      (let [refs (:block/refs (d/entity db eid))]
-        (assoc item :block/title (id-ref->title-ref content refs false)))
-      item)
+  (if-let [content (:block/title item)]
+    (let [refs (:block/refs (d/entity db eid))]
+      (assoc item :block/title (id-ref->title-ref content refs false)))
     item))
 
 (defn replace-tags-with-id-refs

deps/db/src/logseq/db/frontend/entity_plus.cljc

@@ -16,9 +16,9 @@
 
 (def nil-db-ident-entities
   "No such entities with these :db/ident, but `(d/entity <db> <ident>)` has been called somewhere."
-  #{:block/tx-id :block/warning :block/pre-block? :block/uuid :block/scheduled
-    :block/deadline :block/journal-day :block/level :block/heading-level
-    :block/type :block/name :block/marker :block/_refs
+  #{:block/tx-id :block/uuid :block/journal-day :block/_refs :block/level :block/heading-level :block/warning
+    ;; File graph only attributes. Can these be removed if this is only called in db graphs?
+    :block/pre-block? :block/scheduled :block/deadline :block/type :block/name :block/marker
 
     :block.temp/ast-title :block.temp/top? :block.temp/bottom? :block.temp/search?
     :block.temp/fully-loaded? :block.temp/ast-body

deps/db/src/logseq/db/frontend/entity_util.cljs

@@ -1,5 +1,5 @@
 (ns logseq.db.frontend.entity-util
-  "Lower level entity util fns used across db namespaces"
+  "Lower level entity util fns for DB graphs"
   (:require [clojure.string :as string]
             [datascript.db]
             [datascript.impl.entity :as de])
@@ -33,11 +33,7 @@
 (defn whiteboard?
   "Given a page entity or map, check if it is a whiteboard page"
   [entity]
-  (or
-   ;; db based graph
-   (has-tag? entity :logseq.class/Whiteboard)
-   ;; file based graph
-   (identical? "whiteboard" (:block/type entity))))
+  (has-tag? entity :logseq.class/Whiteboard))
 
 (defn closed-value?
   [entity]
@@ -46,24 +42,15 @@
 (defn journal?
   "Given a page entity or map, check if it is a journal page"
   [entity]
-  (or
-   ;; db based graph
-   (has-tag? entity :logseq.class/Journal)
-   ;; file based graph
-   (identical? "journal" (:block/type entity))))
+  (has-tag? entity :logseq.class/Journal))
 
 (defn page?
   [entity]
-  (or
-   ;; db based graph
-   (internal-page? entity)
-   (class? entity)
-   (property? entity)
-   (whiteboard? entity)
-   (journal? entity)
-
-   ;; file based graph
-   (contains? #{"page" "journal" "whiteboard"} (:block/type entity))))
+  (or (internal-page? entity)
+      (class? entity)
+      (property? entity)
+      (whiteboard? entity)
+      (journal? entity)))
 
 (defn asset?
   "Given an entity or map, check if it is an asset block"

deps/db/src/logseq/db/frontend/malli_schema.cljs

@@ -3,10 +3,10 @@
   (:require [clojure.set :as set]
             [clojure.string :as string]
             [datascript.core :as d]
+            [logseq.db.common.order :as db-order]
             [logseq.db.frontend.class :as db-class]
             [logseq.db.frontend.entity-plus :as entity-plus]
             [logseq.db.frontend.entity-util :as entity-util]
-            [logseq.db.frontend.order :as db-order]
             [logseq.db.frontend.property :as db-property]
             [logseq.db.frontend.property.type :as db-property-type]
             [logseq.db.frontend.schema :as db-schema]))
@@ -515,7 +515,7 @@
     :property-value-placeholder property-value-placeholder}))
 
 (def DB
-  "Malli schema for entities from schema/schema-for-db-based-graph. In order to
+  "Malli schema for entities from db-schema/schema. In order to
   thoroughly validate properties, the entities and this schema should be
   prepared with update-properties-in-ents and update-properties-in-schema
   respectively"
@@ -536,9 +536,8 @@
 (let [malli-one-ref-attrs (->> (concat property-attrs page-attrs block-attrs page-or-block-attrs (rest normal-page))
                                (filter #(= (last %) :int))
                                (map first)
-                               set)
-      attrs-to-ignore #{:block/file}]
-  (when-let [undeclared-ref-attrs (seq (remove (some-fn malli-one-ref-attrs attrs-to-ignore) db-schema/card-one-ref-type-attributes))]
+                               set)]
+  (when-let [undeclared-ref-attrs (seq (remove malli-one-ref-attrs db-schema/card-one-ref-type-attributes))]
     (throw (ex-info (str "The malli DB schema is missing the following cardinality-one ref attributes from datascript's schema: "
                          (string/join ", " undeclared-ref-attrs))
                     {}))))

deps/db/src/logseq/db/frontend/property/build.cljs

@@ -2,7 +2,7 @@
   "Builds core property concepts"
   (:require [datascript.core :as d]
             [logseq.common.util :as common-util]
-            [logseq.db.frontend.order :as db-order]
+            [logseq.db.common.order :as db-order]
             [logseq.db.frontend.property :as db-property]
             [logseq.db.frontend.property.type :as db-property-type]
             [logseq.db.sqlite.util :as sqlite-util]))

deps/db/src/logseq/db/frontend/rules.cljc

@@ -1,17 +1,12 @@
 (ns ^:bb-compatible logseq.db.frontend.rules
-  "Datalog rules for use with logseq.db.frontend.schema")
+  "Datalog rules mostly for DB graphs. `rules`
+   is the only var also used by file graphs"
+  (:require [logseq.db.file-based.rules :as file-rules]))
 
 (def ^:large-vars/data-var rules
-  "Rules used mainly in frontend.db.model"
+  "Rules used mainly in frontend.db.model for both DB and file graphs"
   ;; rule "parent" is optimized for parent node -> child node nesting queries
-  {:namespace
-   '[[(namespace ?p ?c)
-      [?c :block/namespace ?p]]
-     [(namespace ?p ?c)
-      [?t :block/namespace ?p]
-      (namespace ?t ?c)]]
-
-   :parent
+  {:parent
    '[[(parent ?p ?c)
       [?c :logseq.property/parent ?p]]
      [(parent ?p ?c)
@@ -67,96 +62,13 @@
 ;;            (not-join [?e ?v]
 ;;                      [?e ?a ?v]))]
 
-(def ^:large-vars/data-var query-dsl-rules
-  "Rules used by frontend.db.query-dsl for file graphs. The symbols ?b and ?p
-  respectively refer to block and page. Do not alter them as they are
-  programmatically built by the query-dsl ns"
-  {:page-property
-   '[(page-property ?p ?key ?val)
-     [?p :block/name]
-     [?p :block/properties ?prop]
-     [(get ?prop ?key) ?v]
-     (or [(= ?v ?val)] [(contains? ?v ?val)])]
-
-   :has-page-property
-   '[(has-page-property ?p ?key)
-     [?p :block/name]
-     [?p :block/properties ?prop]
-     [(get ?prop ?key)]]
-
-   :task
-   '[(task ?b ?markers)
-     [?b :block/marker ?marker]
-     [(contains? ?markers ?marker)]]
-
-   :priority
-   '[(priority ?b ?priorities)
-     [?b :block/priority ?priority]
-     [(contains? ?priorities ?priority)]]
-
-   :page-tags
-   '[(page-tags ?p ?tags)
-     [?p :block/tags ?t]
-     [?t :block/name ?tag]
-     [(contains? ?tags ?tag)]]
-
-   :all-page-tags
-   '[(all-page-tags ?p)
-     [_ :block/tags ?p]]
-
-   :between
-   '[(between ?b ?start ?end)
-     [?b :block/page ?p]
-     [?p :block/type "journal"]
-     [?p :block/journal-day ?d]
-     [(>= ?d ?start)]
-     [(<= ?d ?end)]]
-
-   :has-property
-   '[(has-property ?b ?prop)
-     [?b :block/properties ?bp]
-     [(missing? $ ?b :block/name)]
-     [(get ?bp ?prop)]]
-
-   :block-content
-   '[(block-content ?b ?query)
-     [?b :block/title ?content]
-     [(clojure.string/includes? ?content ?query)]]
-
-   :page
-   '[(page ?b ?page-name)
-     [?b :block/page ?bp]
-     [?bp :block/name ?page-name]]
-
-   :namespace
-   '[(namespace ?p ?namespace)
-     [?p :block/namespace ?parent]
-     [?parent :block/name ?namespace]]
-
-   :property
-   '[(property ?b ?key ?val)
-     [?b :block/properties ?prop]
-     [(missing? $ ?b :block/name)]
-     [(get ?prop ?key) ?v]
-     [(str ?val) ?str-val]
-     (or [(= ?v ?val)]
-         [(contains? ?v ?val)]
-         ;; For integer pages that aren't strings
-         [(contains? ?v ?str-val)])]
-
-   :page-ref
-   '[(page-ref ?b ?page-name)
-     [?b :block/path-refs ?br]
-     [?br :block/name ?page-name]]})
-
 (def ^:large-vars/data-var db-query-dsl-rules
-  "Rules used by frontend.query.dsl for db graphs"
+  "Rules used by frontend.query.dsl for DB graphs"
   (merge
-   (dissoc query-dsl-rules :namespace
+   (dissoc file-rules/query-dsl-rules :namespace
            :page-property :has-page-property
            :page-tags :all-page-tags)
-
-   (dissoc rules :namespace)
+   rules
 
    {:between
     '[(between ?b ?start ?end)

deps/db/src/logseq/db/frontend/schema.cljs

@@ -1,7 +1,8 @@
 (ns logseq.db.frontend.schema
-  "Main datascript schemas for the Logseq app"
+  "Schema related fns for DB and file graphs"
   (:require [clojure.set :as set]
-            [clojure.string :as string]))
+            [clojure.string :as string]
+            [logseq.db.file-based.schema :as file-schema]))
 
 (def schema-version? (every-pred map? :major))
 
@@ -59,137 +60,18 @@
       (str (:major schema-version)))
     :else (throw (ex-info "Not a schema-version" {:data schema-version}))))
 
-;; A page is a special block, a page can corresponds to multiple files with the same ":block/name".
-(def ^:large-vars/data-var schema
-  {:db/ident        {:db/unique :db.unique/identity}
-   :kv/value       {}
-   :recent/pages {}
-
-   ;; :block/type is a string type of the current block
-   ;; "whiteboard" for whiteboards
-   ;; "property" for property blocks
-   ;; "class" for structured page
-   :block/type {:db/index true}
-   :block/uuid {:db/unique :db.unique/identity}
-   :block/parent {:db/valueType :db.type/ref
-                  :db/index true}
-   :block/order {:db/index true}
-   :block/collapsed? {}
-
-   ;; :markdown, :org
-   :block/format {}
-
-   ;; belongs to which page
-   :block/page {:db/valueType :db.type/ref
-                :db/index true}
-   ;; reference blocks
-   :block/refs {:db/valueType :db.type/ref
-                :db/cardinality :db.cardinality/many}
-   ;; referenced pages inherited from the parents
-   :block/path-refs {:db/valueType   :db.type/ref
-                     :db/cardinality :db.cardinality/many}
-
-   ;; tags are structured classes
-   :block/tags {:db/valueType :db.type/ref
-                :db/cardinality :db.cardinality/many}
-
-   ;; which block this block links to, used for tag, embeds
-   :block/link {:db/valueType :db.type/ref
-                :db/index true}
-
-   ;; page's namespace
-   :block/namespace {:db/valueType :db.type/ref}
-
-   ;; for pages
-   :block/alias {:db/valueType :db.type/ref
-                 :db/cardinality :db.cardinality/many}
-
-   ;; todo keywords, e.g. "TODO", "DOING", "DONE"
-   :block/marker {}
-
-   ;; "A", "B", "C"
-   :block/priority {}
-
-   ;; map, key -> set of refs in property value or full text if none are found
-   :block/properties {}
-   ;; vector
-   :block/properties-order {}
-   ;; map, key -> original property value's content
-   :block/properties-text-values {}
-
-   ;; first block that's not a heading or unordered list
-   :block/pre-block? {}
-
-   ;; scheduled day
-   :block/scheduled {}
-
-   ;; deadline day
-   :block/deadline {}
-
-   ;; whether blocks is a repeated block (usually a task)
-   :block/repeated? {}
-
-   :block/created-at {:db/index true}
-   :block/updated-at {:db/index true}
-
-   ;; page additional attributes
-   ;; page's name, lowercase
-   :block/name {:db/unique :db.unique/identity}
-
-   ;; page's original name
-   :block/title {:db/index true}
-
-   ;; page's journal day
-   :block/journal-day {:db/index true}
-
-   ;; macros in block
-   :block/macros {:db/valueType :db.type/ref
-                  :db/cardinality :db.cardinality/many}
-
-   ;; block's file
-   :block/file {:db/valueType :db.type/ref}
-
-   ;; latest tx that affected the block
-   :block/tx-id {}
-
-   ;; file
-   :file/path {:db/unique :db.unique/identity}
-   :file/content {}
-   :file/created-at {}
-   :file/last-modified-at {}
-   :file/size {}})
-
-(def schema-for-db-based-graph
+(def schema
+  "Schema for DB graphs"
   (merge
-   (dissoc schema
-           :block/namespace :block/properties-text-values :block/pre-block? :recent/pages :block/file
-           :block/properties :block/properties-order :block/repeated? :block/deadline :block/scheduled :block/priority
-           :block/marker :block/macros
-           :block/type :block/format)
+   (apply dissoc file-schema/schema file-schema/file-only-attributes)
    {:block/name {:db/index true}        ; remove db/unique for :block/name
     ;; closed value
     :block/closed-value-property {:db/valueType :db.type/ref
                                   :db/cardinality :db.cardinality/many}}))
 
-(def retract-attributes
-  #{:block/refs
-    :block/tags
-    :block/alias
-    :block/marker
-    :block/priority
-    :block/scheduled
-    :block/deadline
-    :block/repeated?
-    :block/pre-block?
-    :block/properties
-    :block/properties-order
-    :block/properties-text-values
-    :block/macros
-    :block/invalid-properties
-    :block/warning})
-
 ;; If only block/title changes
-(def db-version-retract-attributes
+(def retract-attributes
+  "Retract attributes for DB graphs"
   #{:block/refs
     :block/warning})
 
@@ -200,14 +82,14 @@
         (keep (fn [[attr-name attr-body-map]]
                 (when (= :db.type/ref (:db/valueType attr-body-map))
                   attr-name)))
-        schema-for-db-based-graph))
+        schema))
 
 (def card-many-attributes
   (into #{}
         (keep (fn [[attr-name attr-body-map]]
                 (when (= :db.cardinality/many (:db/cardinality attr-body-map))
                   attr-name)))
-        schema-for-db-based-graph))
+        schema))
 
 (def card-many-ref-type-attributes
   (set/intersection card-many-attributes ref-type-attributes))
@@ -216,7 +98,7 @@
   (set/difference ref-type-attributes card-many-attributes))
 
 (def db-non-ref-attributes
-  (->> schema-for-db-based-graph
+  (->> schema
        (keep (fn [[k v]]
                (when (not (:db/valueType v))
                  k)))

deps/db/src/logseq/db/sqlite/build.cljs

@@ -13,11 +13,11 @@
             [logseq.common.util.date-time :as date-time-util]
             [logseq.common.util.page-ref :as page-ref]
             [logseq.common.uuid :as common-uuid]
+            [logseq.db.common.order :as db-order]
             [logseq.db.frontend.class :as db-class]
             [logseq.db.frontend.content :as db-content]
             [logseq.db.frontend.db-ident :as db-ident]
             [logseq.db.frontend.malli-schema :as db-malli-schema]
-            [logseq.db.frontend.order :as db-order]
             [logseq.db.frontend.property :as db-property]
             [logseq.db.frontend.property.build :as db-property-build]
             [logseq.db.frontend.property.type :as db-property-type]

deps/db/src/logseq/db/sqlite/cli.cljs

@@ -1,16 +1,17 @@
 (ns ^:node-only logseq.db.sqlite.cli
-  "Primary ns to interact with DB graphs with node.js based CLIs"
+  "Primary ns to interact with DB files for DB and file graphs with node.js based CLIs"
   (:require ["better-sqlite3" :as sqlite3]
-            [logseq.db.sqlite.common-db :as sqlite-common-db]
+            ["fs" :as fs]
+            ["path" :as node-path]
             ;; FIXME: datascript.core has to come before datascript.storage or else nbb fails
             #_:clj-kondo/ignore
+            [cljs-bean.core :as bean]
             [datascript.core :as d]
             [datascript.storage :refer [IStorage]]
+            [logseq.db.common.sqlite :as sqlite-common-db]
+            [logseq.db.file-based.schema :as file-schema]
             [logseq.db.frontend.schema :as db-schema]
-            [logseq.db.sqlite.util :as sqlite-util]
-            [cljs-bean.core :as bean]
-            ["fs" :as fs]
-            ["path" :as node-path]))
+            [logseq.db.sqlite.util :as sqlite-util]))
 
 ;; Should this check directory name instead if file graphs also
 ;; have this file?
@@ -43,7 +44,7 @@
     (insert-many data)))
 
 (defn- restore-data-from-addr
-"Should be functionally equivalent to db-worker/restore-data-from-addr"
+  "Should be functionally equivalent to db-worker/restore-data-from-addr"
   [db addr]
   (when-let [result (-> (query db (str "select content, addresses from kvs where addr = " addr))
                         first)]
@@ -61,24 +62,24 @@
   (reify IStorage
     (-store [_ addr+data-seq delete-addrs]
             ;; Only difference from db-worker impl is that js data maps don't start with '$' e.g. :$addr -> :addr
-            (let [used-addrs (set (mapcat
-                                   (fn [[addr data]]
-                                     (cons addr
-                                           (when (map? data)
-                                             (:addresses data))))
-                                   addr+data-seq))
-                  delete-addrs (remove used-addrs delete-addrs)
-                  data (map
-                        (fn [[addr data]]
-                          (let [data' (if (map? data) (dissoc data :addresses) data)
-                                addresses (when (map? data)
-                                            (when-let [addresses (:addresses data)]
-                                              (js/JSON.stringify (bean/->js addresses))))]
-                            #js {:addr addr
-                                 :content (sqlite-util/transit-write data')
-                                 :addresses addresses}))
-                        addr+data-seq)]
-              (upsert-addr-content! db data delete-addrs)))
+      (let [used-addrs (set (mapcat
+                             (fn [[addr data]]
+                               (cons addr
+                                     (when (map? data)
+                                       (:addresses data))))
+                             addr+data-seq))
+            delete-addrs (remove used-addrs delete-addrs)
+            data (map
+                  (fn [[addr data]]
+                    (let [data' (if (map? data) (dissoc data :addresses) data)
+                          addresses (when (map? data)
+                                      (when-let [addresses (:addresses data)]
+                                        (js/JSON.stringify (bean/->js addresses))))]
+                      #js {:addr addr
+                           :content (sqlite-util/transit-write data')
+                           :addresses addresses}))
+                  addr+data-seq)]
+        (upsert-addr-content! db data delete-addrs)))
     (-restore [_ addr]
       (restore-data-from-addr db addr))))
 
@@ -91,8 +92,8 @@
         db (new sqlite db-full-path nil)
         ;; For both desktop and CLI, only file graphs have db-name that indicate their db type
         schema (if (sqlite-util/local-file-based-graph? db-name)
-                 db-schema/schema
-                 db-schema/schema-for-db-based-graph)]
+                 file-schema/schema
+                 db-schema/schema)]
     (sqlite-common-db/create-kvs-table! db)
     (let [storage (new-sqlite-storage db)
           conn (sqlite-common-db/get-storage-conn storage schema)]

deps/db/src/logseq/db/sqlite/export.cljs

@@ -8,6 +8,7 @@
             [logseq.db.frontend.class :as db-class]
             [logseq.db.frontend.content :as db-content]
             [logseq.db.frontend.entity-plus :as entity-plus]
+            [logseq.db.frontend.entity-util :as entity-util]
             [logseq.db.frontend.property :as db-property]
             [logseq.db.sqlite.build :as sqlite-build]))
 
@@ -30,7 +31,7 @@
    Pages that are shallow copied are at the edges of export and help keep the export size reasonable and
    avoid exporting unexpected info"
   [page-entity]
-  (if (ldb/journal? page-entity)
+  (if (entity-util/journal? page-entity)
     {:build/journal (:block/journal-day page-entity)}
     {:block/title (block-title page-entity)}))
 
@@ -42,7 +43,7 @@
         [:build/page (cond-> (shallow-copy-page pvalue)
                        (seq (:block/tags pvalue))
                        (assoc :build/tags (->build-tags (:block/tags pvalue))))]
-        (ldb/journal? pvalue)
+        (entity-util/journal? pvalue)
         [:build/page {:build/journal (:block/journal-day pvalue)}]
         :else
         (if (= :node (:logseq.property/type property-ent))
@@ -223,7 +224,7 @@
   [db page-blocks]
   (let [content-ref-uuids (set (mapcat (comp db-content/get-matched-ids block-title) page-blocks))
         content-ref-ents (map #(d/entity db [:block/uuid %]) content-ref-uuids)
-        content-ref-pages (filter #(or (ldb/internal-page? %) (ldb/journal? %)) content-ref-ents)
+        content-ref-pages (filter #(or (ldb/internal-page? %) (entity-util/journal? %)) content-ref-ents)
         content-ref-properties (when-let [prop-ids (seq (map :db/ident (filter ldb/property? content-ref-ents)))]
                                  (update-vals (build-export-properties db prop-ids {:include-uuid? true :shallow-copy? true})
                                               #(merge % {:build/keep-uuid? true})))
@@ -288,7 +289,7 @@
      :pvalue-uuids @*pvalue-uuids}))
 
 (defn- build-uuid-block-export [db pvalue-uuids content-ref-ents {:keys [page-entity]}]
-  (let [content-ref-blocks (set (remove ldb/page? content-ref-ents))
+  (let [content-ref-blocks (set (remove entity-util/page? content-ref-ents))
         uuid-block-ents-to-export (concat (map #(d/entity db [:block/uuid %]) pvalue-uuids)
                                           content-ref-blocks)
         uuid-block-pages

deps/db/src/logseq/db/sqlite/util.cljs

@@ -8,7 +8,8 @@
             [datascript.transit :as dt]
             [logseq.common.util :as common-util]
             [logseq.common.uuid :as common-uuid]
-            [logseq.db.frontend.order :as db-order]
+            [logseq.db.common.order :as db-order]
+            [logseq.db.file-based.schema :as file-schema]
             [logseq.db.frontend.property :as db-property]
             [logseq.db.frontend.property.type :as db-property-type]
             [logseq.db.frontend.schema :as db-schema]))
@@ -65,8 +66,8 @@
   "Returns schema for given repo"
   [repo]
   (if (db-based-graph? repo)
-    db-schema/schema-for-db-based-graph
-    db-schema/schema))
+    db-schema/schema
+    file-schema/schema))
 
 (def block-with-timestamps common-util/block-with-timestamps)
 

deps/db/src/logseq/db/test/helper.cljs

@@ -76,7 +76,7 @@
 (defn create-conn
   "Create a conn for a DB graph seeded with initial data"
   []
-  (let [conn (d/create-conn db-schema/schema-for-db-based-graph)
+  (let [conn (d/create-conn db-schema/schema)
         _ (d/transact! conn (sqlite-create-graph/build-db-initial-data "{}"))]
     (entity-plus/reset-immutable-entities-cache!)
     conn))

deps/db/test/logseq/db/sqlite/common_db_test.cljs → deps/db/test/logseq/db/common/sqlite_test.cljs

@@ -1,9 +1,9 @@
-(ns logseq.db.sqlite.common-db-test
+(ns logseq.db.common.sqlite-test
   (:require [cljs.test :refer [deftest async use-fixtures is testing]]
             ["fs" :as fs]
             ["path" :as node-path]
             [datascript.core :as d]
-            [logseq.db.sqlite.common-db :as sqlite-common-db]
+            [logseq.db.common.sqlite :as sqlite-common-db]
             [logseq.common.util.date-time :as date-time-util]
             [logseq.db.sqlite.cli :as sqlite-cli]
             [clojure.string :as string]))

deps/db/test/logseq/db/frontend/inputs_test.cljs

@@ -1,11 +1,11 @@
 (ns logseq.db.frontend.inputs-test
-  (:require [cljs.test :refer [deftest is]]
-            [cljs-time.core :as t]
+  (:require [cljs-time.core :as t]
+            [cljs.test :refer [deftest is]]
             [datascript.core :as d]
             [logseq.common.util.date-time :as date-time-util]
+            [logseq.db.frontend.inputs :as db-inputs]
             [logseq.db.frontend.rules :as rules]
             [logseq.db.frontend.schema :as db-schema]
-            [logseq.db.frontend.inputs :as db-inputs]
             [logseq.db.sqlite.build :as sqlite-build]
             [logseq.db.test.helper :as db-test]))
 
@@ -17,7 +17,7 @@
          (map first))))
 
 (deftest resolve-input-for-page-and-block-inputs
-  (let [conn (d/create-conn db-schema/schema-for-db-based-graph)
+  (let [conn (d/create-conn db-schema/schema)
         _ (d/transact! conn [{:db/ident :logseq.class/Page}])
         _ (sqlite-build/create-blocks
            conn

deps/db/test/logseq/db/sqlite/create_graph_test.cljs

@@ -134,7 +134,7 @@
         "New graph has no validation errors")))
 
 (deftest property-types
-  (let [conn (d/create-conn db-schema/schema-for-db-based-graph)
+  (let [conn (d/create-conn db-schema/schema)
         _ (d/transact! conn (sqlite-create-graph/build-db-initial-data
                              (pr-str {:macros {"docs-base-url" "https://docs.logseq.com/#/page/$1"}})))]
 

deps/db/test/logseq/db_test.cljs

@@ -2,7 +2,7 @@
   (:require [cljs.test :refer [deftest is]]
             [datascript.core :as d]
             [logseq.db :as ldb]
-            [logseq.db.frontend.schema :as db-schema]
+            [logseq.db.file-based.schema :as file-schema]
             [logseq.db.test.helper :as db-test]))
 
 ;;; datoms
@@ -21,7 +21,7 @@
     :block/parent 2}])
 
 (deftest get-block-children-ids-on-bad-outliner-data
-  (let [db (d/db-with (d/empty-db db-schema/schema)
+  (let [db (d/db-with (d/empty-db file-schema/schema)
                       broken-outliner-data-with-cycle)]
     (is (= "bad outliner data, need to re-index to fix"
            (try (ldb/get-block-children-ids db #uuid "e538d319-48d4-4a6d-ae70-c03bb55b6fe4")

deps/graph-parser/src/logseq/graph_parser.cljs

@@ -4,17 +4,17 @@
   (:require [clojure.set :as set]
             [clojure.string :as string]
             [datascript.core :as d]
-            [logseq.db.frontend.schema :as db-schema]
-            [logseq.graph-parser.extract :as extract]
-            [logseq.common.util :as common-util]
             [logseq.common.config :as common-config]
-            [logseq.db :as ldb]))
+            [logseq.common.util :as common-util]
+            [logseq.db :as ldb]
+            [logseq.db.file-based.schema :as file-schema]
+            [logseq.graph-parser.extract :as extract]))
 
 (defn- retract-blocks-tx
   [blocks retain-uuids]
   (mapcat (fn [{uuid' :block/uuid eid :db/id}]
             (if (and uuid' (contains? retain-uuids uuid'))
-              (map (fn [attr] [:db.fn/retractAttribute eid attr]) db-schema/retract-attributes)
+              (map (fn [attr] [:db.fn/retractAttribute eid attr]) file-schema/retract-attributes)
               (when eid [[:db.fn/retractEntity eid]])))
           blocks))
 

deps/graph-parser/src/logseq/graph_parser/block.cljs

@@ -14,8 +14,8 @@
             [logseq.common.util.page-ref :as page-ref]
             [logseq.common.uuid :as common-uuid]
             [logseq.db :as ldb]
+            [logseq.db.common.order :as db-order]
             [logseq.db.frontend.class :as db-class]
-            [logseq.db.frontend.order :as db-order]
             [logseq.graph-parser.mldoc :as gp-mldoc]
             [logseq.graph-parser.property :as gp-property]
             [logseq.graph-parser.text :as text]

deps/graph-parser/src/logseq/graph_parser/db.cljs

@@ -1,11 +1,11 @@
 (ns logseq.graph-parser.db
   "File graph specific db fns"
-  (:require [datascript.core :as d]
-            [clojure.set :as set]
+  (:require [clojure.set :as set]
             [clojure.string :as string]
+            [datascript.core :as d]
             [logseq.common.util :as common-util]
-            [logseq.db.frontend.schema :as db-schema]
-            [logseq.db :as ldb]))
+            [logseq.db :as ldb]
+            [logseq.db.file-based.schema :as file-schema]))
 
 (defonce built-in-markers
   ["NOW" "LATER" "DOING" "DONE" "CANCELED" "CANCELLED" "IN-PROGRESS" "TODO" "WAIT" "WAITING"])
@@ -50,7 +50,7 @@
 (defn start-conn
   "Create datascript conn with schema and default data"
   []
-  (let [db-conn (d/create-conn db-schema/schema)]
+  (let [db-conn (d/create-conn file-schema/schema)]
     (create-default-pages! db-conn)
     db-conn))
 

deps/graph-parser/src/logseq/graph_parser/exporter.cljs

@@ -16,11 +16,11 @@
             [logseq.common.util.page-ref :as page-ref]
             [logseq.common.uuid :as common-uuid]
             [logseq.db :as ldb]
+            [logseq.db.common.order :as db-order]
             [logseq.db.frontend.class :as db-class]
             [logseq.db.frontend.content :as db-content]
             [logseq.db.frontend.db-ident :as db-ident]
             [logseq.db.frontend.malli-schema :as db-malli-schema]
-            [logseq.db.frontend.order :as db-order]
             [logseq.db.frontend.property :as db-property]
             [logseq.db.frontend.property.build :as db-property-build]
             [logseq.db.frontend.property.type :as db-property-type]

deps/graph-parser/src/logseq/graph_parser/whiteboard.cljs

@@ -1,6 +1,6 @@
 (ns logseq.graph-parser.whiteboard
   "Whiteboard related parser utilities"
-  (:require [logseq.db.frontend.property.util :as db-property-util]
+  (:require [logseq.db.common.property-util :as db-property-util]
             [logseq.db.sqlite.util :as sqlite-util]))
 
 (defn block->shape [block]
@@ -37,14 +37,13 @@
       (assoc block :block/properties properties))
     block))
 
-
 (defn- get-shape-refs [shape]
   (let [portal-refs (when (= "logseq-portal" (:type shape))
                       [{:block/uuid (uuid (:pageId shape))}])
         shape-link-refs (->> (:refs shape)
                              (filter (complement empty?))
                              (keep (fn [ref] (when (parse-uuid ref)
-                                              {:block/uuid (parse-uuid ref)}))))]
+                                               {:block/uuid (parse-uuid ref)}))))]
     (concat portal-refs shape-link-refs)))
 
 (defn- with-whiteboard-block-refs
@@ -59,10 +58,10 @@
   "Main purpose of this function is to populate contents when shapes are used as references in outliner."
   [shape]
   {:block/title (case (:type shape)
-                    "text" (:text shape)
-                    "logseq-portal" ""
-                    "line" (str "whiteboard arrow" (when-let [label (:label shape)] (str ": " label)))
-                    (str "whiteboard " (:type shape)))})
+                  "text" (:text shape)
+                  "logseq-portal" ""
+                  "line" (str "whiteboard arrow" (when-let [label (:label shape)] (str ": " label)))
+                  (str "whiteboard " (:type shape)))})
 
 (defn with-whiteboard-block-props
   "Builds additional block attributes for a whiteboard block. Expects :block/properties

deps/graph-parser/test/logseq/graph_parser/extract_test.cljs

@@ -1,8 +1,8 @@
 (ns logseq.graph-parser.extract-test
   (:require [cljs.test :refer [deftest is are]]
-            [logseq.graph-parser.extract :as extract]
             [datascript.core :as d]
-            [logseq.db.frontend.schema :as db-schema]))
+            [logseq.db.file-based.schema :as file-schema]
+            [logseq.graph-parser.extract :as extract]))
 
 ;; This is a copy of frontend.util.fs/multiplatform-reserved-chars for reserved chars testing
 (def multiplatform-reserved-chars ":\\*\\?\"<>|\\#\\\\")
@@ -45,7 +45,7 @@
 (defn- extract [file content & [options]]
   (extract/extract file
                    content
-                   (merge {:block-pattern "-" :db (d/empty-db db-schema/schema)
+                   (merge {:block-pattern "-" :db (d/empty-db file-schema/schema)
                            :verbose false}
                           options)))
 

deps/outliner/src/logseq/outliner/core.cljs

@@ -6,8 +6,8 @@
             [datascript.impl.entity :as de :refer [Entity]]
             [logseq.common.util :as common-util]
             [logseq.db :as ldb]
-            [logseq.db.frontend.order :as db-order]
-            [logseq.db.frontend.property.util :as db-property-util]
+            [logseq.db.common.order :as db-order]
+            [logseq.db.file-based.schema :as file-schema]
             [logseq.db.frontend.schema :as db-schema]
             [logseq.db.sqlite.create-graph :as sqlite-create-graph]
             [logseq.db.sqlite.util :as sqlite-util]
@@ -284,8 +284,8 @@
                   (seq retract-attributes))
           (let [retract-attributes (concat
                                     (if db-based?
-                                      db-schema/db-version-retract-attributes
-                                      db-schema/retract-attributes)
+                                      db-schema/retract-attributes
+                                      file-schema/retract-attributes)
                                     retract-attributes)]
             (swap! txs-state (fn [txs]
                                (vec
@@ -444,7 +444,7 @@
                        (if (sqlite-util/db-based-graph? repo)
                          ;; Get raw id since insert-blocks doesn't auto-handle raw property values
                          (:db/id (:logseq.property/order-list-type block))
-                         (get (:block/properties block) (db-property-util/get-pid repo :logseq.property/order-list-type))))
+                         (get (:block/properties block) :logseq.order-list-type)))
         db-based? (sqlite-util/db-based-graph? repo)]
     (if-let [list-type (and target-block (list-type-fn target-block))]
       (mapv
@@ -456,7 +456,7 @@
              ((fn [b]
                 (if db-based?
                   (assoc b :logseq.property/order-list-type list-type)
-                  (update b :block/properties assoc (db-property-util/get-pid repo :logseq.property/order-list-type) list-type))))
+                  (update b :block/properties assoc :logseq.order-list-type list-type))))
 
              (not db-based?)
              (assoc :block/title (gp-property/insert-property repo format title :logseq.order-list-type list-type)))))

deps/outliner/src/logseq/outliner/property.cljs

@@ -6,10 +6,10 @@
             [datascript.impl.entity :as de]
             [logseq.common.util :as common-util]
             [logseq.db :as ldb]
+            [logseq.db.common.order :as db-order]
             [logseq.db.frontend.db-ident :as db-ident]
             [logseq.db.frontend.entity-plus :as entity-plus]
             [logseq.db.frontend.malli-schema :as db-malli-schema]
-            [logseq.db.frontend.order :as db-order]
             [logseq.db.frontend.property :as db-property]
             [logseq.db.frontend.property.build :as db-property-build]
             [logseq.db.frontend.property.type :as db-property-type]
@@ -289,7 +289,7 @@
   (let [block-eid (->eid block-eid)
         _ (assert (qualified-keyword? property-id) "property-id should be a keyword")
         block (d/entity @conn block-eid)
-        db-attribute? (some? (db-schema/schema-for-db-based-graph property-id))]
+        db-attribute? (some? (db-schema/schema property-id))]
     (when (= property-id :block/tags)
       (outliner-validate/validate-tags-property @conn [block-eid] v))
     (when (= property-id :logseq.property/parent)

deps/outliner/src/logseq/outliner/tree.cljs

@@ -1,9 +1,9 @@
 (ns logseq.outliner.tree
   "Provides tree fns and INode protocol"
-  (:require [logseq.db :as ldb]
-            [logseq.db.frontend.property.util :as db-property-util]
-            [datascript.core :as d]
-            [datascript.impl.entity :as de]))
+  (:require [datascript.core :as d]
+            [datascript.impl.entity :as de]
+            [logseq.db :as ldb]
+            [logseq.db.common.property-util :as db-property-util]))
 
 (defprotocol INode
   (-save [this txs-state conn repo date-formatter opts])

deps/outliner/test/logseq/outliner/pipeline_test.cljs

@@ -23,7 +23,7 @@
 
 (deftest compute-block-path-refs-tx
   (testing "when a block's :refs change, descendants of block have correct :block/path-refs"
-    (let [conn (d/create-conn db-schema/schema-for-db-based-graph)
+    (let [conn (d/create-conn db-schema/schema)
           ;; needed in order for path-refs to be setup correctly with init data
           _ (db-pipeline/add-listener conn)
           _ (d/transact! conn (sqlite-create-graph/build-db-initial-data "{}"))

scripts/src/logseq/tasks/dev/db_and_file_graphs.clj

@@ -1,5 +1,6 @@
 (ns logseq.tasks.dev.db-and-file-graphs
   (:require [babashka.process :refer [shell]]
+            [clojure.set :as set]
             [clojure.string :as string]))
 
 (defn- escape-shell-regex
@@ -12,7 +13,7 @@
 (def db-graph-ns
   "Namespaces or parent namespaces _only_ for DB graphs. Use a '.' at end of a namespace for parent namespaces"
   (mapv escape-shell-regex
-        ["logseq.db.sqlite." "logseq.db.frontend.property" "logseq.db.frontend.malli-schema"
+        ["logseq.db.sqlite." "logseq.db.frontend."
          "electron.db"
          "frontend.handler.db-based."
          "frontend.worker.handler.page.db-based"
@@ -33,18 +34,23 @@
          "frontend.components.file-based"
          "frontend.util.fs"]))
 
+(def block-name-db-graph-paths
+  "DB graph paths with :block/name"
+  ["deps/db/src/logseq/db/frontend"
+   "deps/db/src/logseq/db/sqlite"
+   "src/main/frontend/worker/handler/page/db_based"])
+
 (def db-graph-paths
   "Paths _only_ for DB graphs"
-  ["src/main/frontend/handler/db_based"
-   ;; TODO: Re-enable if db namespace support is moved elsewhere
-  ;;  "src/main/frontend/worker/handler/page/db_based"
-   "src/main/frontend/components/class.cljs"
-   "src/main/frontend/components/property.cljs"
-   "src/main/frontend/components/property"
-   "src/main/frontend/components/objects.cljs"
-   "src/main/frontend/components/db_based"
-   "src/main/frontend/components/query/view.cljs"
-   "src/electron/electron/db.cljs"])
+  (into block-name-db-graph-paths
+        ["src/main/frontend/handler/db_based"
+         "src/main/frontend/components/class.cljs"
+         "src/main/frontend/components/property.cljs"
+         "src/main/frontend/components/property"
+         "src/main/frontend/components/objects.cljs"
+         "src/main/frontend/components/db_based"
+         "src/main/frontend/components/query/view.cljs"
+         "src/electron/electron/db.cljs"]))
 
 (def file-graph-paths
   "Paths _only_ for file graphs"
@@ -55,11 +61,16 @@
    "src/main/frontend/components/file_based"
    "src/main/frontend/util/fs.cljs"])
 
+(defn- grep-many
+  "Git greps a coll of patterns for given paths. Returns result from process/shell"
+  [patterns paths]
+  (apply shell {:out :string :continue true}
+         "git grep -E" (str "(" (string/join "|" patterns) ")")
+         paths))
+
 (defn- validate-db-ns-not-in-file
   []
-  (let [res (apply shell {:out :string :continue true}
-                   "git grep -E" (str "(" (string/join "|" db-graph-ns) ")")
-                   file-graph-paths)]
+  (let [res (grep-many db-graph-ns file-graph-paths)]
     (when-not (and (= 1 (:exit res)) (= "" (:out res)))
       (println "The following db graph namespaces should not be in file graph files:")
       (println (:out res))
@@ -67,9 +78,7 @@
 
 (defn- validate-file-ns-not-in-db
   []
-  (let [res (apply shell {:out :string :continue true}
-                   "git grep -E" (str "(" (string/join "|" file-graph-ns) ")")
-                   db-graph-paths)]
+  (let [res (grep-many file-graph-ns db-graph-paths)]
     (when-not (and (= 1 (:exit res)) (= "" (:out res)))
       (println "The following file graph namespaces should not be in db graph files:")
       (println (:out res))
@@ -77,10 +86,11 @@
 
 (defn- validate-multi-graph-fns-not-in-file-or-db
   []
-  (let [multi-graph-fns ["config/db-based-graph\\?" "sqlite-util/db-based-graph\\?"]
-        res (apply shell {:out :string :continue true}
-                   "git grep -E" (str "(" (string/join "|" multi-graph-fns) ")")
-                   (into file-graph-paths db-graph-paths))]
+  ;; TODO: Lint `(db-based-graph?` when db.frontend.entity-plus is split into separate graph contexts
+  (let [multi-graph-fns ["/db-based-graph\\?"
+                         ;; Use file-entity-util and entity-util when in a single graph context
+                         "ldb/whiteboard\\?" "ldb/journal\\?" "ldb/page\\?"]
+        res (grep-many multi-graph-fns (into file-graph-paths db-graph-paths))]
     (when-not (and (= 1 (:exit res)) (= "" (:out res)))
       (println "The following files should not have fns meant to be used in multi-graph contexts:")
       (println (:out res))
@@ -89,37 +99,47 @@
 (defn- validate-file-concepts-not-in-db
   []
   (let [file-concepts (->>
-                       ;; from logseq.db.frontend.schema
+                       ;; from logseq.db.file-based.schema
                        [:block/namespace :block/properties-text-values :block/pre-block :recent/pages :block/file :block/properties-order
                         :block/repeated :block/deadline :block/scheduled :block/priority :block/marker :block/macros
                         :block/type :block/format]
                        (map str)
                        (into [;; e.g. block/properties :title
                               "block/properties :"
-                              "block/name"
-                              ;; anything org mode
-                              "org"
+                              ;; anything org mode except for org.babashka
+                              "org[^\\.]"
                               "#+BEGIN_"
                               "#+END_"
-                              "pre-block"
-                              "db/get-page"
-                              "/page-name-sanity-lc"]))
+                              "pre-block"]))
         ;; For now use the whole code line. If this is too brittle can make this smaller
-        allowed-exceptions #{"{:block/name page-title})))"
+        allowed-exceptions #{":block/pre-block? :block/scheduled :block/deadline :block/type :block/name :block/marker"
+                             "(dissoc :block/format))]"
+                             ;; The next 3 are from components.property.value
                              "{:block/name page-title})"
                              "(when-not (db/get-page journal)"
-                             "(let [value (if datetime? (tc/to-long d) (db/get-page journal))]"
-                             "(dissoc :block/format))]"}
-        res (apply shell {:out :string :continue true}
-                   "git grep -E" (str "(" (string/join "|" file-concepts) ")")
-                   db-graph-paths)
+                             "(let [value (if datetime? (tc/to-long d) (db/get-page journal))]"}
+        res (grep-many file-concepts db-graph-paths)
         invalid-lines (when (= 0 (:exit res))
-                        (remove #(->> (string/split % #":\s+") second string/trim (contains? allowed-exceptions))
-                                (string/split-lines (:out res))))]
-    (when (> (:exit res) 1) (System/exit 1))
-    (when (and (= 0 (:exit res)) (seq invalid-lines))
+                        (remove #(some->> (string/split % #":\s+") second string/trim (contains? allowed-exceptions))
+                                (string/split-lines (:out res))))
+        _ (when (> (:exit res) 1) (System/exit 1))
+        _ (when (and (= 0 (:exit res)) (seq invalid-lines))
+            (println "The following files should not have contained file specific concepts:")
+            (println (string/join "\n" invalid-lines))
+            (System/exit 1))
+
+        ;; :block/name isn't used in db graphs except for fns with journal or internal-page
+        block-name-file-concepts #{"block/name" "/page-name-sanity-lc" "db/get-page"}
+        no-block-name-db-graph-paths (set/difference (set db-graph-paths) (set block-name-db-graph-paths))
+        block-name-res (grep-many block-name-file-concepts no-block-name-db-graph-paths)
+        block-name-invalid-lines (when (= 0 (:exit block-name-res))
+                                   (remove #(some->> (string/split % #":\s+") second string/trim (contains? allowed-exceptions))
+                                           (string/split-lines (:out block-name-res))))]
+
+    (when (> (:exit block-name-res) 1) (System/exit 1))
+    (when (and (= 0 (:exit block-name-res)) (seq block-name-invalid-lines))
       (println "The following files should not have contained file specific concepts:")
-      (println (string/join "\n" invalid-lines))
+      (println (string/join "\n" block-name-invalid-lines))
       (System/exit 1))))
 
 (defn- validate-db-concepts-not-in-file
@@ -128,9 +148,7 @@
         ;; from logseq.db.frontend.schema
         ["closed-value" "class/properties" "classes" "property/parent"
          "logseq.property" "logseq.class"]
-        res (apply shell {:out :string :continue true}
-                   "git grep -E" (str "(" (string/join "|" db-concepts) ")")
-                   file-graph-paths)]
+        res (grep-many db-concepts file-graph-paths)]
     (when-not (and (= 1 (:exit res)) (= "" (:out res)))
       (println "The following files should not have contained db specific concepts:")
       (println (:out res))

src/electron/electron/db.cljs

@@ -1,9 +1,9 @@
 (ns electron.db
   "Provides SQLite dbs for electron and manages files of those dbs"
-  (:require ["path" :as node-path]
+  (:require ["electron" :refer [app]]
             ["fs-extra" :as fs]
-            ["electron" :refer [app]]
-            [logseq.db.sqlite.common-db :as sqlite-common-db]))
+            ["path" :as node-path]
+            [logseq.db.common.sqlite :as sqlite-common-db]))
 
 (defn get-graphs-dir
   []

src/electron/electron/handler.cljs

@@ -31,7 +31,7 @@
             [electron.window :as win]
             [goog.functions :refer [debounce]]
             [logseq.common.graph :as common-graph]
-            [logseq.db.sqlite.common-db :as sqlite-common-db]
+            [logseq.db.common.sqlite :as sqlite-common-db]
             [logseq.db.sqlite.util :as sqlite-util]
             [promesa.core :as p]))
 

src/main/frontend/common/file/core.cljs

@@ -1,12 +1,13 @@
 (ns frontend.common.file.core
-  "Save file to disk. Used by both file and DB graphs and shared
-   by worker and frontend namespaces"
+  "Convert blocks to file content. Used for exports and saving file to disk. Shared
+  by worker and frontend namespaces"
   (:require [clojure.string :as string]
-            [logseq.graph-parser.property :as gp-property]
             [datascript.core :as d]
             [logseq.db :as ldb]
             [logseq.db.frontend.content :as db-content]
+            [logseq.db.frontend.entity-plus :as entity-plus]
             [logseq.db.sqlite.util :as sqlite-util]
+            [logseq.graph-parser.property :as gp-property]
             [logseq.outliner.tree :as otree]))
 
 (defn- indented-block-content
@@ -97,8 +98,8 @@
               content (if page? nil (transform-content repo db f level opts context))
               new-content
               (if-let [children (seq (:block/children f))]
-                     (cons content (tree->file-content-aux repo db children {:init-level (inc level)} context))
-                     [content])]
+                (cons content (tree->file-content-aux repo db children {:init-level (inc level)} context))
+                [content])]
           (conj! block-contents new-content)
           (recur r level))))))
 
@@ -107,6 +108,12 @@
   [repo db tree opts context]
   (->> (tree->file-content-aux repo db tree opts context) (string/join "\n")))
 
+(defn- update-block-content
+  [db item eid]
+  ;; This may not be needed if this becomes a file-graph only context
+  (if (entity-plus/db-based-graph? db)
+    (db-content/update-block-content db item eid)
+    item))
 
 (defn block->content
   "Converts a block including its children (recursively) to plain-text."
@@ -117,7 +124,7 @@
                          0
                          1))
         blocks (->> (d/pull-many db '[*] (keep :db/id (ldb/get-block-and-children db root-block-uuid)))
-                    (map #(db-content/update-block-content db % (:db/id %))))
+                    (map #(update-block-content db % (:db/id %))))
         tree (otree/blocks->vec-tree repo db blocks (str root-block-uuid))]
     (tree->file-content repo db tree
                         (assoc tree->file-opts :init-level init-level)

src/main/frontend/components/objects.cljs

@@ -13,7 +13,7 @@
             [frontend.modules.outliner.op :as outliner-op]
             [frontend.modules.outliner.ui :as ui-outliner-tx]
             [frontend.state :as state]
-            [logseq.db :as ldb]
+            [logseq.db.frontend.entity-util :as entity-util]
             [logseq.db.frontend.property :as db-property]
             [logseq.outliner.property :as outliner-property]
             [logseq.shui.ui :as shui]
@@ -103,8 +103,8 @@
                                                                            :class-schema? true}]))
                  :on-delete-rows (fn [table selected-rows]
                                      ;; Built-in objects must not be deleted e.g. Tag, Property and Root
-                                   (let [pages (->> selected-rows (filter ldb/page?) (remove :logseq.property/built-in?))
-                                         blocks (->> selected-rows (remove ldb/page?) (remove :logseq.property/built-in?))]
+                                   (let [pages (->> selected-rows (filter entity-util/page?) (remove :logseq.property/built-in?))
+                                         blocks (->> selected-rows (remove entity-util/page?) (remove :logseq.property/built-in?))]
                                      (p/do!
                                       (when-let [f (get-in table [:data-fns :set-row-selection!])]
                                         (f {}))
@@ -178,8 +178,8 @@
                  ;; Relationships with built-in properties must not be deleted e.g. built-in? or parent
                  :on-delete-rows (when-not (:logseq.property/built-in? property)
                                    (fn [table selected-rows]
-                                     (let [pages (->> selected-rows (filter ldb/page?) (remove :logseq.property/built-in?))
-                                           blocks (->> selected-rows (remove ldb/page?) (remove :logseq.property/built-in?))]
+                                     (let [pages (->> selected-rows (filter entity-util/page?) (remove :logseq.property/built-in?))
+                                           blocks (->> selected-rows (remove entity-util/page?) (remove :logseq.property/built-in?))]
                                        (p/do!
                                         (set-data! (get-property-related-objects (state/get-current-repo) property))
                                         (when-let [f (get-in table [:data-fns :set-row-selection!])]

src/main/frontend/components/plugins.cljs

@@ -650,7 +650,11 @@
 
       (when market?
         (let [aim-icon #(if (= sort-by %) "check" "circle")
-              items [{:title (t :plugin/downloads)
+              items [{:title (t :plugin/popular)
+                      :options {:on-click #(reset! *sort-by :default)}
+                      :icon (ui/icon (aim-icon :default))}
+
+                     {:title (t :plugin/downloads)
                       :options {:on-click #(reset! *sort-by :downloads)}
                       :icon (ui/icon (aim-icon :downloads))}
 
@@ -664,7 +668,7 @@
 
           (ui/button
            (ui/icon "arrows-sort")
-           :class (str (when-not (contains? #{:default :downloads} sort-by) "picked ") "sort-or-filter-by")
+           :class (str (when-not (contains? #{:default :popular} sort-by) "picked ") "sort-or-filter-by")
            :on-click #(shui/popup-show! (.-target %)
                                         (fn [{:keys [id]}]
                                           (render-classic-dropdown-items id items))
@@ -749,13 +753,61 @@
     [:div {:ref (.-ref inViewState)}
      [:p.py-1.text-center.opacity-0 (when (.-inView inViewState) "·")]]))
 
+(defn weighted-sort-by
+  [key pkgs]
+  (let [default? (or (nil? key) (= key :default))
+        grouped-pkgs (if default?
+                       (some->> pkgs
+                                (group-by (fn [{:keys [addedAt]}]
+                                            (and (number? addedAt)
+                                                 (< (- (js/Date.now) addedAt)
+                                         ;; under 6 days
+                                                    (* 1000 60 60 24 6)))))
+                                (into {}))
+                       {false pkgs})
+        pinned-pkgs (get grouped-pkgs true)
+        pkgs (get grouped-pkgs false)
+        ;; calculate weight
+        [key pkgs] (if default?
+                     (let [decay-factor 0.01
+                           download-weight 0.8
+                           star-weight 0.2]
+                       (letfn [(normalize [vals val]
+                                 (let [min-val (apply min vals)
+                                       max-val (apply max vals)]
+                                   (if (= max-val min-val) 1
+                                       (/ (- val min-val) (- max-val val)))))
+                               (time-diff-in-days [ts]
+                                 (let [time-diff (- (js/Date.now) ts)]
+                                   (/ time-diff (* 1000 60 60 24))))]
+                         [:weight
+                          (let [all-downloads (map :downloads pkgs)
+                                all-stars (map :stars pkgs)]
+                            (->> pkgs
+                                 (map (fn [{:keys [downloads stars latestAt] :as pkg}]
+                                        (let [days-since-latest (time-diff-in-days latestAt)
+                                              decay (js/Math.exp (* -1 decay-factor days-since-latest))
+                                              normalized-downloads (normalize all-downloads downloads)
+                                              normalize-stars (normalize all-stars stars)
+                                              download-score (* normalized-downloads download-weight)
+                                              star-score (* normalize-stars star-weight)]
+                                          (assoc pkg :weight (+ download-score star-score decay)))))))]))
+                     [key pkgs])]
+    (->> (apply sort-by
+                (conj
+                 (case key
+                   :letters [#(util/safe-lower-case (or (:title %) (:name %)))]
+                   [key #(compare %2 %1)])
+                 pkgs))
+         (concat pinned-pkgs))))
+
 (rum/defcs ^:large-vars/data-var marketplace-plugins
   < rum/static rum/reactive
   plugin-items-list-mixins
   (rum/local false ::fetching)
   (rum/local "" ::search-key)
   (rum/local :plugins ::category)
-  (rum/local :downloads ::sort-by)                        ;; downloads / stars / letters / updates
+  (rum/local :default ::sort-by)        ;; default (weighted) / downloads / stars / letters / updates
   (rum/local :default ::filter-by)
   (rum/local nil ::error)
   (rum/local nil ::cached-query-flag)
@@ -811,17 +863,16 @@
                              filtered-pkgs)
         filtered-pkgs      (map #(if-let [stat (get stats (keyword (:id %)))]
                                    (let [downloads (:total_downloads stat)
-                                         stars     (:stargazers_count stat)]
-                                     (assoc % :stat stat
+                                         stars     (:stargazers_count stat)
+                                         latest-at (some-> (:updated_at stat) (js/Date.) (.getTime))]
+                                     (assoc %
+                                            :stat stat
                                             :stars stars
+                                            :latestAt latest-at
                                             :downloads downloads))
                                    %) filtered-pkgs)
-        sorted-plugins     (apply sort-by
-                                  (conj
-                                   (case @*sort-by
-                                     :letters [#(util/safe-lower-case (or (:title %) (:name %)))]
-                                     [@*sort-by #(compare %2 %1)])
-                                   filtered-pkgs))
+        _ (def debug-pkgs filtered-pkgs)
+        sorted-plugins     (weighted-sort-by @*sort-by filtered-pkgs)
 
         fn-query-flag      (fn [] (string/join "_" (map #(str @%) [*filter-by *sort-by *search-key *category])))
         str-query-flag     (fn-query-flag)

src/main/frontend/components/property.cljs

@@ -25,7 +25,8 @@
             [frontend.ui :as ui]
             [frontend.util :as util]
             [logseq.db :as ldb]
-            [logseq.db.frontend.order :as db-order]
+            [logseq.db.common.order :as db-order]
+            [logseq.db.frontend.entity-util :as entity-util]
             [logseq.db.frontend.property :as db-property]
             [logseq.db.frontend.property.type :as db-property-type]
             [logseq.outliner.core :as outliner-core]
@@ -355,7 +356,7 @@
         *show-new-property-config? (::show-new-property-config? state)
         *show-class-select? (::show-class-select? state)
         *property-schema (::property-schema state)
-        page? (ldb/page? block)
+        page? (entity-util/page? block)
         block-types (let [types (ldb/get-entity-types block)]
                       (cond-> types
                         (and page? (not (contains? types :page)))
@@ -675,7 +676,7 @@
             properties' (remove (fn [[k _v]] (contains? remove-properties k)) full-properties)
             properties'' (->> properties'
                               (remove (fn [[k _v]] (= k :logseq.property.class/properties))))
-            page? (ldb/page? block)]
+            page? (entity-util/page? block)]
         [:div.ls-properties-area
          {:id id
           :class (util/classnames [{:ls-page-properties page?}])

src/main/frontend/components/property/config.cljs

@@ -21,7 +21,7 @@
             [frontend.util :as util]
             [goog.dom :as gdom]
             [logseq.db :as ldb]
-            [logseq.db.frontend.order :as db-order]
+            [logseq.db.common.order :as db-order]
             [logseq.db.frontend.property :as db-property]
             [logseq.db.frontend.property.type :as db-property-type]
             [logseq.outliner.core :as outliner-core]

src/main/frontend/components/property/value.cljs

@@ -33,6 +33,7 @@
             [lambdaisland.glogi :as log]
             [logseq.common.util.macro :as macro-util]
             [logseq.db :as ldb]
+            [logseq.db.frontend.entity-util :as entity-util]
             [logseq.db.frontend.property :as db-property]
             [logseq.db.frontend.property.type :as db-property-type]
             [logseq.outliner.property :as outliner-property]
@@ -604,7 +605,7 @@
     "hash"
     (ldb/property? node)
     "letter-p"
-    (ldb/page? node)
+    (entity-util/page? node)
     "page"
     :else
     "letter-n"))
@@ -732,7 +733,7 @@
                                                   (when-not (string/blank? (string/trim chosen))
                                                     (p/let [result (<create-page-if-not-exists! block property classes' chosen)]
                                                       [result true])))
-                                      _ (when (and (integer? id) (not (ldb/page? (db/entity id))))
+                                      _ (when (and (integer? id) (not (entity-util/page? (db/entity id))))
                                           (db-async/<get-block repo id))]
                                 (p/do!
                                  (if id
@@ -947,7 +948,7 @@
               (property-normal-block-value block property v-block)
 
               ;; page/class/etc.
-              (ldb/page? v-block)
+              (entity-util/page? v-block)
               (rum/with-key
                 (page-cp {:disable-preview? true
                           :tag? class?} v-block)
@@ -997,7 +998,7 @@
        closed-values?
        (closed-value-item value opts)
 
-       (or (ldb/page? value)
+       (or (entity-util/page? value)
            (and (seq (:block/tags value))
                 ;; FIXME: page-cp should be renamed to node-cp and
                 ;; support this case and maybe other complex cases.

src/main/frontend/components/query/view.cljs

@@ -7,6 +7,7 @@
             [frontend.modules.outliner.ui :as ui-outliner-tx]
             [frontend.state]
             [logseq.db :as ldb]
+            [logseq.db.frontend.entity-util :as entity-util]
             [promesa.core :as p]
             [rum.core :as rum]))
 
@@ -56,8 +57,8 @@
        :set-data! set-data!
        :columns columns'
        :on-delete-rows (fn [table selected-rows]
-                         (let [pages (filter ldb/page? selected-rows)
-                               blocks (remove ldb/page? selected-rows)
+                         (let [pages (filter entity-util/page? selected-rows)
+                               blocks (remove entity-util/page? selected-rows)
                                selected (set (map :id (remove :logseq.property/built-in? selected-rows)))
                                data' (remove (fn [row] (contains? selected (:id row))) (:data table))]
                            (p/do!

src/main/frontend/db/conn.cljs

@@ -1,18 +1,18 @@
 (ns frontend.db.conn
   "Contains db connections."
   (:require [clojure.string :as string]
-            [frontend.util :as util]
-            [frontend.mobile.util :as mobile-util]
+            [datascript.core :as d]
             [frontend.config :as config]
+            [frontend.db.conn-state :as db-conn-state]
+            [frontend.mobile.util :as mobile-util]
+            [frontend.state :as state]
+            [frontend.util :as util]
             [frontend.util.text :as text-util]
-            [logseq.graph-parser.text :as text]
-            [logseq.graph-parser.db :as gp-db]
-            [datascript.core :as d]
-            [logseq.db :as ldb]
             [logseq.common.util :as common-util]
+            [logseq.db :as ldb]
             [logseq.db.frontend.schema :as db-schema]
-            [frontend.db.conn-state :as db-conn-state]
-            [frontend.state :as state]))
+            [logseq.graph-parser.db :as gp-db]
+            [logseq.graph-parser.text :as text]))
 
 (defonce conns db-conn-state/conns)
 (def get-repo-path db-conn-state/get-repo-path)
@@ -81,7 +81,7 @@
   ([repo {:keys [listen-handler]}]
    (let [db-name (db-conn-state/get-repo-path repo)
          db-conn (if (config/db-based-graph? repo)
-                   (d/create-conn db-schema/schema-for-db-based-graph)
+                   (d/create-conn db-schema/schema)
                    (gp-db/start-conn))]
      (swap! conns assoc db-name db-conn)
      (when listen-handler

src/main/frontend/db/query_custom.cljs

@@ -1,13 +1,14 @@
 (ns frontend.db.query-custom
   "Handles executing custom queries a.k.a. advanced queries"
-  (:require [frontend.state :as state]
-            [frontend.db.query-react :as query-react]
-            [frontend.db.query-dsl :as query-dsl]
+  (:require [clojure.walk :as walk]
+            [frontend.config :as config]
             [frontend.db.model :as model]
-            [logseq.db.frontend.rules :as rules]
+            [frontend.db.query-dsl :as query-dsl]
+            [frontend.db.query-react :as query-react]
+            [frontend.state :as state]
             [frontend.util.datalog :as datalog-util]
-            [clojure.walk :as walk]
-            [frontend.config :as config]))
+            [logseq.db.file-based.rules :as file-rules]
+            [logseq.db.frontend.rules :as rules]))
 
 ;; FIXME: what if users want to query other attributes than block-attrs?
 (defn- replace-star-with-block-attrs!
@@ -27,7 +28,7 @@
   "Searches query's :where for rules and adds them to query if used"
   [{:keys [query] :as query-m} {:keys [db-graph?]}]
   (let [{:keys [where in]} (datalog-util/query-vec->map query)
-        query-dsl-rules (if db-graph? rules/db-query-dsl-rules rules/query-dsl-rules)
+        query-dsl-rules (if db-graph? rules/db-query-dsl-rules file-rules/query-dsl-rules)
         rules-found (datalog-util/find-rules-in-where where (-> query-dsl-rules keys set))]
     (if (seq rules-found)
       (if (and (= '% (last in)) (vector? (last (:inputs query-m))))

src/main/frontend/db/query_dsl.cljs

@@ -18,6 +18,7 @@
             [logseq.common.util :as common-util]
             [logseq.common.util.date-time :as date-time-util]
             [logseq.common.util.page-ref :as page-ref]
+            [logseq.db.file-based.rules :as file-rules]
             [logseq.db.frontend.rules :as rules]
             [logseq.graph-parser.text :as text]))
 
@@ -696,7 +697,7 @@ Some bindings in this fn:
       {:query result'
        :rules (if db-graph?
                 (rules/extract-rules rules/db-query-dsl-rules rules {:deps rules/rules-dependencies})
-                (mapv rules/query-dsl-rules rules))
+                (mapv file-rules/query-dsl-rules rules))
        :sort-by @sort-by
        :blocks? (boolean @blocks?)
        :sample sample})))

src/main/frontend/db/restore.cljs

@@ -1,12 +1,12 @@
 (ns frontend.db.restore
   "Fns for DB restore(from text or sqlite)"
-  (:require [frontend.db.conn :as db-conn]
-            [frontend.state :as state]
+  (:require [cljs-time.core :as t]
+            [datascript.transit :as dt]
+            [frontend.db.conn :as db-conn]
             [frontend.persist-db :as persist-db]
-            [promesa.core :as p]
-            [cljs-time.core :as t]
-            [logseq.db.sqlite.common-db :as sqlite-common-db]
-            [datascript.transit :as dt]))
+            [frontend.state :as state]
+            [logseq.db.common.sqlite :as sqlite-common-db]
+            [promesa.core :as p]))
 
 (defn restore-graph!
   "Restore db from SQLite"

src/main/frontend/db/utils.cljs

@@ -1,9 +1,10 @@
 (ns frontend.db.utils
   "Some utils are required by other namespace in frontend.db package."
   (:require [datascript.core :as d]
-            [frontend.state :as state]
             [frontend.db.conn :as conn]
-            [logseq.db.frontend.content :as db-content]))
+            [frontend.state :as state]
+            [logseq.db.frontend.content :as db-content]
+            [logseq.db.frontend.entity-plus :as entity-plus]))
 
 ;; transit serialization
 
@@ -36,17 +37,19 @@
      (let [eid (if (uuid? eid) [:block/uuid eid] eid)]
        (when-let [db (if (string? repo-or-db)
                      ;; repo
-                      (let [repo (or repo-or-db (state/get-current-repo))]
-                        (conn/get-db repo))
+                       (let [repo (or repo-or-db (state/get-current-repo))]
+                         (conn/get-db repo))
                      ;; db
-                      repo-or-db)]
-        (d/entity db eid))))))
+                       repo-or-db)]
+         (d/entity db eid))))))
 
 (defn update-block-content
   "Replace `[[internal-id]]` with `[[page name]]`"
   [item eid]
   (if-let [db (conn/get-db)]
-    (db-content/update-block-content db item eid)
+    (if (entity-plus/db-based-graph? db)
+      (db-content/update-block-content db item eid)
+      item)
     item))
 
 (defn pull

src/main/frontend/handler/db_based/rtc.cljs

@@ -12,7 +12,7 @@
             [frontend.util :as util]
             [logseq.common.util :as common-util]
             [logseq.db :as ldb]
-            [logseq.db.sqlite.common-db :as sqlite-common-db]
+            [logseq.db.common.sqlite :as sqlite-common-db]
             [logseq.shui.ui :as shui]
             [missionary.core :as m]
             [promesa.core :as p]))

src/main/frontend/handler/editor.cljs

@@ -55,9 +55,9 @@
             [logseq.common.util.block-ref :as block-ref]
             [logseq.common.util.page-ref :as page-ref]
             [logseq.db :as ldb]
+            [logseq.db.file-based.schema :as file-schema]
             [logseq.db.frontend.entity-plus :as entity-plus]
             [logseq.db.frontend.property :as db-property]
-            [logseq.db.frontend.schema :as db-schema]
             [logseq.graph-parser.block :as gp-block]
             [logseq.graph-parser.mldoc :as gp-mldoc]
             [logseq.graph-parser.property :as gp-property]
@@ -385,7 +385,7 @@
         selection-end (util/get-selection-end input)
         [fst-block-text snd-block-text] (compute-fst-snd-block-text value selection-start selection-end)
         current-block (assoc block :block/title fst-block-text)
-        current-block (apply dissoc current-block db-schema/retract-attributes)
+        current-block (apply dissoc current-block file-schema/retract-attributes)
         new-m {:block/uuid (db/new-block-id)
                :block/title snd-block-text}
         next-block (-> (merge (select-keys block [:block/parent :block/format :block/page])

src/main/frontend/handler/export.cljs

@@ -24,7 +24,7 @@
    [goog.dom :as gdom]
    [lambdaisland.glogi :as log]
    [logseq.db :as ldb]
-   [logseq.db.sqlite.common-db :as sqlite-common-db]
+   [logseq.db.common.sqlite :as sqlite-common-db]
    [logseq.publishing.html :as publish-html]
    [promesa.core :as p])
   (:import

src/main/frontend/handler/file_based/editor.cljs

@@ -23,7 +23,7 @@
             [logseq.common.util :as common-util]
             [logseq.common.util.block-ref :as block-ref]
             [logseq.db :as ldb]
-            [logseq.db.frontend.schema :as db-schema]))
+            [logseq.db.file-based.schema :as file-schema]))
 
 (defn- remove-non-existed-refs!
   [refs]
@@ -119,7 +119,7 @@
         block (assoc block
                      :block/title content'
                      :block/format format)
-        block (apply dissoc block (remove #{:block/pre-block?} db-schema/retract-attributes))
+        block (apply dissoc block (remove #{:block/pre-block?} file-schema/retract-attributes))
         block (block/parse-block block)
         block (if (and first-block? (:block/pre-block? block))
                 block

src/main/frontend/handler/property/util.cljs

@@ -4,7 +4,7 @@
   compatible with file graphs"
   (:require [frontend.db.conn :as conn]
             [frontend.state :as state]
-            [logseq.db.frontend.property.util :as db-property-util]))
+            [logseq.db.common.property-util :as db-property-util]))
 
 (defn lookup
   "Get the property value by a built-in property's db-ident from block. For file and db graphs"

src/main/frontend/handler/whiteboard.cljs

@@ -1,26 +1,26 @@
 (ns frontend.handler.whiteboard
   "Whiteboard related handlers"
-  (:require [datascript.core :as d]
+  (:require [cljs-bean.core :as bean]
+            [clojure.set :as set]
+            [clojure.string :as string]
+            [datascript.core :as d]
             [dommy.core :as dom]
+            [frontend.config :as config]
             [frontend.db :as db]
             [frontend.db.model :as model]
             [frontend.handler.editor :as editor-handler]
-            [frontend.handler.route :as route-handler]
             [frontend.handler.property.util :as pu]
+            [frontend.handler.route :as route-handler]
             [frontend.state :as state]
-            [frontend.config :as config]
             [frontend.storage :as storage]
             [frontend.util :as util]
-            [logseq.common.util :as common-util]
-            [logseq.graph-parser.whiteboard :as gp-whiteboard]
-            [promesa.core :as p]
             [goog.object :as gobj]
-            [clojure.set :as set]
-            [clojure.string :as string]
-            [cljs-bean.core :as bean]
+            [logseq.common.util :as common-util]
+            [logseq.db.common.order :as db-order]
             [logseq.db.sqlite.util :as sqlite-util]
-            [logseq.db.frontend.order :as db-order]
-            [logseq.outliner.core :as outliner-core]))
+            [logseq.graph-parser.whiteboard :as gp-whiteboard]
+            [logseq.outliner.core :as outliner-core]
+            [promesa.core :as p]))
 
 (defn js->clj-keywordize
   [obj]

src/main/frontend/worker/db/migrate.cljs

@@ -13,9 +13,9 @@
             [logseq.common.util.page-ref :as page-ref]
             [logseq.common.uuid :as common-uuid]
             [logseq.db :as ldb]
+            [logseq.db.common.order :as db-order]
             [logseq.db.frontend.class :as db-class]
             [logseq.db.frontend.content :as db-content]
-            [logseq.db.frontend.order :as db-order]
             [logseq.db.frontend.property :as db-property]
             [logseq.db.frontend.property.build :as db-property-build]
             [logseq.db.frontend.schema :as db-schema]

src/main/frontend/worker/db_worker.cljs

@@ -32,9 +32,9 @@
             [logseq.common.config :as common-config]
             [logseq.common.util :as common-util]
             [logseq.db :as ldb]
-            [logseq.db.frontend.order :as db-order]
+            [logseq.db.common.order :as db-order]
+            [logseq.db.common.sqlite :as sqlite-common-db]
             [logseq.db.frontend.schema :as db-schema]
-            [logseq.db.sqlite.common-db :as sqlite-common-db]
             [logseq.db.sqlite.create-graph :as sqlite-create-graph]
             [logseq.db.sqlite.export :as sqlite-export]
             [logseq.db.sqlite.util :as sqlite-util]
@@ -114,7 +114,7 @@
   "Persistent-sorted-set has been broken, used addresses can't be found"
   [datascript-conn sqlite-db import-type]
   (let [datoms (get-all-datoms-from-sqlite-db sqlite-db)
-        db (d/init-db [] db-schema/schema-for-db-based-graph
+        db (d/init-db [] db-schema/schema
                       {:storage (storage/storage @datascript-conn)})
         db (d/db-with db
                       (map (fn [d]
@@ -959,6 +959,18 @@
     :delete-page (fn [repo conn [page-uuid]]
                    (delete-page! repo conn page-uuid))}))
 
+(defn- <ratelimit-file-writes!
+  []
+  (file/<ratelimit-file-writes!
+   (fn [col]
+     (when (seq col)
+       (let [repo (ffirst col)
+             conn (worker-state/get-datascript-conn repo)]
+         (if conn
+           (when-not (ldb/db-based-graph? @conn)
+             (file/write-files! conn col (worker-state/get-context)))
+           (js/console.error (str "DB is not found for " repo))))))))
+
 (defn init
   "web worker entry"
   []
@@ -967,7 +979,7 @@
   (let [^js obj (DBWorker.)]
     (outliner-register-op-handlers!)
     (worker-state/set-worker-object! obj)
-    (file/<ratelimit-file-writes!)
+    (<ratelimit-file-writes!)
     (js/setInterval #(.postMessage js/self "keepAliveResponse") (* 1000 25))
     (Comlink/expose obj)
     (reset! worker-state/*main-thread (Comlink/wrap js/self))))

src/main/frontend/worker/file.cljs

@@ -17,6 +17,7 @@
             [logseq.common.path :as path]
             [logseq.common.util :as common-util]
             [logseq.db :as ldb]
+            [logseq.db.file-based.entity-util :as file-entity-util]
             [logseq.outliner.tree :as otree]
             [malli.core :as m]))
 
@@ -81,7 +82,7 @@
     (let [format (name (get page-block :block/format (:preferred-format context)))
           date-formatter (:date-formatter context)
           title (string/capitalize (:block/name page-block))
-          whiteboard-page? (ldb/whiteboard? page-block)
+          whiteboard-page? (file-entity-util/whiteboard? page-block)
           format (if whiteboard-page? "edn" format)
           journal-page? (common-date/valid-journal-title? title date-formatter)
           journal-title (common-date/normalize-journal-title title date-formatter)
@@ -112,7 +113,7 @@
         file-db-id (-> page-block :block/file :db/id)
         file-path (-> (d/entity db file-db-id) :file/path)
         result (if (and (string? file-path) (not-empty file-path))
-                 (let [new-content (if (ldb/whiteboard? page-block)
+                 (let [new-content (if (file-entity-util/whiteboard? page-block)
                                      (->
                                       (wfu/ugly-pr-str {:blocks tree
                                                         :pages (list (remove-transit-ids page-block))})
@@ -148,7 +149,7 @@
   [repo conn page-db-id outliner-op context request-id]
   (let [page-block (d/pull @conn '[*] page-db-id)
         page-db-id (:db/id page-block)
-        whiteboard? (ldb/whiteboard? page-block)
+        whiteboard? (file-entity-util/whiteboard? page-block)
         blocks-count (ldb/get-page-blocks-count @conn page-db-id)
         blocks-just-deleted? (and (zero? blocks-count)
                                   (contains? #{:delete-blocks :move-blocks} outliner-op))]
@@ -202,15 +203,7 @@
       (async/put! file-writes-chan [repo page-id (:outliner-op tx-meta) (tc/to-long (t/now)) request-id]))))
 
 (defn <ratelimit-file-writes!
-  []
+  [flush-fn]
   (async-util/<ratelimit file-writes-chan batch-write-interval
-                          :filter-fn (fn [_] true)
-                          :flush-fn
-                          (fn [col]
-                            (when (seq col)
-                              (let [repo (ffirst col)
-                                    conn (worker-state/get-datascript-conn repo)]
-                                (if conn
-                                  (when-not (ldb/db-based-graph? @conn)
-                                    (write-files! conn col (worker-state/get-context)))
-                                  (js/console.error (str "DB is not found for ") repo)))))))
+                         :filter-fn (fn [_] true)
+                         :flush-fn flush-fn))

src/main/frontend/worker/handler/page/db_based/page.cljs

@@ -6,13 +6,13 @@
             [logseq.common.util :as common-util]
             [logseq.common.util.namespace :as ns-util]
             [logseq.db :as ldb]
+            [logseq.db.common.order :as db-order]
+            [logseq.db.common.property-util :as db-property-util]
             [logseq.db.frontend.class :as db-class]
             [logseq.db.frontend.entity-plus :as entity-plus]
             [logseq.db.frontend.entity-util :as entity-util]
             [logseq.db.frontend.malli-schema :as db-malli-schema]
-            [logseq.db.frontend.order :as db-order]
             [logseq.db.frontend.property.build :as db-property-build]
-            [logseq.db.frontend.property.util :as db-property-util]
             [logseq.db.sqlite.util :as sqlite-util]
             [logseq.graph-parser.block :as gp-block]
             [logseq.graph-parser.text :as text]

src/main/frontend/worker/handler/page/file_based/page.cljs

@@ -1,16 +1,16 @@
 (ns frontend.worker.handler.page.file-based.page
   "Page operations for file graphs"
-  (:require [logseq.db :as ldb]
-            [logseq.graph-parser.block :as gp-block]
-            [logseq.graph-parser.property :as gp-property]
-            [logseq.outliner.core :as outliner-core]
+  (:require [clojure.string :as string]
             [datascript.core :as d]
-            [clojure.string :as string]
-            [logseq.graph-parser.text :as text]
-            [logseq.common.util :as common-util]
             [logseq.common.config :as common-config]
             [logseq.common.date :as common-date]
-            [logseq.db.frontend.order :as db-order]))
+            [logseq.common.util :as common-util]
+            [logseq.db :as ldb]
+            [logseq.db.common.order :as db-order]
+            [logseq.graph-parser.block :as gp-block]
+            [logseq.graph-parser.property :as gp-property]
+            [logseq.graph-parser.text :as text]
+            [logseq.outliner.core :as outliner-core]))
 
 (defn- file-based-properties-block
   [repo conn config date-formatter properties format page]

src/main/frontend/worker/handler/page/file_based/rename.cljs

@@ -1,18 +1,18 @@
 (ns frontend.worker.handler.page.file-based.rename
   "File based page rename"
-  (:require [frontend.worker.handler.page :as worker-page]
-            [datascript.core :as d]
-            [clojure.string :as string]
+  (:require [clojure.string :as string]
             [clojure.walk :as walk]
-            [logseq.common.util.page-ref :as page-ref]
+            [datascript.core :as d]
             [frontend.common.file.util :as wfu]
-            [logseq.db :as ldb]
-            [logseq.common.util :as common-util]
+            [frontend.worker.handler.page :as worker-page]
             [logseq.common.config :as common-config]
-            [logseq.graph-parser.text :as text]
+            [logseq.common.util :as common-util]
+            [logseq.common.util.page-ref :as page-ref]
+            [logseq.db :as ldb]
+            [logseq.db.common.order :as db-order]
+            [logseq.db.file-based.entity-util :as file-entity-util]
             [logseq.graph-parser.property :as gp-property]
-            [logseq.db.frontend.order :as db-order]))
-
+            [logseq.graph-parser.text :as text]))
 
 (defn- replace-page-ref-aux
   "Unsanitized names"
@@ -102,10 +102,10 @@
   [refs from-id to-id]
   (if to-id
     (->> refs
-        (remove #{{:db/id from-id}})
-        (cons {:db/id to-id})
-        (distinct)
-        (vec))
+         (remove #{{:db/id from-id}})
+         (cons {:db/id to-id})
+         (distinct)
+         (vec))
     ;; New page not exists so that we keep using the old page's block as a ref
     refs))
 
@@ -205,7 +205,7 @@
   [db old-page-name new-page-name]
   (let [page (d/entity db [:block/name old-page-name])
         file (:block/file page)]
-    (when (and file (not (ldb/journal? page)))
+    (when (and file (not (file-entity-util/journal? page)))
       (let [old-path (:file/path file)
             new-file-name (wfu/file-name-sanity new-page-name) ;; w/o file extension
             new-path (compute-new-file-path old-path new-file-name)]
@@ -317,8 +317,8 @@
       :invalid-empty-name
 
       (and page-e new-page-e
-           (or (ldb/whiteboard? page-e)
-               (ldb/whiteboard? new-page-e)))
+           (or (file-entity-util/whiteboard? page-e)
+               (file-entity-util/whiteboard? new-page-e)))
       :merge-whiteboard-pages
 
       (and old-name new-name name-changed?)

src/main/frontend/worker/rtc/full_upload_download_graph.cljs

@@ -266,7 +266,7 @@
 (defn- create-graph-for-rtc-test
   "it's complex to setup db-worker related stuff, when I only want to test rtc related logic"
   [repo init-tx-data other-tx-data]
-  (let [conn (d/create-conn db-schema/schema-for-db-based-graph)
+  (let [conn (d/create-conn db-schema/schema)
         db-initial-data (sqlite-create-graph/build-db-initial-data "")]
     (swap! worker-state/*datascript-conns assoc repo conn)
     (d/transact! conn db-initial-data {:initial-db? true

src/main/frontend/worker/rtc/remote_update.cljs

@@ -17,8 +17,8 @@
             [logseq.common.defkeywords :refer [defkeywords]]
             [logseq.common.util :as common-util]
             [logseq.db :as ldb]
+            [logseq.db.common.property-util :as db-property-util]
             [logseq.db.frontend.property :as db-property]
-            [logseq.db.frontend.property.util :as db-property-util]
             [logseq.graph-parser.whiteboard :as gp-whiteboard]
             [logseq.outliner.batch-tx :as batch-tx]
             [logseq.outliner.core :as outliner-core]

src/main/frontend/worker/util.cljc

@@ -6,7 +6,7 @@
                      [goog.crypt :as crypt]
                      [goog.crypt.Hmac]
                      [goog.crypt.Sha256]
-                     [logseq.db.sqlite.common-db :as sqlite-common-db]
+                     [logseq.db.common.sqlite :as sqlite-common-db]
                      [frontend.common.file.util :as wfu])))
 
 ;; Copied from https://github.com/tonsky/datascript-todo

src/main/logseq/api.cljs

@@ -49,7 +49,7 @@
             [logseq.common.util :as common-util]
             [logseq.common.util.date-time :as date-time-util]
             [logseq.db :as ldb]
-            [logseq.db.frontend.property.util :as db-property-util]
+            [logseq.db.common.property-util :as db-property-util]
             [logseq.outliner.core :as outliner-core]
             [logseq.sdk.assets :as sdk-assets]
             [logseq.sdk.core]

src/resources/dicts/en.edn

@@ -528,6 +528,7 @@
  :plugin/uninstall "Uninstall"
  :plugin/marketplace "Marketplace"
  :plugin/downloads "Downloads"
+ :plugin/popular "Popular"
  :plugin/stars "Stars"
  :plugin/title "Title ({1})"
  :plugin/all "All"

src/rtc_e2e_test/const.cljs

@@ -1,6 +1,6 @@
 (ns const
   "Consts for rtc e2e tests"
-  (:require [logseq.db.frontend.order :as db-order]))
+  (:require [logseq.db.common.order :as db-order]))
 
 (assert (exists? js/__karma__))
 (def seed js/__karma__.config.seed)

src/rtc_e2e_test/helper.cljs

@@ -10,7 +10,7 @@
             [frontend.worker.rtc.log-and-state :as rtc-log-and-state]
             [frontend.worker.state :as worker-state]
             [logseq.db :as ldb]
-            [logseq.db.frontend.order :as db-order]
+            [logseq.db.common.order :as db-order]
             [logseq.outliner.batch-tx :as batch-tx]
             [meander.epsilon :as me]
             [missionary.core :as m]))

src/test/frontend/test/fixtures.cljs

@@ -1,10 +1,10 @@
 (ns frontend.test.fixtures
   (:require [datascript.core :as d]
-            [logseq.db.frontend.schema :as db-schema]
             [frontend.db.conn :as conn]
             [frontend.db.react :as react]
             [frontend.state :as state]
-            [frontend.test.helper :as test-helper]))
+            [frontend.test.helper :as test-helper]
+            [logseq.db.file-based.schema :as file-schema]))
 
 (defn react-components
   [f]
@@ -16,7 +16,7 @@
 (defn- reset-datascript
   [repo]
   (let [db-name (conn/get-repo-path repo)
-        db-conn (d/create-conn db-schema/schema)]
+        db-conn (d/create-conn file-schema/schema)]
     (state/set-current-repo! repo)
     (swap! conn/conns assoc db-name db-conn)))
 

src/test/frontend/test/helper.cljs

@@ -13,7 +13,7 @@
             [frontend.state :as state]
             [frontend.worker.handler.page :as worker-page]
             [frontend.worker.pipeline :as worker-pipeline]
-            [logseq.db.frontend.order :as db-order]
+            [logseq.db.common.order :as db-order]
             [logseq.db.sqlite.build :as sqlite-build]
             [logseq.db.sqlite.create-graph :as sqlite-create-graph]
             [logseq.db.sqlite.util :as sqlite-util]

src/test/frontend/util/datalog_test.cljs

@@ -1,7 +1,7 @@
 (ns frontend.util.datalog-test
   (:require [cljs.test :refer [deftest is]]
             [frontend.util.datalog :as datalog-util]
-            [logseq.db.frontend.rules :as rules]))
+            [logseq.db.file-based.rules :as file-rules]))
 
 (deftest add-to-end-of-query-in
   (is (= '[:find ?b
@@ -35,4 +35,4 @@
          (datalog-util/find-rules-in-where
           ['(page-property ?b :foo "bar")
            '(page-property ?b :bar "baz")]
-          (-> rules/query-dsl-rules keys set)))))
+          (-> file-rules/query-dsl-rules keys set)))))

src/test/frontend/worker/rtc/client_test.cljs

@@ -6,7 +6,7 @@
    [logseq.db :as ldb]
    [logseq.db.frontend.schema :as db-schema]))
 
-(def empty-db (d/empty-db db-schema/schema-for-db-based-graph))
+(def empty-db (d/empty-db db-schema/schema))
 
 (deftest local-block-ops->remote-ops-test
   (testing "user.class/yyy creation"

src/test/frontend/worker/rtc/remote_update_test.cljs

@@ -8,7 +8,7 @@
 
 (deftest remote-op-value->tx-data-test
   (let [[block-uuid ref-uuid1 ref-uuid2] (repeatedly random-uuid)
-        db (d/db-with (d/empty-db db-schema/schema-for-db-based-graph)
+        db (d/db-with (d/empty-db db-schema/schema)
                       (sqlite-create-graph/build-db-initial-data "{}" {}))]
     (testing ":block/title"
       (let [db (d/db-with db [{:block/uuid block-uuid