Merge pull request #11177 from logseq/refactor/db-properties-schema

DB properties storage refactoring

.clj-kondo/config.edn

@@ -7,7 +7,7 @@
   ;; false positive with match/match and _
   frontend.handler.paste {:linters {:used-underscored-binding {:level :off}}}
   frontend.db {:linters {:aliased-namespace-symbol
-                         {:exclude [frontend.db.conn frontend.db.react logseq.db.frontend.default]}}}}
+                         {:exclude [frontend.db.conn frontend.db.react]}}}}
 
  :linters
  {:path-invalid-construct/string-join {:level :info}
@@ -139,9 +139,12 @@
              logseq.db ldb
              logseq.db.frontend.content db-content
              logseq.db.frontend.inputs db-inputs
+             logseq.db.frontend.order db-order
              logseq.db.frontend.property db-property
+             logseq.db.frontend.property.build db-property-build
              logseq.db.frontend.property.type db-property-type
              logseq.db.frontend.property.util db-property-util
+             logseq.db.frontend.entity-plus entity-plus
              logseq.db.frontend.rules rules
              logseq.db.frontend.schema db-schema
              logseq.db.frontend.validate db-validate

.gitignore

@@ -63,3 +63,4 @@ deps/shui/.lsp
 deps/shui/.lsp-cache
 deps/shui/.clj-kondo
 deps/shui/shui-graph/logseq/bak
+tx-log*

bb.edn

@@ -1,7 +1,7 @@
 {:paths ["scripts/src" "src/main" "src/resources"]
  :deps
  {metosin/malli
-  {:mvn/version "0.10.0"}
+  {:mvn/version "0.16.1"}
   logseq/bb-tasks
   #_{:local/root "../bb-tasks"}
   {:git/url "https://github.com/logseq/bb-tasks"

deps.edn

@@ -4,7 +4,7 @@
   rum/rum                               {:mvn/version "0.12.9"}
 
   datascript/datascript                 {:git/url "https://github.com/logseq/datascript" ;; fork
-                                         :sha     "fc3645fa4a6b8160e087e2ad1ce1c242675680cb"}
+                                         :sha     "aa2f963be5d507b55d05c4d9a6ee3ce9454c4415"}
 
   datascript-transit/datascript-transit {:mvn/version "0.3.0"}
   borkdude/rewrite-edn                  {:mvn/version "0.4.7"}
@@ -16,6 +16,7 @@
   org.clojure/core.match                {:mvn/version "1.0.0"}
   com.andrewmcveigh/cljs-time           {:git/url "https://github.com/logseq/cljs-time" ;; fork
                                          :sha     "5704fbf48d3478eedcf24d458c8964b3c2fd59a9"}
+  ;; TODO: delete cljs-drag-n-drop and use dnd-kit
   cljs-drag-n-drop/cljs-drag-n-drop     {:mvn/version "0.1.0"}
   cljs-http/cljs-http                   {:mvn/version "0.1.46"}
   org.babashka/sci                      {:mvn/version "0.3.2"}
@@ -36,12 +37,12 @@
   logseq/outliner                       {:local/root "deps/outliner"}
   logseq/publishing                     {:local/root "deps/publishing"}
   logseq/shui                           {:local/root "deps/shui"}
-  metosin/malli                         {:mvn/version "0.10.0"}
+  metosin/malli                         {:mvn/version "0.16.1"}
   com.cognitect/transit-cljs            {:mvn/version "0.8.280"}
   missionary/missionary                 {:mvn/version "b.39"}}
 
  :aliases {:cljs {:extra-paths ["src/dev-cljs/" "src/test/" "src/electron/"]
-                  :extra-deps  {org.clojure/clojurescript        {:mvn/version "1.11.54"}
+                  :extra-deps  {org.clojure/clojurescript        {:mvn/version "1.11.132"}
                                 org.clojure/tools.namespace      {:mvn/version "0.2.11"}
                                 cider/cider-nrepl                {:mvn/version "0.47.0"}
                                 org.clojars.knubie/cljs-run-test {:mvn/version "1.0.1"}
@@ -49,7 +50,7 @@
                   :main-opts   ["-m" "shadow.cljs.devtools.cli"]}
 
            :test {:extra-paths ["src/test/"]
-                  :extra-deps  {org.clojure/clojurescript        {:mvn/version "1.11.54"}
+                  :extra-deps  {org.clojure/clojurescript        {:mvn/version "1.11.132"}
                                 org.clojure/test.check           {:mvn/version "1.1.1"}
                                 pjstadig/humane-test-output      {:mvn/version "0.11.0"}
                                 org.clojars.knubie/cljs-run-test {:mvn/version "1.0.1"}

deps/common/.carve/config.edn

@@ -1,5 +1,7 @@
 {:paths ["src"]
  :api-namespaces [logseq.common.path
+                  logseq.common.fractional-index
+                  logseq.common.missionary-util
                   logseq.common.util.page-ref
                   logseq.common.util.block-ref
                   logseq.common.util

deps/common/deps.edn

@@ -5,7 +5,7 @@
  :aliases
  {:test {:extra-paths ["test"]
          :extra-deps  {olical/cljs-test-runner   {:mvn/version "3.8.0"}
-                       org.clojure/clojurescript {:mvn/version "1.11.54"}}
+                       org.clojure/clojurescript {:mvn/version "1.11.132"}}
          :main-opts   ["-m" "cljs-test-runner.main"]}
   :clj-kondo
   {:replace-deps {clj-kondo/clj-kondo {:mvn/version "2023.05.26"}}

deps/common/package.json

@@ -3,7 +3,7 @@
   "version": "1.0.0",
   "private": true,
   "devDependencies": {
-    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v10"
+    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v12"
   },
   "scripts": {
     "test": "yarn nbb-logseq -cp test -m nextjournal.test-runner"

deps/common/resources/templates/config.edn

@@ -215,7 +215,6 @@
             [?h :block/marker ?marker]
             [(contains? #{"NOW" "DOING"} ?marker)]
             [?h :block/page ?p]
-            [?p :block/journal? true]
             [?p :block/journal-day ?d]
             [(>= ?d ?start)]
             [(<= ?d ?today)]]
@@ -232,7 +231,6 @@
             [?h :block/marker ?marker]
             [(contains? #{"NOW" "LATER" "TODO"} ?marker)]
             [?h :block/page ?p]
-            [?p :block/journal? true]
             [?p :block/journal-day ?d]
             [(> ?d ?start)]
             [(< ?d ?next)]]

deps/common/src/logseq/common/fractional_index.cljs

@@ -0,0 +1,163 @@
+(ns logseq.common.fractional-index
+  "Fractional indexing to create an ordering that can be used for Realtime Editing of Ordered Sequences")
+
+;; Original code from https://github.com/rocicorp/fractional-indexing,
+;; It's converted to cljs by using AI.
+
+(def base-62-digits
+  "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz")
+
+(defn char->int
+  [^js c]
+  (.charCodeAt c))
+
+(defn get-integer-length
+  [head]
+  (let [head-char (char->int head)]
+    (cond
+      (and (>= head-char (char->int \a)) (<= head-char (char->int \z)))
+      (+ (- head-char (char->int \a)) 2)
+
+      (and (>= head-char (char->int \A)) (<= head-char (char->int \Z)))
+      (+ (- (char->int \Z) head-char) 2)
+
+      :else
+      (throw (js/Error. (str "invalid order key head: " head))))))
+
+(defn validate-integer
+  [int]
+  (when-not (= (count int) (get-integer-length (first int)))
+    (throw (js/Error. (str "invalid integer part of order key: " int)))))
+
+(defn get-integer-part
+  [key]
+  (let [integer-part-length (get-integer-length (first key))]
+    (when (> integer-part-length (count key))
+      (throw (js/Error. (str "invalid order key: " key))))
+    (subs key 0 integer-part-length)))
+
+(defn validate-order-key
+  [key digits]
+  (when (= key (str "A" (repeat 26 (first digits))))
+    (throw (js/Error. (str "invalid order key: " key))))
+  (let [i (get-integer-part key)
+        f (subs key (count i))]
+    (when (= (last f) (first digits))
+      (throw (js/Error. (str "invalid order key: " key))))))
+
+(defn increment-integer
+  [x digits]
+  (validate-integer x)
+  (let [[head & digs] (seq x)
+        [carry? diff] (reduce
+                        (fn [[carry? digs] dig]
+                          (if carry?
+                            (let [d (inc (.indexOf digits dig))]
+                              (if (= d (count digits))
+                                [true (conj digs (first digits))]
+                                [false (conj digs (nth digits d))]))
+                            [carry? digs]))
+                        [true []]
+                        (reverse digs))
+        digs (into (subvec (vec digs) 0 (- (count digs)
+                                           (count diff)))
+                   (reverse diff))]
+    (if carry?
+      (cond
+        (= head \Z) (str "a" (first digits))
+        (= head \z) nil
+        :else (let [h (char (inc (.charCodeAt head 0)))
+                    digs (if (> (compare h \a) 0)
+                           (conj digs (first digits))
+                           (pop digs))]
+                (str h (apply str digs))))
+      (str head (apply str digs)))))
+
+(defn decrement-integer
+  [x digits]
+  (validate-integer x)
+  (let [[head & digs] (seq x)
+        [borrow digs] (reduce
+                       (fn [[_ digs] dig]
+                         (let [d (dec (.indexOf digits dig))]
+                           (if (= d -1)
+                             [true (conj digs (last digits))]
+                             [false (conj digs (nth digits d))])))
+                       [true []]
+                       (reverse digs))]
+    (if borrow
+      (cond
+        (= head \a) (str "Z" (last digits))
+        (= head \A) nil
+        :else (let [h (char (- (.charCodeAt head 0) 1))
+                    digs (if (< (compare h \Z) 0)
+                           (conj digs (last digits))
+                           (pop digs))]
+                (str h (apply str digs))))
+      (str head (apply str digs)))))
+
+(defn midpoint
+  [a b digits]
+  (let [zero (first digits)]
+    (when (and b (or (>= (compare a b) 0) (= (last a) zero) (= (last b) zero)))
+      (throw (js/Error. (str a " >= " b " or trailing zero"))))
+    (let [n (when b
+              (first (keep-indexed (fn [i _c] (when-not (= (nth a i zero) (nth b i)) i)) b)))]
+      (if (and n (> n 0))
+        (str (subs b 0 n) (midpoint (subs a n) (subs b n) digits))
+        (let [digit-a (if (seq a) (.indexOf digits (first a)) 0)
+              digit-b (if (seq b) (.indexOf digits (first b)) (count digits))]
+          (if (> (- digit-b digit-a) 1)
+            (str (nth digits (Math/round (* 0.5 (+ digit-a digit-b)))))
+            (if (and (seq b) (> (count b) 1))
+              (subs b 0 1)
+              (str (nth digits digit-a) (midpoint (subs a 1) nil digits)))))))))
+
+(defn generate-key-between
+  [a b & {:keys [digits]
+          :or {digits base-62-digits}}]
+  (when a (validate-order-key a digits))
+  (when b (validate-order-key b digits))
+  (when (and a b (>= (compare a b) 0))
+    (throw (js/Error. (str a " >= " b))))
+  (cond
+    (nil? a) (if (nil? b)
+               (str "a" (first digits))
+               (let [ib (get-integer-part b)
+                     fb (subs b (count ib))]
+                 (if (= ib (str "A" (apply str (repeat 26 (first digits)))))
+                   (str ib (midpoint "" fb digits))
+                   (if (< (compare ib b) 0)
+                     ib
+                     (let [res (decrement-integer ib digits)]
+                       (if (nil? res)
+                         (throw (js/Error. "cannot decrement any more"))
+                         res))))))
+    (nil? b) (let [ia (get-integer-part a)
+                   fa (subs a (count ia))
+                   i (increment-integer ia digits)]
+               (if (nil? i)
+                 (str ia (midpoint fa nil digits))
+                 i))
+    :else (let [ia (get-integer-part a)
+                fa (subs a (count ia))
+                ib (get-integer-part b)
+                fb (subs b (count ib))]
+            (if (= ia ib)
+              (str ia (midpoint fa fb digits))
+              (let [i (increment-integer ia digits)]
+                (if (nil? i)
+                  (throw (js/Error. "cannot increment any more"))
+                  (if (< (compare i b) 0) i (str ia (midpoint fa nil digits)))))))))
+
+(defn generate-n-keys-between
+  [a b n & {:keys [digits]
+            :or {digits base-62-digits}}]
+  (cond
+    (= n 0) []
+    (= n 1) [(generate-key-between a b digits)]
+    :else (let [c (generate-key-between a b digits)]
+            (concat
+             (generate-n-keys-between a c (Math/floor (/ n 2)) digits)
+             [c]
+             (generate-n-keys-between c b (- n (Math/floor (/ n 2)) 1) digits)))))

deps/common/test/logseq/common/fractional_index_test.cljs

@@ -0,0 +1,62 @@
+(ns logseq.common.fractional-index-test
+    (:require [clojure.test :refer [deftest are]]
+              [logseq.common.fractional-index :as index]))
+
+(deftest increment-integer-test
+  (are [x y]
+       (= (index/increment-integer x index/base-62-digits) y)
+    "a0" "a1"
+    "r3333333333333333zz" "r333333333333333400"))
+
+(deftest generate-key-between-test
+  (are [x y]
+       (= (index/generate-key-between x nil) y)
+    "a0" "a1"
+    "rzzzzzzzzzzzzzzzzzz" "s0000000000000000000"))
+
+(deftest generate-n-keys-between-test
+  (are [x y]
+       (= (index/generate-n-keys-between (first x) (second x) 20) y)
+    ["ZxV" "Zy7"]
+    ["ZxX"
+     "ZxZ"
+     "Zxd"
+     "Zxf"
+     "Zxh"
+     "Zxl"
+     "Zxn"
+     "Zxp"
+     "Zxt"
+     "Zxx"
+     "Zy"
+     "Zy0V"
+     "Zy1"
+     "Zy2"
+     "Zy3"
+     "Zy4"
+     "Zy4V"
+     "Zy5"
+     "Zy6"
+     "Zy6V"]
+
+    ["Zy7" "axV"]
+    ["ZyB"
+     "ZyE"
+     "ZyL"
+     "ZyP"
+     "ZyS"
+     "ZyZ"
+     "Zyd"
+     "Zyg"
+     "Zyn"
+     "Zyu"
+     "Zz"
+     "Zz8"
+     "ZzG"
+     "ZzV"
+     "Zzl"
+     "a0"
+     "a0G"
+     "a0V"
+     "a1"
+     "a2"]))

deps/common/yarn.lock

@@ -2,9 +2,9 @@
 # yarn lockfile v1
 
 
-"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v10":
-  version "1.2.173-feat-db-v10"
-  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/8a4bfaae55d85292476ac8f976516b86e6ae4036"
+"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v12":
+  version "1.2.173-feat-db-v12"
+  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/8274d8b40b7fec84532c54e16b5a8c6f78bf673b"
   dependencies:
     import-meta-resolve "^2.1.0"
 

deps/db/.carve/config.edn

@@ -5,8 +5,10 @@
                   logseq.db.sqlite.util
                   logseq.db.sqlite.cli
                   logseq.db.frontend.property
+                  logseq.db.frontend.property.build
                   logseq.db.frontend.property.util
                   logseq.db.frontend.content
+                  logseq.db.frontend.order
                   logseq.db.sqlite.create-graph
                   logseq.db.frontend.malli-schema
                   ;; Some fns are used by frontend but not worth moving over yet

deps/db/.carve/ignore

@@ -5,8 +5,6 @@ logseq.db.frontend.rules/db-query-dsl-rules
 ;; API
 logseq.db.frontend.rules/extract-rules
 ;; API
-logseq.db.frontend.property.type/type-or-closed-value?
-;; API
 logseq.db.frontend.property.type/infer-property-type-from-value
 ;; Internal API
 logseq.db.frontend.rules/rules

deps/db/deps.edn

@@ -1,13 +1,14 @@
 {:deps
  ;; External deps should be kept in sync with https://github.com/logseq/nbb-logseq/blob/main/bb.edn
  {datascript/datascript {:git/url "https://github.com/logseq/datascript" ;; fork
-                         :sha     "fc3645fa4a6b8160e087e2ad1ce1c242675680cb"}
+                         :sha     "aa2f963be5d507b55d05c4d9a6ee3ce9454c4415"}
   datascript-transit/datascript-transit {:mvn/version "0.3.0"
                                          :exclusions [datascript/datascript]}
   cljs-bean/cljs-bean         {:mvn/version "1.5.0"}
   com.cognitect/transit-cljs   {:mvn/version "0.8.280"}
   logseq/common                {:local/root "../common"}
-  funcool/promesa              {:mvn/version "4.0.2"}}
+  funcool/promesa              {:mvn/version "4.0.2"}
+  org.flatland/ordered         {:mvn/version "1.15.11"}}
 
  :aliases
  {:clj-kondo

deps/db/nbb.edn

@@ -3,6 +3,6 @@
  {logseq/common
   {:local/root "../common"}
   metosin/malli
-  {:mvn/version "0.10.0"}
+  {:mvn/version "0.16.1"}
   io.github.nextjournal/nbb-test-runner
   {:git/sha "60ed57aa04bca8d604f5ba6b28848bd887109347"}}}

deps/db/package.json

@@ -3,7 +3,7 @@
   "version": "1.0.0",
   "private": true,
   "devDependencies": {
-    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v10"
+    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v12"
   },
   "dependencies": {
     "better-sqlite3": "9.3.0"

deps/db/script/query.cljs

@@ -27,8 +27,17 @@
   (let [[graph-dir query*] args
         [dir db-name] (get-dir-and-db-name graph-dir)
         conn (sqlite-db/open-db! dir db-name)
-        query (into (edn/read-string query*) [:in '$ '%]) ;; assumes no :in are in queries
-        results (mapv first (d/q query @conn (rules/extract-rules rules/db-query-dsl-rules)))]
+        results (if ((set args) "-e")
+                  (map #(when-let [ent (d/entity @conn (edn/read-string %))]
+                          (cond-> (into {:db/id (:db/id ent)} ent)
+                            (seq (:block/properties ent))
+                            (update :block/properties (fn [props] (map (fn [m] (into {} m)) props)))))
+                       (drop 2 args))
+                  ;; assumes no :in are in queries
+                  (let [query (into (edn/read-string query*) [:in '$ '%])
+                        res (d/q query @conn (rules/extract-rules rules/db-query-dsl-rules))]
+                    ;; Remove nesting for most queries which just have one :find binding
+                    (if (= 1 (count (first res))) (mapv first res) res)))]
     (when ((set args) "-v") (println "DB contains" (count (d/datoms @conn :eavt)) "datoms"))
     (prn results)))
 

deps/db/script/validate_client_db.cljs

@@ -4,6 +4,7 @@
   (:require [logseq.db.sqlite.db :as sqlite-db]
             [logseq.db.frontend.malli-schema :as db-malli-schema]
             [logseq.db.frontend.validate :as db-validate]
+            [logseq.db.frontend.property :as db-property]
             [datascript.core :as d]
             [clojure.string :as string]
             [nbb.core :as nbb]
@@ -11,31 +12,42 @@
             [babashka.cli :as cli]
             ["path" :as node-path]
             ["os" :as os]
-            [cljs.pprint :as pprint]))
+            [cljs.pprint :as pprint]
+            [malli.error :as me]))
 
 (defn validate-client-db
   "Validate datascript db as a vec of entity maps"
-  [db ent-maps* {:keys [verbose group-errors closed-maps]}]
-  (let [ent-maps (vec (db-malli-schema/update-properties-in-ents ent-maps*))
+  [db ent-maps* {:keys [verbose group-errors humanize closed-maps]}]
+  (let [ent-maps (db-malli-schema/update-properties-in-ents db ent-maps*)
         schema (db-validate/update-schema db-malli-schema/DB db {:closed-schema? closed-maps})]
-    (if-let [errors (->> ent-maps
+    (if-let [explanation (->> ent-maps
                          (m/explain schema)
-                         :errors)]
+                         not-empty)]
       (do
         (if group-errors
-          (let [ent-errors (db-validate/group-errors-by-entity db ent-maps errors)]
+          (let [ent-errors (db-validate/group-errors-by-entity db ent-maps (:errors explanation))]
             (println "Found" (count ent-errors) "entities in errors:")
-            (if verbose
+            (cond
+              verbose
               (pprint/pprint ent-errors)
+              humanize
+              (pprint/pprint (map #(-> (dissoc % :errors-by-type)
+                                       (update :errors (fn [errs] (me/humanize {:errors errs}))))
+                                  ent-errors))
+              :else
               (pprint/pprint (map :entity ent-errors))))
-          (do
+          (let [errors (:errors explanation)]
             (println "Found" (count errors) "errors:")
-            (if verbose
+            (cond
+              verbose
               (pprint/pprint
                (map #(assoc %
                             :entity (get ent-maps (-> % :in first))
                             :schema (m/form (:schema %)))
                     errors))
+              humanize
+              (pprint/pprint (me/humanize {:errors errors}))
+              :else
               (pprint/pprint errors))))
         (js/process.exit 1))
       (println "Valid!"))))
@@ -44,6 +56,8 @@
   "Options spec"
   {:help {:alias :h
           :desc "Print help"}
+   :humanize {:alias :H
+              :desc "Humanize errors as an alternative to -v"}
    :verbose {:alias :v
              :desc "Print more info"}
    :closed-maps {:alias :c
@@ -62,7 +76,7 @@
                     (println "Error: For graph" (str (pr-str graph-dir) ":") (str e))
                     (js/process.exit 1)))
         datoms (d/datoms @conn :eavt)
-        ent-maps (vals (db-malli-schema/datoms->entity-maps datoms))]
+        ent-maps (db-malli-schema/datoms->entities datoms)]
     (println "Read graph" (str db-name " with " (count datoms) " datoms, "
                                (count ent-maps) " entities, "
                                (count (filter :block/name ent-maps)) " pages, "
@@ -70,7 +84,7 @@
                                (count (filter #(contains? (:block/type %) "class") ent-maps)) " classes, "
                                (count (filter #(seq (:block/tags %)) ent-maps)) " objects, "
                                (count (filter #(contains? (:block/type %) "property") ent-maps)) " properties and "
-                               (count (mapcat :block/properties ent-maps)) " property values"))
+                               (count (mapcat db-property/properties ent-maps)) " property pairs"))
     (validate-client-db @conn ent-maps options)))
 
 (defn -main [argv]

deps/db/src/logseq/db.cljs

@@ -6,18 +6,19 @@
             [logseq.common.util :as common-util]
             [logseq.common.config :as common-config]
             [logseq.db.frontend.content :as db-content]
-            [clojure.set :as set]
             [logseq.db.frontend.rules :as rules]
-            [logseq.db.frontend.entity-plus]
+            [logseq.db.frontend.entity-plus :as entity-plus]
             [logseq.db.sqlite.util :as sqlite-util]
-            [logseq.db.frontend.delete-blocks :as delete-blocks]))
+            [logseq.db.sqlite.common-db :as sqlite-common-db]
+            [logseq.db.frontend.delete-blocks :as delete-blocks]
+            [datascript.impl.entity :as de]))
 
 ;; Use it as an input argument for datalog queries
 (def block-attrs
   '[:db/id
     :block/uuid
     :block/parent
-    :block/left
+    :block/order
     :block/collapsed?
     :block/collapsed-properties
     :block/format
@@ -77,23 +78,9 @@
        (let [f (or @*transact-fn d/transact!)]
          (f repo-or-conn tx-data tx-meta))))))
 
-(defn sort-by-left
-  [blocks parent]
-  (let [left->blocks (->> (reduce (fn [acc b] (assoc! acc (:db/id (:block/left b)) b))
-                                  (transient {}) blocks)
-                          (persistent!))]
-    (loop [block parent
-           result (transient [])]
-      (if-let [next (get left->blocks (:db/id block))]
-        (recur next (conj! result next))
-        (vec (persistent! result))))))
-
-(defn try-sort-by-left
-  [blocks parent]
-  (let [result' (sort-by-left blocks parent)]
-    (if (= (count result') (count blocks))
-      result'
-      blocks)))
+(defn sort-by-order
+  [blocks]
+  (sort-by :block/order blocks))
 
 ;; TODO: use the tree directly
 (defn flatten-tree
@@ -118,73 +105,70 @@
           (->> (map #(db-content/update-block-content repo db % (:db/id %))))))
 
 (defn whiteboard-page?
-  "Given a page name or a page object, check if it is a whiteboard page"
-  [db page]
-  (cond
-    (string? page)
-    (let [page (d/entity db [:block/name (common-util/page-name-sanity-lc page)])]
-      (or
-       (contains? (set (:block/type page)) "whiteboard")
-       (when-let [file (:block/file page)]
-         (when-let [path (:file/path (d/entity db (:db/id file)))]
-           (common-config/whiteboard? path)))))
-
-    (seq page)
-    (contains? (set (:block/type page)) "whiteboard")
-
-    :else false))
+  "Given a page entity or map, check if it is a whiteboard page"
+  [page]
+  (contains? (set (:block/type page)) "whiteboard"))
+
+(defn journal-page?
+  "Given a page entity or map, check if it is a journal page"
+  [page]
+  (contains? (set (:block/type page)) "journal"))
 
 (defn get-page-blocks
   "Return blocks of the designated page, without using cache.
-   page - name / title of the page"
-  [db page {:keys [pull-keys]
-            :or {pull-keys '[*]}}]
-  (when page
-    (let [page (common-util/page-name-sanity-lc page)
-          page-id (:db/id (d/entity db [:block/name page]))]
-      (when page-id
-        (let [datoms (d/datoms db :avet :block/page page-id)
-              block-eids (mapv :e datoms)]
-          (d/pull-many db pull-keys block-eids))))))
-
-(defn get-page-blocks-by-uuid
-  [db page-uuid & {:keys [pull-keys]
-                   :or {pull-keys '[*]}}]
-  (when-let [page-id (and page-uuid (:db/id (d/entity db [:block/uuid page-uuid])))]
+   page-id - eid"
+  [db page-id {:keys [pull-keys]
+               :or {pull-keys '[*]}}]
+  (when page-id
     (let [datoms (d/datoms db :avet :block/page page-id)
           block-eids (mapv :e datoms)]
       (d/pull-many db pull-keys block-eids))))
 
-
 (defn get-page-blocks-count
   [db page-id]
   (count (d/datoms db :avet :block/page page-id)))
 
-(defn get-by-parent-&-left
-  [db parent-id left-id]
-  (when (and parent-id left-id)
-    (let [lefts (:block/_left (d/entity db left-id))]
-      (some (fn [node] (when (and (= parent-id (:db/id (:block/parent node)))
-                                  (not= parent-id (:db/id node)))
-                         node)) lefts))))
-
 (defn get-right-sibling
-  [db db-id]
-  (when-let [block (d/entity db db-id)]
-    (get-by-parent-&-left db
-                          (:db/id (:block/parent block))
-                          db-id)))
+  [block]
+  (assert (or (de/entity? block) (nil? block)))
+  (when-let [parent (:block/parent block)]
+    (let [from-property (:logseq.property/created-from-property block)
+          children (sort-by-order (if from-property
+                                    (filter (fn [e]
+                                              (= (:db/id (:logseq.property/created-from-property e))
+                                                 (:db/id from-property)))
+                                            (:block/_raw-parent parent))
+                                    (:block/_parent parent)))
+          right (some (fn [child] (when (> (compare (:block/order child) (:block/order block)) 0) child)) children)]
+      (when (not= (:db/id right) (:db/id block))
+        right))))
+
+(defn get-left-sibling
+  [block]
+  (assert (or (de/entity? block) (nil? block)))
+  (when-let [parent (:block/parent block)]
+    (let [from-property (:logseq.property/created-from-property block)
+          children (reverse (sort-by-order (if from-property
+                                             (filter (fn [e]
+                                                       (= (:db/id (:logseq.property/created-from-property e))
+                                                          (:db/id from-property)))
+                                                     (:block/_raw-parent parent))
+                                             (:block/_parent parent))))
+          left (some (fn [child] (when (< (compare (:block/order child) (:block/order block)) 0) child)) children)]
+      (when (not= (:db/id left) (:db/id block))
+        left))))
+
+(defn get-down
+  [block]
+  (assert (or (de/entity? block) (nil? block)))
+  (first (sort-by-order (:block/_parent block))))
 
-(defn get-by-id
-  [db id]
-  (d/pull db '[*] id))
 
 (defn hidden-page?
   [page]
   (when page
     (if (string? page)
-      (or (and (string/starts-with? page "$$$")
-               (common-util/uuid-string? (common-util/safe-subs page 3)))
+      (or (string/starts-with? page "$$$")
           (= common-config/favorites-page-name page))
       (contains? (set (:block/type page)) "hidden"))))
 
@@ -195,22 +179,42 @@
           :where
           [?page :block/name ?page-name]
           [(get-else $ ?page :block/original-name ?page-name) ?page-original-name]]
-         db)
+        db)
        (map first)
        (remove hidden-page?)))
 
+(def get-first-page-by-name sqlite-common-db/get-first-page-by-name)
+
 (defn page-exists?
   "Whether a page exists."
   [db page-name]
   (when page-name
-    (d/entity db [:block/name (common-util/page-name-sanity-lc page-name)])))
+    (some? (get-first-page-by-name db page-name))))
+
+(defn get-page
+  "Get a page given its unsanitized name"
+  [db page-name-or-uuid]
+  (when db
+    (if-let [id (if (uuid? page-name-or-uuid) page-name-or-uuid
+                    (parse-uuid page-name-or-uuid))]
+      (d/entity db [:block/uuid id])
+      (d/entity db (get-first-page-by-name db (name page-name-or-uuid))))))
+
+(defn get-case-page
+  "Case sensitive version of get-page. For use with DB graphs"
+  [db page-name-or-uuid]
+  (when db
+    (if-let [id (if (uuid? page-name-or-uuid) page-name-or-uuid
+                    (parse-uuid page-name-or-uuid))]
+      (d/entity db [:block/uuid id])
+      (d/entity db (sqlite-common-db/get-first-page-by-original-name db page-name-or-uuid)))))
 
 (defn page-empty?
   "Whether a page is empty. Does it has a non-page block?
   `page-id` could be either a string or a db/id."
   [db page-id]
   (let [page-id (if (string? page-id)
-                  [:block/name (common-util/page-name-sanity-lc page-id)]
+                  (get-first-page-by-name db page-id)
                   page-id)
         page (d/entity db page-id)]
     (nil? (:block/_left page))))
@@ -225,8 +229,8 @@
         orphaned-pages (->>
                         (map
                          (fn [page]
-                           (let [name (common-util/page-name-sanity-lc page)]
-                             (when-let [page (d/entity db [:block/name name])]
+                           (when-let [page (get-page db page)]
+                             (let [name (:block/name page)]
                                (and
                                 (empty-ref-f page)
                                 (or
@@ -238,12 +242,12 @@
                                     (= 1 (count children))
                                     (contains? #{"" "-" "*"} (string/trim (:block/content first-child))))))
                                 (not (contains? built-in-pages name))
-                                (not (whiteboard-page? db page))
+                                (not (whiteboard-page? page))
                                 (not (:block/_namespace page))
                                 (not (contains? (:block/type page) "property"))
                                  ;; a/b/c might be deleted but a/b/c/d still exists (for backward compatibility)
                                 (not (and (string/includes? name "/")
-                                          (not (:block/journal? page))))
+                                          (not (journal-page? page))))
                                 page))))
                          pages)
                         (remove false?)
@@ -268,16 +272,22 @@
      (when (if not-collapsed?
              (not (collapsed-and-has-children? db block))
              true)
-       (let [children (:block/_parent block)
-             all-left (set (concat (map (comp :db/id :block/left) children) [db-id]))
-             all-ids (set (map :db/id children))]
-         (first (set/difference all-ids all-left)))))))
+       (let [children (sort-by :block/order (:block/_parent block))]
+         (:db/id (last children)))))))
 
-(defn get-block-immediate-children
+(defn get-children
   "Doesn't include nested children."
-  [db block-uuid]
-  (when-let [parent (d/entity db [:block/uuid block-uuid])]
-    (sort-by-left (:block/_parent parent) parent)))
+  ([block-entity]
+   (get-children nil block-entity))
+  ([db block-entity-or-eid]
+   (when-let [parent (cond
+                       (number? block-entity-or-eid)
+                       (d/entity db block-entity-or-eid)
+                       (uuid? block-entity-or-eid)
+                       (d/entity db [:block/uuid block-entity-or-eid])
+                       :else
+                       block-entity-or-eid)]
+     (sort-by-order (:block/_parent parent)))))
 
 (defn get-block-parents
   [db block-id {:keys [depth] :or {depth 100}}]
@@ -290,43 +300,23 @@
         (recur (:block/uuid parent) (conj parents parent) (inc d))
         parents))))
 
-(defn get-block-children-ids
-  "Returns children UUIDs"
-  [db block-uuid]
-  (when-let [eid (:db/id (d/entity db [:block/uuid block-uuid]))]
-    (let [seen   (volatile! [])]
-      (loop [steps          100      ;check result every 100 steps
-             eids-to-expand [eid]]
-        (when (seq eids-to-expand)
-          (let [eids-to-expand*
-                (mapcat (fn [eid] (map first (d/datoms db :avet :block/parent eid))) eids-to-expand)
-                uuids-to-add (remove nil? (map #(:block/uuid (d/entity db %)) eids-to-expand*))]
-            (when (and (zero? steps)
-                       (seq (set/intersection (set @seen) (set uuids-to-add))))
-              (throw (ex-info "bad outliner data, need to re-index to fix"
-                              {:seen @seen :eids-to-expand eids-to-expand})))
-            (vswap! seen (partial apply conj) uuids-to-add)
-            (recur (if (zero? steps) 100 (dec steps)) eids-to-expand*))))
-      @seen)))
-
-(defn get-block-children
-  "Including nested children."
-  [db block-uuid]
-  (let [ids (get-block-children-ids db block-uuid)]
-    (when (seq ids)
-      (let [ids' (map (fn [id] [:block/uuid id]) ids)]
-        (d/pull-many db '[*] ids')))))
+(def get-block-children-ids sqlite-common-db/get-block-children-ids)
+(def get-block-children sqlite-common-db/get-block-children)
+
+(defn get-first-child
+  [db id]
+  (first (sort-by-order (:block/_parent (d/entity db id)))))
 
 (defn- get-sorted-page-block-ids
   [db page-id]
   (let [root (d/entity db page-id)]
     (loop [result []
-           children (sort-by-left (:block/_parent root) root)]
+           children (sort-by-order (:block/_parent root))]
       (if (seq children)
         (let [child (first children)]
           (recur (conj result (:db/id child))
                  (concat
-                  (sort-by-left (:block/_parent child) child)
+                  (sort-by-order (:block/_parent child))
                   (rest children))))
         result))))
 
@@ -340,13 +330,6 @@
         blocks-map (zipmap (map :db/id blocks) blocks)]
     (keep blocks-map sorted-ids)))
 
-(defn get-prev-sibling
-  [db id]
-  (when-let [e (d/entity db id)]
-    (let [left (:block/left e)]
-      (when (not= (:db/id left) (:db/id (:block/parent e)))
-        left))))
-
 (defn last-child-block?
   "The child block could be collapsed."
   [db parent-id child-id]
@@ -355,7 +338,7 @@
       (= parent-id child-id)
       true
 
-      (get-right-sibling db child-id)
+      (get-right-sibling child)
       false
 
       :else
@@ -367,8 +350,8 @@
                  (and (= (:block/page block-1) (:block/page block-2))
                       (or
                        ;; sibling or child
-                       (= (:db/id (:block/left block-2)) (:db/id block-1))
-                       (when-let [prev-sibling (get-prev-sibling db (:db/id block-2))]
+                       (= (:db/id (get-left-sibling block-2)) (:db/id block-1))
+                       (when-let [prev-sibling (get-left-sibling (d/entity db (:db/id block-2)))]
                          (last-child-block? db (:db/id prev-sibling) (:db/id block-1))))))]
     (or (aux-fn block-1 block-2) (aux-fn block-2 block-1))))
 
@@ -398,17 +381,8 @@
 
 (defn get-classes-with-property
   "Get classes which have given property as a class property"
-  [db property-uuid]
-  (d/q
-   '[:find [?b ...]
-     :in $ ?property-uuid
-     :where
-     [?b :block/schema ?schema]
-     [(get ?schema :properties) ?schema-properties*]
-     [(set ?schema-properties*) ?schema-properties]
-     [(contains? ?schema-properties ?property-uuid)]]
-   db
-   property-uuid))
+  [db property-id]
+  (:class/_schema.properties (d/entity db property-id)))
 
 (defn get-block-property-values
   "Get blocks which have this property."
@@ -425,51 +399,11 @@
 
 (defn get-alias-source-page
   "return the source page (page-name) of an alias"
-  [db alias]
-  (let [alias (common-util/page-name-sanity-lc alias)
-        pages (d/q '[:find [(pull ?p [*]) ...]
-                     :in $ ?alias
-                     :where
-                     [?a :block/name ?alias]
-                     [?p :block/alias ?a]]
-                   db
-                   alias)]
-    ;; may be a case that a user added same alias into multiple pages.
+  [db alias-id]
+  (when alias-id
+      ;; may be a case that a user added same alias into multiple pages.
       ;; only return the first result for idiot-proof
-    (when (seq pages)
-      (first pages))))
-
-(defn get-namespace-pages
-  "Accepts both sanitized and unsanitized namespaces"
-  [db namespace {:keys [db-graph?]}]
-  (assert (string? namespace))
-  (let [namespace (common-util/page-name-sanity-lc namespace)
-        pull-attrs  (cond-> [:db/id :block/name :block/original-name :block/namespace]
-                      (not db-graph?)
-                      (conj {:block/file [:db/id :file/path]}))]
-    (d/q
-     [:find [(list 'pull '?c pull-attrs) '...]
-      :in '$ '% '?namespace
-      :where
-      ['?p :block/name '?namespace]
-      (list 'namespace '?p '?c)]
-     db
-     (:namespace rules/rules)
-     namespace)))
-
-(defn get-pages-by-name-partition
-  [db partition]
-  (when-not (string/blank? partition)
-    (let [partition (common-util/page-name-sanity-lc (string/trim partition))
-          ids (->> (d/datoms db :aevt :block/name)
-                   (filter (fn [datom]
-                             (let [page (:v datom)]
-                               (string/includes? page partition))))
-                   (map :e))]
-      (when (seq ids)
-        (d/pull-many db
-                     '[:db/id :block/name :block/original-name]
-                     ids)))))
+    (first (:block/_alias (d/entity db alias-id)))))
 
 (defn get-page-alias
   [db page-id]
@@ -527,16 +461,17 @@
 
 (defn built-in?
   "Built-in page or block"
-  [db entity]
-  (get (:block/properties entity) (:block/uuid (d/entity db :logseq.property/built-in?))))
+  [entity]
+  (:logseq.property/built-in? entity))
 
 (defn built-in-class-property?
   "Whether property a built-in property for the specific class"
-  [db class-entity property-entity]
-  (and (built-in? db class-entity)
+  [class-entity property-entity]
+  (and (built-in? class-entity)
        (contains? (:block/type class-entity) "class")
-       (built-in? db property-entity)
-       (contains? (set (get-in class-entity [:block/schema :properties])) (:block/uuid property-entity))))
+       (built-in? property-entity)
+       (contains? (set (map :db/ident (:class/schema.properties class-entity)))
+                  (:db/ident property-entity))))
 
 (def write-transit-str sqlite-util/write-transit-str)
 (def read-transit-str sqlite-util/read-transit-str)
@@ -550,7 +485,6 @@
      {:block/uuid (d/squuid)
       :block/name common-config/favorites-page-name
       :block/original-name common-config/favorites-page-name
-      :block/journal? false
       :block/type #{"hidden"}
       :block/format :markdown})]))
 
@@ -565,14 +499,38 @@
   [db]
   (when db (:graph/uuid (d/entity db :logseq.kv/graph-uuid))))
 
-(defn page?
-  "Whether `block` is a page"
-  [block]
-  (and (:block/name block)
-       (not (:block/page block))))
-
-(comment
-  (defn db-based-graph?
-    "Whether the current graph is db-only"
-    [db]
-    (= "db" (:db/type (d/entity db :logseq.kv.db/type)))))
+(def page? sqlite-util/page?)
+(def db-based-graph? entity-plus/db-based-graph?)
+
+;; File based fns
+(defn get-namespace-pages
+  "Accepts both sanitized and unsanitized namespaces"
+  [db namespace {:keys [db-graph?]}]
+  (assert (string? namespace))
+  (let [namespace (common-util/page-name-sanity-lc namespace)
+        pull-attrs  (cond-> [:db/id :block/name :block/original-name :block/namespace]
+                      (not db-graph?)
+                      (conj {:block/file [:db/id :file/path]}))]
+    (d/q
+     [:find [(list 'pull '?c pull-attrs) '...]
+      :in '$ '% '?namespace
+      :where
+      ['?p :block/name '?namespace]
+      (list 'namespace '?p '?c)]
+     db
+     (:namespace rules/rules)
+     namespace)))
+
+(defn get-pages-by-name-partition
+  [db partition]
+  (when-not (string/blank? partition)
+    (let [partition (common-util/page-name-sanity-lc (string/trim partition))
+          ids (->> (d/datoms db :aevt :block/name)
+                   (filter (fn [datom]
+                             (let [page (:v datom)]
+                               (string/includes? page partition))))
+                   (map :e))]
+      (when (seq ids)
+        (d/pull-many db
+                     '[:db/id :block/name :block/original-name]
+                     ids)))))

deps/db/src/logseq/db/frontend/default.cljs

@@ -1,9 +0,0 @@
-(ns logseq.db.frontend.default
-  "Provides vars and fns for dealing with default/built-in? data"
-  (:require [datascript.core :as d]))
-
-(defn mark-block-as-built-in
-  "Marks built-in blocks as built-in? including pages, classes, properties and closed values"
-  [db block]
-  (let [built-in-property-id (:block/uuid (d/entity db :logseq.property/built-in?))]
-    (update block :block/properties assoc built-in-property-id true)))

deps/db/src/logseq/db/frontend/entity_plus.cljc

@@ -8,17 +8,33 @@
   (:require [cljs.core]
             #?(:org.babashka/nbb [datascript.db])
             [datascript.impl.entity :as entity :refer [Entity]]
-            [logseq.db.frontend.content :as db-content]))
+            [logseq.db.frontend.content :as db-content]
+            [datascript.core :as d]
+            [logseq.db.frontend.property :as db-property]))
+
+(defn db-based-graph?
+  "Whether the current graph is db-only"
+  [db]
+  (= "db" (:db/type (d/entity db :logseq.kv/db-type))))
 
 (def lookup-entity @#'entity/lookup-entity)
 (defn lookup-kv-then-entity
   ([e k] (lookup-kv-then-entity e k nil))
   ([^Entity e k default-value]
-   (cond
-     (= k :block/raw-content)
+   (case k
+     :block/raw-content
      (lookup-entity e :block/content default-value)
 
-     (= k :block/content)
+     :block/properties
+     (let [db (.-db e)]
+       (if (db-based-graph? db)
+         (lookup-entity e :block/properties
+                        (->> (into {} e)
+                             (filter (fn [[k _]] (db-property/property? k)))
+                             (into {})))
+         (lookup-entity e :block/properties nil)))
+
+     :block/content
      (or
       (get (.-kv e) k)
       (let [result (lookup-entity e k default-value)
@@ -29,7 +45,14 @@
            (db-content/special-id-ref->page-ref result (distinct (concat refs tags))))
          default-value)))
 
-     :else
+     :block/_parent
+     (->> (lookup-entity e k default-value)
+          (remove (fn [e] (:logseq.property/created-from-property e)))
+          seq)
+
+     :block/_raw-parent
+     (lookup-entity e :block/_parent default-value)
+
      (or (get (.-kv e) k)
          (lookup-entity e k default-value)))))
 

deps/db/src/logseq/db/frontend/malli_schema.cljs

@@ -2,73 +2,180 @@
   "Malli schemas and fns for logseq.db.frontend.*"
   (:require [clojure.walk :as walk]
             [clojure.string :as string]
-            [datascript.core :as d]
             [logseq.db.frontend.schema :as db-schema]
-            [logseq.db.frontend.property.type :as db-property-type]))
+            [logseq.db.frontend.property.type :as db-property-type]
+            [datascript.core :as d]
+            [logseq.db.frontend.property :as db-property]
+            [logseq.db.sqlite.util :as sqlite-util]))
+
+;; :db/ident malli schemas
+;; =======================
+
+(def db-attribute-ident
+  (into [:enum] db-property/db-attribute-properties))
+
+(def logseq-property-ident
+  [:and :keyword [:fn
+                  {:error/message "should be a valid logseq property namespace"}
+                  db-property/logseq-property?]])
+
+(def internal-property-ident
+  [:or logseq-property-ident db-attribute-ident])
+
+(defn- user-property?
+  "Determines if keyword/ident is a user property"
+  [kw]
+  (db-property/user-property-namespace? (namespace kw)))
+
+(def user-property-ident
+  [:and :qualified-keyword [:fn
+                            {:error/message "should be a valid user property namespace"}
+                            user-property?]])
+
+(def property-ident
+  [:or internal-property-ident user-property-ident])
+
+(def logseq-ident-namespaces
+  "Set of all namespaces Logseq uses for :db/ident except for
+  db-attribute-ident. It's important to grow this list purposefully and have it
+  start with 'logseq' to allow for users and 3rd party plugins to provide their
+  own namespaces to core concepts."
+  (into db-property/logseq-property-namespaces #{"logseq.class" "logseq.kv"}))
+
+(def logseq-ident
+  [:and :keyword [:fn
+                  {:error/message "should be a valid :db/ident namespace"}
+                  (fn logseq-namespace? [k]
+                    (contains? logseq-ident-namespaces (namespace k)))]])
+
+(defn- class?
+  "Determines if keyword/ident is a logseq or user class"
+  [kw]
+  (string/includes? (namespace kw) ".class"))
 
+(def class-ident
+  [:and :qualified-keyword [:fn
+                            {:error/message "should be a valid class namespace"}
+                            class?]])
 ;; Helper fns
 ;; ==========
-(defn- validate-property-value
-  "Validates the value in a property tuple. The property value can be one or
-  many of a value to validated"
-  [prop-type schema-fn val]
-  (if (and (or (sequential? val) (set? val))
-           (not= :coll prop-type))
-    (every? schema-fn val)
-    (schema-fn val)))
+(defn validate-property-value
+  "Validates the property value in a property tuple. The property value can be
+  one or many of a value to validated. validate-fn is a fn that is called
+  directly on each value to return a truthy value. validate-fn varies by
+  property type"
+  [db validate-fn [{:block/keys [schema] :as property} property-val] & {:keys [new-closed-value?]}]
+  ;; For debugging
+  ;; (when (not= "logseq.property" (namespace (:db/ident property))) (prn :validate-val property property-val))
+  (let [validate-fn' (if (db-property-type/property-types-with-db (:type schema)) (partial validate-fn db) validate-fn)
+        validate-fn'' (if (and (db-property-type/closed-value-property-types (:type schema))
+                               ;; new closed values aren't associated with the property yet
+                               (not new-closed-value?)
+                               (seq (:values schema)))
+                        (fn closed-value-valid? [val]
+                          (and (validate-fn' val)
+                               (contains? (set (:values schema))
+                                          (:block/uuid (d/entity db val)))))
+                        validate-fn')]
+    (if (= (:cardinality schema) :many)
+      (every? validate-fn'' property-val)
+      (or (validate-fn'' property-val)
+          (if (= :db.type/ref (:db/valueType property))
+            (and (integer? property-val)
+                 (= :logseq.property/empty-placeholder (:db/ident (d/entity db property-val))))
+            (= :logseq.property/empty-placeholder property-val))))))
 
 (defn update-properties-in-schema
   "Needs to be called on the DB schema to add the datascript db to it"
   [db-schema db]
   (walk/postwalk (fn [e]
                    (let [meta' (meta e)]
-                     (cond
-                       (:add-db meta')
-                       (partial e db)
-                       (:property-value meta')
-                       (let [[property-type schema-fn] e
-                             schema-fn' (if (db-property-type/property-types-with-db property-type) (partial schema-fn db) schema-fn)
-                             validation-fn #(validate-property-value property-type schema-fn' %)]
-                         [property-type [:tuple :uuid [:fn validation-fn]]])
-                       :else
-                       e)))
+                     (if (:add-db meta') (partial e db) e)))
                  db-schema))
 
 (defn update-properties-in-ents
-  "Prepares entities to be validated by DB schema"
-  [ents]
-  (map #(if (:block/properties %)
-          (update % :block/properties (fn [x] (mapv identity x)))
-          %)
-       ents))
+  "Prepares properties in entities to be validated by DB schema"
+  [db ents]
+  (mapv
+   (fn [ent]
+     (reduce (fn [m [k v]]
+               (if-let [property (and (db-property/property? k)
+                                      (d/entity db k))]
+                 (update m :block/properties (fnil conj [])
+                         [(assoc (select-keys property [:db/ident :db/valueType])
+                                 :block/schema
+                                 (select-keys (:block/schema property) [:type :cardinality :values]))
+                          v])
+                 (assoc m k v)))
+             {}
+             ent))
+   ents))
 
 (defn datoms->entity-maps
-  "Returns entity maps for given :eavt datoms"
+  "Returns entity maps for given :eavt datoms indexed by db/id. Optional keys:
+   * :entity-fn - Optional fn that given an entity id, returns entity. Defaults
+     to just doing a lookup based on existing entity-maps to be as performant as possible"
+  [datoms & {:keys [entity-fn]}]
+  (let [ent-maps
+        (reduce (fn [acc {:keys [a e v]}]
+                  (if (contains? db-schema/card-many-attributes a)
+                    (update acc e update a (fnil conj #{}) v)
+                    ;; If there's already a val, don't clobber it and automatically start collecting it as a :many
+                    (if-let [existing-val (get-in acc [e a])]
+                      (if (set? existing-val)
+                        (update acc e assoc a (conj existing-val v))
+                        (update acc e assoc a #{existing-val v}))
+                      (update acc e assoc a v))))
+                {}
+                datoms)
+        entity-fn' (or entity-fn
+                       (let [db-ident-maps (dissoc (into {} (map (juxt :db/ident identity) (vals ent-maps))) nil)]
+                         #(get db-ident-maps %)))]
+    (-> ent-maps
+        (update-vals
+         (fn [m]
+           (->> m
+                (map (fn [[k v]]
+                       (if-let [property (and (db-property/property? k)
+                                              (entity-fn' k))]
+                         (if (and (= :db.cardinality/many (:db/cardinality property))
+                                  (not (set? v)))
+                           ;; Fix :many property values that only had one value
+                           [k #{v}]
+                           [k v])
+                         [k v])))
+                (into {})))))))
+
+(defn datoms->entities
+  "Returns a vec of entity maps given :eavt datoms"
   [datoms]
-  (->> datoms
-       (reduce (fn [acc m]
-                 (if (contains? db-schema/card-many-attributes (:a m))
-                   (update acc (:e m) update (:a m) (fnil conj #{}) (:v m))
-                   (update acc (:e m) assoc (:a m) (:v m))))
-               {})))
-
-;; Malli schemas
-;; =============
+  (mapv (fn [[db-id m]] (with-meta m {:db/id db-id}))
+        (datoms->entity-maps datoms)))
+
+(defn internal-ident?
+  "Determines if given ident is created by Logseq"
+  [ident]
+  (or (contains? db-property/db-attribute-properties ident)
+      (contains? logseq-ident-namespaces (namespace ident))))
+
+;; Main malli schemas
+;; ==================
 ;; These schemas should be data vars to remain as simple and reusable as possible
+
 (def property-tuple
-  "Represents a tuple of a property and its property value. This schema
-   has 2 metadata hooks which are used to inject a datascript db later"
+  "A tuple of a property map and a property value. This schema
+   has 1 metadata hook which is used to inject a datascript db later"
   (into
-   [:multi {:dispatch ^:add-db (fn [db property-tuple]
-                                 (get-in (d/entity db [:block/uuid (first property-tuple)])
-                                         [:block/schema :type]))}]
+   [:multi {:dispatch #(-> % first :block/schema :type)}]
    (map (fn [[prop-type value-schema]]
-          ^:property-value [prop-type (if (vector? value-schema) (last value-schema) value-schema)])
+          [prop-type
+           (let [schema-fn (if (vector? value-schema) (last value-schema) value-schema)]
+             [:fn (with-meta (fn [db tuple] (validate-property-value db schema-fn tuple)) {:add-db true})])])
         db-property-type/built-in-validation-schemas)))
 
 (def block-properties
-  "Validates a slightly modified version of :block/properties. Properties are
-  expected to be a vector of tuples instead of a map in order to validate each
+  "Validates a block's properties as property pairs. Properties are
+  a vector of tuples instead of a map in order to validate each
   property with its property value that is valid for its type"
   [:sequential property-tuple])
 
@@ -78,6 +185,7 @@
    [:block/created-at :int]
    [:block/updated-at :int]
    [:block/format [:enum :markdown]]
+   ;; Injected by update-properties-in-ents
    [:block/properties {:optional true} block-properties]
    [:block/refs {:optional true} [:set :int]]
    [:block/tags {:optional true} [:set :int]]
@@ -88,12 +196,20 @@
   "Common attributes for pages"
   [[:block/name :string]
    [:block/original-name :string]
-   [:block/type {:optional true} [:enum #{"property"} #{"class"} #{"whiteboard"} #{"hidden"}]]
-   [:block/journal? :boolean]
-   [:block/namespace {:optional true} :int]
+   ;; FIXME: a class can also be a property?
+   [:block/type {:optional true} [:enum #{"class"} #{"property"} #{"whiteboard"} #{"journal"} #{"hidden"}]]
    [:block/alias {:optional true} [:set :int]]
     ;; TODO: Should this be here or in common?
-   [:block/path-refs {:optional true} [:set :int]]])
+   [:block/path-refs {:optional true} [:set :int]]
+   ;; file-based
+   [:block/namespace {:optional true} :int]])
+
+(def property-attrs
+  "Common attributes for properties"
+  [[:db/index {:optional true} :boolean]
+   [:db/valueType {:optional true} [:enum :db.type/ref]]
+   [:db/cardinality {:optional true} [:enum :db.cardinality/many :db.cardinality/one]]
+   [:block/order {:optional true} :string]])
 
 (def normal-page
   (vec
@@ -106,29 +222,20 @@
     page-attrs
     page-or-block-attrs)))
 
-(def logseq-ident-namespaces
-  "Set of all namespaces Logseq uses for :db/ident. It's important to grow this
-  list purposefully and have it start with 'logseq' to allow for users and 3rd
-  party plugins to provide their own namespaces to core concepts."
-  #{"logseq.property" "logseq.property.table" "logseq.property.tldraw"
-    "logseq.class" "logseq.task" "logseq.kv"})
-
-(def logseq-ident
-  [:and :keyword [:fn
-                  {:error/message "should be a valid :db/ident namespace"}
-                  (fn logseq-namespace? [k]
-                    (contains? logseq-ident-namespaces (namespace k)))]])
+(def class-attrs
+  [[:db/ident {:optional true} class-ident]
+   [:class/parent {:optional true} :int]
+   [:class/schema.properties {:optional true} [:set :int]]])
 
 (def class-page
   (vec
    (concat
     [:map
-     [:class/parent {:optional true} :int]
-     [:db/ident {:optional true} logseq-ident]
      [:block/schema
       {:optional true}
       [:map
-       [:properties {:optional true} [:vector :uuid]]]]]
+       [:properties {:optional true} [:vector property-ident]]]]]
+    class-attrs
     page-attrs
     page-or-block-attrs)))
 
@@ -152,7 +259,7 @@
   (vec
    (concat
     [:map
-     [:db/ident logseq-ident]
+     [:db/ident internal-property-ident]
      [:block/schema
       (vec
        (concat
@@ -163,6 +270,7 @@
          [:view-context {:optional true} [:enum :page :block]]]
         property-common-schema-attrs
         property-type-schema-attrs))]]
+    property-attrs
     page-attrs
     page-or-block-attrs)))
 
@@ -180,40 +288,39 @@
          property-common-schema-attrs
          (remove #(not (db-property-type/property-type-allows-schema-attribute? prop-type (first %)))
                  property-type-schema-attrs)))])
-    db-property-type/user-built-in-property-types)))
+     db-property-type/user-built-in-property-types)))
 
 (def user-property
   (vec
    (concat
     [:map
+     [:db/ident user-property-ident]
      [:block/schema {:optional true} user-property-schema]]
+    property-attrs
     page-attrs
     page-or-block-attrs)))
 
 (def property-page
-  [:multi {:dispatch (fn [m] (contains? m :db/ident))}
+  [:multi {:dispatch (fn [m]
+                       (or (some->> (:db/ident m) db-property/logseq-property?)
+                           (contains? db-property/db-attribute-properties (:db/ident m))))}
    [true internal-property]
    [:malli.core/default user-property]])
 
 (def hidden-page
   (vec
    (concat
-    [:map]
+    [:map
+     ;; pages from :default property uses this but closed-value pages don't
+     [:block/order {:optional true} :string]]
     page-attrs
     page-or-block-attrs)))
 
-(def page
-  [:multi {:dispatch :block/type}
-   [#{"property"} property-page]
-   [#{"class"} class-page]
-   [#{"hidden"} hidden-page]
-   [:malli.core/default normal-page]])
-
 (def block-attrs
   "Common attributes for normal blocks"
   [[:block/content :string]
-   [:block/left :int]
    [:block/parent :int]
+   [:block/order :string]
    ;; refs
    [:block/page :int]
    [:block/path-refs {:optional true} [:set :int]]
@@ -246,12 +353,12 @@
     [:map]
     [[:block/type [:= #{"closed value"}]]
      ;; for built-in properties
-     [:db/ident {:optional true} logseq-ident]
+     [:db/ident {:optional true} logseq-property-ident]
      [:block/schema {:optional true}
       [:map
        [:value [:or :string :double]]
        [:description {:optional true} :string]]]]
-    (remove #(#{:block/content :block/left} (first %)) block-attrs)
+    (remove #(#{:block/content :block/order} (first %)) block-attrs)
     page-or-block-attrs)))
 
 (def normal-block
@@ -299,32 +406,62 @@
                 [:block/tx-id {:optional true} :int]])
              db-ident-keys)))
 
-(def macro
+(def property-value-placeholder
   [:map
-   [:db/ident :string]
-   [:block/uuid :uuid]
-   [:block/type [:= #{"macro"}]]
-   [:block/properties block-properties]
-   ;; Should this be removed?
+   [:db/ident [:= :logseq.property/empty-placeholder]]
    [:block/tx-id {:optional true} :int]])
 
+(defn- type-set
+  [d]
+  (when-let [type (:block/type d)]
+    (if (coll? type)
+      (set type)
+      #{type})))
+
+(def Data
+  (into
+   [:multi {:dispatch (fn [d]
+                        (cond
+                          (contains? (type-set d) "property")
+                          :property
+                          (contains? (type-set d) "class")
+                          :class
+                          (contains? (type-set d) "hidden")
+                          :hidden
+                          (contains? (type-set d) "whiteboard")
+                          :normal-page
+                          (sqlite-util/page? d)
+                          :normal-page
+                          (:file/path d)
+                          :file-block
+                          (:block/uuid d)
+                          :block
+                          (:asset/uuid d)
+                          :asset-block
+                          (= (:db/ident d) :logseq.property/empty-placeholder)
+                          :property-value-placeholder
+                          (:db/ident d)
+                          :db-ident-key-value))}]
+   {:property property-page
+    :class class-page
+    :hidden hidden-page
+    :normal-page normal-page
+    :block block
+    :file-block file-block
+    :db-ident-key-value db-ident-key-val
+    :asset-block asset-block
+    :property-value-placeholder property-value-placeholder}))
+
 (def DB
   "Malli schema for entities from schema/schema-for-db-based-graph. In order to
   thoroughly validate properties, the entities and this schema should be
   prepared with update-properties-in-ents and update-properties-in-schema
   respectively"
-  [:sequential
-   [:or
-    page
-    block
-    file-block
-    db-ident-key-val
-    macro
-    asset-block]])
+  [:sequential Data])
 
 ;; Keep malli schema in sync with db schema
 ;; ========================================
-(let [malli-many-ref-attrs (->> (concat page-attrs block-attrs page-or-block-attrs)
+(let [malli-many-ref-attrs (->> (concat class-attrs page-attrs block-attrs page-or-block-attrs)
                                 (filter #(= (last %) [:set :int]))
                                 (map first)
                                 set)]
@@ -333,7 +470,7 @@
                          (string/join ", " undeclared-ref-attrs))
                     {}))))
 
-(let [malli-one-ref-attrs (->> (concat page-attrs block-attrs page-or-block-attrs (rest normal-page))
+(let [malli-one-ref-attrs (->> (concat class-attrs page-attrs block-attrs page-or-block-attrs (rest normal-page))
                                (filter #(= (last %) :int))
                                (map first)
                                set)
@@ -343,7 +480,7 @@
                          (string/join ", " undeclared-ref-attrs))
                     {}))))
 
-(let [malli-non-ref-attrs (->> (concat page-attrs block-attrs page-or-block-attrs (rest normal-page))
+(let [malli-non-ref-attrs (->> (concat class-attrs page-attrs block-attrs page-or-block-attrs (rest normal-page))
                                (concat (rest file-block) (rest asset-block)
                                        db-ident-keys (rest class-page))
                                (remove #(= (last %) [:set :int]))

deps/db/src/logseq/db/frontend/order.cljs

@@ -0,0 +1,40 @@
+(ns logseq.db.frontend.order
+  "Use fractional-indexing order for blocks/properties/closed values/etc."
+  (:require [logseq.common.fractional-index :as index]
+            [datascript.core :as d]))
+
+(defonce *max-key (atom nil))
+
+(defn reset-max-key!
+  [key]
+  (when (and key (or (nil? @*max-key)
+                     (> (compare key @*max-key) 0)))
+    (reset! *max-key key)))
+
+(defn gen-key
+  ([]
+   (gen-key @*max-key nil))
+  ([end]
+   (gen-key @*max-key end))
+  ([start end]
+   (let [k (index/generate-key-between start end)]
+     (reset-max-key! k)
+     k)))
+
+(defn get-max-order
+  [db]
+  (:v (first (d/rseek-datoms db :avet :block/order))))
+
+(defn get-prev-order
+  [db current-key]
+  (:v (second (d/rseek-datoms db :avet :block/order current-key))))
+
+(defn get-next-order
+  [db current-key]
+  (:v (second (d/seek-datoms db :avet :block/order current-key))))
+
+(defn gen-n-keys
+  [n start end]
+  (let [ks (index/generate-n-keys-between start end n)]
+    (reset! *max-key (last ks))
+    ks))

deps/db/src/logseq/db/frontend/property.cljs

@@ -1,12 +1,11 @@
 (ns logseq.db.frontend.property
   "Property related fns for DB graphs and frontend/datascript usage"
-  (:require [logseq.db.sqlite.util :as sqlite-util]
-            [datascript.core :as d]
-            [logseq.common.util :as common-util]
-            [clojure.string :as string]))
+  (:require [datascript.core :as d]
+            [clojure.string :as string]
+            [flatland.ordered.map :refer [ordered-map]]))
 
-(def first-stage-properties
-  #{:logseq.property/built-in? :logseq.property/created-from-property})
+;; Main property vars
+;; ==================
 
 (def ^:large-vars/data-var built-in-properties*
   "Map of built in properties for db graphs with their :db/ident as keys.
@@ -23,15 +22,15 @@
    * :name - Property's :block/name as a keyword. If none given, one is derived from the db/ident
    * :attribute - Property keyword that is saved to a datascript attribute outside of :block/properties
    * :closed-values - Vec of closed-value maps for properties with choices. Map
-     has keys :value, :db-ident, :uuid and :icon
-   * :db-ident - Keyword to set :db/ident and give property unique id in db"
-  {:logseq.property/alias {:original-name "Alias"
+     has keys :value, :db-ident, :uuid and :icon"
+  (ordered-map
+   :block/alias           {:original-name "Alias"
                            :attribute :block/alias
                            :schema {:type :page
                                     :cardinality :many
                                     :view-context :page
                                     :public? true}}
-   :logseq.property/tags {:original-name "Tags"
+   :block/tags           {:original-name "Tags"
                           :attribute :block/tags
                           :schema {:type :page
                                    :cardinality :many
@@ -42,47 +41,58 @@
                                         :public? true
                                         :view-context :page
                                         :cardinality :many}}
-   :logseq.property/background-color {:schema {:type :default :hide? true}}
+   :logseq.property/background-color {:schema {:type :string :hide? true}}
    :logseq.property/background-image {:schema
-                                      {:type :default
-                                       :hide? true
+                                      {:type :string
                                        :view-context :block
                                        :public? true}}
    ;; number (1-6) or boolean for auto heading
    :logseq.property/heading {:schema {:type :any :hide? true}}
-   :logseq.property/created-from-block    {:schema {:type :uuid}}
-   :logseq.property/created-from-property {:schema {:type :uuid}}
-   :logseq.property/created-from-template {:schema {:type :uuid}}
-   :logseq.property/source-page-id        {:schema {:type :uuid}}
-   :logseq.property/built-in?             {:schema {:type :checkbox}}
-   :logseq.property/hide-properties?      {:schema {:type :checkbox}}
-   :logseq.property/query-table {:schema {:type :checkbox}}
+   :logseq.property/created-from-property {:schema {:type :entity
+                                                    :hide? true}}
+   :logseq.property/created-from-template {:schema {:type :entity
+                                                    :hide? true}}
+   :logseq.property/source-page           {:schema {:type :entity
+                                                    :hide? true}}
+   :logseq.property/built-in?             {:schema {:type :checkbox
+                                                    :hide? true}}
+   :logseq.property/hide-properties?      {:schema {:type :checkbox
+                                                    :hide? true}}
+   :logseq.property/query-table {:schema {:type :checkbox
+                                          :hide? true}}
    ;; query-properties is a coll of property uuids and keywords where keywords are special frontend keywords
-   :logseq.property/query-properties {:schema {:type :coll}}
+   :logseq.property/query-properties {:schema {:type :coll
+                                               :hide? true}}
    ;; query-sort-by is either a property uuid or a keyword where keyword is a special frontend keyword
-   :logseq.property/query-sort-by {:schema {:type :any}}
-   :logseq.property/query-sort-desc {:schema {:type :checkbox}}
-   :logseq.property/ls-type {:schema {:type :keyword}}
-   :logseq.property/hl-type {:schema {:type :keyword}}
-   :logseq.property/hl-page {:schema {:type :number}}
-   :logseq.property/hl-stamp {:schema {:type :number}}
-   :logseq.property/hl-color {:schema {:type :default}}
-   :logseq.property/macro-name {:name :logseq.macro-name
-                                :schema {:type :default}}
-   :logseq.property/macro-arguments {:name :logseq.macro-arguments
-                                     :schema {:type :coll}}
+   :logseq.property/query-sort-by {:schema {:type :any
+                                            :hide? true}}
+   :logseq.property/query-sort-desc {:schema {:type :checkbox
+                                              :hide? true}}
+   :logseq.property/ls-type {:schema {:type :keyword
+                                      :hide? true}}
+   :logseq.property/hl-type {:schema {:type :keyword
+                                      :hide? true}}
+   :logseq.property/hl-page {:schema {:type :number
+                                      :hide? true}}
+   :logseq.property/hl-stamp {:schema {:type :number
+                                       :hide? true}}
+   :logseq.property/hl-color {:schema {:type :string
+                                       :hide? true}}
    :logseq.property/order-list-type {:name :logseq.order-list-type
-                                     :schema {:type :default}}
+                                     :schema {:type :string
+                                              :hide? true}}
    :logseq.property.tldraw/page {:name :logseq.tldraw.page
-                                 :schema {:type :map}}
+                                 :schema {:type :map
+                                          :hide? true}}
    :logseq.property.tldraw/shape {:name :logseq.tldraw.shape
-                                  :schema {:type :map}}
+                                  :schema {:type :map
+                                           :hide? true}}
 
    ;; Task props
    :logseq.task/status
    {:original-name "Status"
     :schema
-    {:type :default
+    {:type :string
      :public? true}
     :closed-values
     (mapv (fn [[db-ident value icon]]
@@ -99,7 +109,7 @@
    :logseq.task/priority
    {:original-name "Priority"
     :schema
-    {:type :default
+    {:type :string
      :public? true}
     :closed-values
     (mapv (fn [[db-ident value]]
@@ -125,7 +135,7 @@
    :logseq.property/color
    {:name :logseq.color
     :schema
-    {:type :default :hide? true :public? true}
+    {:type :string :hide? true :public? true}
     :closed-values
     (mapv #(hash-map :db-ident (keyword "logseq.property" (str "color." %))
                      :value %
@@ -140,7 +150,7 @@
    :logseq.property.table/headers
    {:name :logseq.table.headers
     :schema
-    {:type :default :hide? true :public? true :view-context :block}
+    {:type :string :hide? true :public? true :view-context :block}
     :closed-values
     (mapv #(hash-map :db-ident (keyword "logseq.property.table" (str "headers." %))
                      :value %
@@ -149,7 +159,7 @@
    :logseq.property.table/hover
    {:name :logseq.table.hover
     :schema
-    {:type :default :hide? true :public? true :view-context :block}
+    {:type :string :hide? true :public? true :view-context :block}
     :closed-values
     (mapv #(hash-map :db-ident (keyword "logseq.property.table" (str "hover." %))
                      :value %
@@ -169,12 +179,13 @@
                              :hide? true
                              :view-context :page
                              :public? true}}
-   :logseq.property/filters {:schema {:type :map}}
+   :logseq.property/filters {:schema {:type :map
+                                      :hide? true}}
    :logseq.property/exclude-from-graph-view {:schema
                                              {:type :checkbox
                                               :hide? true
                                               :view-context :page
-                                              :public? true}}})
+                                              :public? true}}))
 
 (def built-in-properties
   (->> built-in-properties*
@@ -185,6 +196,49 @@
                (if (:name v)
                  v
                  (assoc v :name (keyword (string/lower-case (name k)))))]))
+       (into (ordered-map))))
+
+(def db-attribute-properties
+  "Internal properties that are also db schema attributes"
+  #{:block/alias :block/tags})
+
+(assert (= db-attribute-properties
+           (set (keep (fn [[k {:keys [attribute]}]] (when attribute k))
+                      built-in-properties)))
+        "All db attribute properties are configured in built-in-properties")
+
+(def logseq-property-namespaces
+  #{"logseq.property" "logseq.property.table" "logseq.property.tldraw"
+    "logseq.task"})
+
+(defn logseq-property?
+  "Determines if keyword is a logseq property"
+  [kw]
+  (contains? logseq-property-namespaces (namespace kw)))
+
+(defn user-property-namespace?
+  "Determines if namespace string is a user property"
+  [s]
+  (string/includes? s ".property"))
+
+(defn property?
+  "Determines if ident kw is a property"
+  [k]
+  (let [k-name (namespace k)]
+    (and k-name
+         (or (contains? logseq-property-namespaces k-name)
+             (user-property-namespace? k-name)
+             (and (keyword? k) (contains? db-attribute-properties k))))))
+
+;; Helper fns
+;; ==========
+
+(defn properties
+  "Returns a block's properties as a map indexed by property's db-ident.
+   Use this in deps because nbb can't use :block/properties from entity-plus"
+  [e]
+  (->> (into {} e)
+       (filter (fn [[k _]] (property? k)))
        (into {})))
 
 (defn valid-property-name?
@@ -193,46 +247,9 @@
   ;; Disallow tags or page refs as they would create unreferenceable page names
   (not (re-find #"^(#|\[\[)" s)))
 
-(defn get-pid
-  "Get a built-in property's id (keyword name for file graph and uuid for db graph)
-  given its db-ident. Use this fn on a file or db graph. Use
-  db-pu/get-built-in-property-uuid if only in a db graph context"
-  [repo db db-ident]
-  (if (sqlite-util/db-based-graph? repo)
-    (when db (:block/uuid (d/entity db db-ident)))
-    (get-in built-in-properties [db-ident :name])))
-
-(defn lookup
-  "Get the value of coll by db-ident. For file and db graphs"
-  [repo db coll db-ident]
-  (get coll (get-pid repo db db-ident)))
-
-(defn get-block-property-value
-  "Get the value of built-in block's property by its db-ident"
-  [repo db block db-ident]
-  (when db
-    (let [block' (or (d/entity db (:db/id block)) block)]
-      (get (:block/properties block') (get-pid repo db db-ident)))))
-
-(defn shape-block?
-  [repo db block]
-  (= :whiteboard-shape (get-block-property-value repo db block :logseq.property/ls-type)))
-
-(defn get-built-in
-  "Gets a built-in page/class/property/X by its :db/ident"
-  [db db-ident]
-  (d/entity db db-ident))
-
-(defn get-by-ident-or-name
-  "Gets a property by db-ident or name if it's a user property"
-  [db ident-or-name]
-  (if (and (keyword? ident-or-name) (namespace ident-or-name))
-    (get-built-in db ident-or-name)
-    (d/entity db [:block/name (common-util/page-name-sanity-lc (name ident-or-name))])))
-
 (defn get-closed-property-values
-  [db ident-or-name]
-  (when-let [property (get-by-ident-or-name db ident-or-name)]
+  [db property-id]
+  (when-let [property (d/entity db property-id)]
     (get-in property [:block/schema :values])))
 
 (defn closed-value-name
@@ -249,3 +266,39 @@
             (let [e (d/entity db [:block/uuid id])]
               (when (= (closed-value-name e) value-name)
                 e))) values)))
+
+;; TODO: db ident should obey clojure's rules for keywords
+(defn create-db-ident-from-name
+  "Creates a :db/ident by sanitizing the given name.
+
+   NOTE: Only use this when creating a db-ident for a string name. Using this
+   in read-only contexts like querying can result in db-ident conflicts"
+  [user-namespace name-string]
+  (let [n (-> name-string
+              (string/replace #"(^:\s*|\s*:$)" "")
+              (string/replace #"\s*:\s*$" "")
+              (string/replace-first #"^\d+" "")
+              (string/replace " " "-")
+              (string/replace "#" "")
+              (string/trim))]
+    (assert (seq n) "name is not empty")
+    (keyword user-namespace n)))
+
+(defn create-user-property-ident-from-name
+  "Creates a property :db/ident for a default user namespace"
+  [property-name]
+  (create-db-ident-from-name "user.property" property-name))
+
+(defn get-class-ordered-properties
+  [class-entity]
+  (->> (:class/schema.properties class-entity)
+       (sort-by :block/order)))
+
+(defn property-created-block?
+  "`block` has been created in a property and it's not a closed value."
+  [block]
+  (and (map? block)
+       (:logseq.property/created-from-property block)
+       (:block/page block)
+       ;; not closed value
+       (not (some? (get-in block [:block/schema :value])))))

deps/db/src/logseq/db/frontend/property/build.cljs

@@ -0,0 +1,73 @@
+(ns logseq.db.frontend.property.build
+  "Builds core property concepts"
+  (:require [logseq.db.sqlite.util :as sqlite-util]
+            [logseq.db.frontend.property.type :as db-property-type]))
+
+(defonce hidden-page-name-prefix "$$$")
+
+(defn- closed-value-new-block
+  [page-id block-id value property]
+  {:block/type #{"closed value"}
+   :block/format :markdown
+   :block/uuid block-id
+   :block/page page-id
+   :logseq.property/created-from-property (:db/ident property)
+   :block/schema {:value value}
+   :block/parent page-id})
+
+(defn build-closed-value-block
+  "Builds a closed value block to be transacted"
+  [block-uuid block-value page-id property {:keys [db-ident icon description]}]
+  (assert block-uuid (uuid? block-uuid))
+  (cond->
+   (closed-value-new-block page-id block-uuid block-value property)
+    (and db-ident (keyword? db-ident))
+    (assoc :db/ident db-ident)
+
+    icon
+    (assoc :logseq.property/icon icon)
+
+    ;; For now, only closed values with :db/ident are built-in?
+    (and db-ident (keyword? db-ident))
+    sqlite-util/mark-block-as-built-in
+
+    description
+    (update :block/schema assoc :description description)
+
+    true
+    sqlite-util/block-with-timestamps))
+
+(defn build-property-hidden-page
+  "Builds a hidden property page for closed values to be transacted"
+  [property]
+  (let [page-name (str hidden-page-name-prefix (:block/uuid property))]
+    (-> (sqlite-util/build-new-page page-name)
+        (assoc :block/type #{"hidden"}))))
+
+(defn build-closed-values
+  "Builds all the tx needed for property with closed values including
+   the hidden page and closed value blocks as needed"
+  [db-ident prop-name property {:keys [property-attributes]}]
+  (let [property-schema (assoc (:block/schema property)
+                               :values (mapv :uuid (:closed-values property)))
+        property-tx (merge (sqlite-util/build-new-property db-ident property-schema {:original-name prop-name
+                                                                                     :ref-type? true})
+                           property-attributes)
+        ref-type? (contains? (set (remove #{:default} db-property-type/ref-property-types))
+                             (get-in property [:block/schema :type]))
+        hidden-tx
+        ;; closed ref types don't have hidden tx
+        (if ref-type?
+          []
+          (let [page-tx (build-property-hidden-page property-tx)
+                closed-value-blocks-tx
+                (map (fn [{:keys [db-ident value icon description uuid]}]
+                       (build-closed-value-block
+                        uuid
+                        value
+                        [:block/uuid (:block/uuid page-tx)]
+                        property
+                        {:db-ident db-ident :icon icon :description description}))
+                     (:closed-values property))]
+            (into [page-tx] closed-value-blocks-tx)))]
+    (into [property-tx] hidden-tx)))

deps/db/src/logseq/db/frontend/property/type.cljs

@@ -12,25 +12,35 @@
 
 (def internal-built-in-property-types
   "Valid property types only for use by internal built-in-properties"
-  #{:keyword :map :coll :any :uuid})
+  #{:keyword :map :coll :any :entity})
 
 (def user-built-in-property-types
   "Valid property types for users in order they appear in the UI"
-  [:default :number :date :checkbox :url :page :template])
+  [:default :string :number :date :checkbox :url :page :template])
 
 (def closed-value-property-types
   "Valid schema :type for closed values"
-  #{:default :number :url :page :date})
+  #{:string :number :url :page :date})
 
 (assert (set/subset? closed-value-property-types (set user-built-in-property-types))
         "All closed value types are valid property types")
 
+(def ref-property-types
+  "User facing ref types"
+  #{:default :page :date})
+
+(assert (set/subset? ref-property-types
+                     (set user-built-in-property-types))
+        "All ref types are valid property types")
+
 (def ^:private user-built-in-allowed-schema-attributes
   "Map of types to their set of allowed :schema attributes"
   (merge-with into
               (zipmap closed-value-property-types (repeat #{:values}))
-              (zipmap #{:default :number :url} (repeat #{:position}))
-              {:number #{:cardinality}
+              (zipmap #{:string :number :url} (repeat #{:position}))
+              {:default #{:cardinality}
+               :string #{:cardinality}
+               :number #{:cardinality}
                :date #{:cardinality}
                :url #{:cardinality}
                :page #{:cardinality :classes}
@@ -60,69 +70,72 @@
   ;; TODO: Confirm that macro expanded value is url when it's easier to pass data into validations
   (macro-util/macro? s))
 
-(defn- logseq-page?
-  [db id]
-  (and (uuid? id)
-       (when-let [e (d/entity db [:block/uuid id])]
-         (nil? (:block/page e)))))
-
-;; FIXME: template instance check
-(defn- logseq-template?
+(defn- entity?
   [db id]
-  (and (uuid? id)
-       (some? (d/entity db [:block/uuid id]))))
-
-(defn- existing-closed-value-valid?
-  "Validates that the given existing closed value is valid"
-  [db property type-validate-fn value]
-  (boolean
-   (when-let [e (and (uuid? value)
-                     (d/entity db [:block/uuid value]))]
-     (let [values (get-in property [:block/schema :values])]
-       (and
-        (contains? (set values) value)
-        (if (contains? (:block/type e) "closed value")
-          (type-validate-fn (:value (:block/schema e)))
-          ;; page uuids aren't closed value types
-          (type-validate-fn value)))))))
-
-(defn type-or-closed-value?
-  "The `value` could be either a closed value (when `property` has pre-defined values) or it can be validated by `type-validate-fn`.
-  Args:
-  `new-closed-value?`: a new value will be added, so we'll check it using `type-validate-fn`."
-  [type-validate-fn]
-  (fn [db property value new-closed-value?]
-    (if (and (seq (get-in property [:block/schema :values]))
-             (not new-closed-value?))
-      (existing-closed-value-valid? db property type-validate-fn value)
-      (type-validate-fn value))))
+  (some? (d/entity db id)))
+
+(defn- url-or-closed-url?
+  [db val]
+  (or (url? val)
+      (macro-url? val)
+      (when-let [ent (d/entity db val)]
+        (url? (get-in ent [:block/schema :value])))))
+
+(defn- property-value-block?
+  [db s]
+  (when-let [ent (d/entity db s)]
+    (and (:block/content ent)
+         (:logseq.property/created-from-property ent))))
+
+(defn- page?
+  [db val]
+  (when-let [ent (d/entity db val)]
+    (some? (:block/original-name ent))))
+
+(defn- date?
+  [db val]
+  (when-let [ent (d/entity db val)]
+    (and (some? (:block/original-name ent))
+         (contains? (:block/type ent) "journal"))))
+
+(defn- string-or-closed-string?
+  [db s]
+  (or (string? s)
+      (when-let [entity (d/entity db s)]
+        (string? (get-in entity [:block/schema :value])))))
 
 (def built-in-validation-schemas
   "Map of types to malli validation schemas that validate a property value for that type"
   {:default  [:fn
-              {:error/message "should be a text"}
-              ;; uuid check needed for property block values
-              (some-fn string? uuid?)]                     ; refs/tags will not be extracted
+              {:error/message "should be a text block"}
+              property-value-block?]
+   :string   [:fn
+              {:error/message "should be a string"}
+              string-or-closed-string?]
    :number   [:fn
               {:error/message "should be a number"}
-              ;; TODO: Remove uuid? for :number and :url when type-or-closed-value? is used in this ns
-              (some-fn number? uuid?)]
+              ;; Also handles entity? so no need to use it
+              number?]
    :date     [:fn
               {:error/message "should be a journal date"}
-              logseq-page?]
+              date?]
    :checkbox boolean?
    :url      [:fn
               {:error/message "should be a URL"}
-              (some-fn url? uuid? macro-url?)]
+              url-or-closed-url?]
    :page     [:fn
               {:error/message "should be a page"}
-              logseq-page?]
+              page?]
+   ;; TODO: strict check on template
    :template [:fn
               {:error/message "should has #template"}
-              logseq-template?]
-   ;; internal usage
+              entity?]
+
+   ;; Internal usage
+   ;; ==============
+
+   :entity   entity?
    :keyword  keyword?
-   :uuid     uuid?
    :map      map?
    ;; coll elements are ordered as it's saved as a vec
    :coll     coll?
@@ -135,7 +148,7 @@
 
 (def property-types-with-db
   "Property types whose validation fn requires a datascript db"
-  #{:date :page :template})
+  #{:default :string :url :date :page :template :entity})
 
 ;; Helper fns
 ;; ==========

deps/db/src/logseq/db/frontend/property/util.cljs

@@ -1,72 +1,31 @@
 (ns logseq.db.frontend.property.util
-  "Util fns for building core property concepts"
-  (:require [logseq.db.sqlite.util :as sqlite-util]
-            [logseq.db.frontend.default :as default-db]
-            [datascript.core :as d]))
-
-(defonce hidden-page-name-prefix "$$$")
-
-(defn- closed-value-new-block
-  [db page-id block-id value property]
-  {:block/type #{"closed value"}
-   :block/format :markdown
-   :block/uuid block-id
-   :block/page page-id
-   :block/properties {(:block/uuid (d/entity db :logseq.property/created-from-property)) (:block/uuid property)}
-   :block/schema {:value value}
-   :block/parent page-id})
-
-(defn build-closed-value-block
-  "Builds a closed value block to be transacted"
-  [db block-uuid block-value page-id property {:keys [db-ident icon-id icon description]}]
-  (cond->
-   (closed-value-new-block db page-id (or block-uuid (d/squuid)) block-value property)
-    (and db-ident (keyword? db-ident))
-    (assoc :db/ident db-ident)
-
-    icon
-    (update :block/properties assoc icon-id icon)
-
-    ;; For now, only closed values with :db/ident are built-in?
-    (and db-ident (keyword? db-ident))
-    ((fn [b] (default-db/mark-block-as-built-in db b)))
-
-    description
-    (update :block/schema assoc :description description)
-
-    true
-    sqlite-util/block-with-timestamps))
-
-(defn build-property-hidden-page
-  "Builds a hidden property page for closed values to be transacted"
-  [property]
-  (let [page-name (str hidden-page-name-prefix (:block/uuid property))]
-    (-> (sqlite-util/build-new-page page-name)
-        (assoc :block/type #{"hidden"}
-               :block/format :markdown))))
-
-(defn build-closed-values
-  "Builds all the tx needed for property with closed values including
-   the hidden page and closed value blocks as needed"
-  [db prop-name property {:keys [icon-id db-ident translate-closed-page-value-fn property-attributes]
-                          :or {translate-closed-page-value-fn identity}}]
-  (let [page-tx (build-property-hidden-page property)
-        page-id [:block/uuid (:block/uuid page-tx)]
-        closed-value-page-uuids? (contains? #{:page :date} (get-in property [:block/schema :type]))
-        closed-value-blocks-tx
-        (if closed-value-page-uuids?
-          (map translate-closed-page-value-fn (:closed-values property))
-          (map (fn [{:keys [db-ident value icon description uuid]}]
-                 (build-closed-value-block db
-                  uuid value page-id property {:db-ident db-ident
-                                               :icon-id icon-id
-                                               :icon icon
-                                               :description description}))
-               (:closed-values property)))
-        property-schema (assoc (:block/schema property)
-                               :values (mapv :block/uuid closed-value-blocks-tx))
-        property-tx (merge (sqlite-util/build-new-property prop-name property-schema (:block/uuid property)
-                                                           {:db-ident db-ident})
-                           property-attributes)]
-    (into [property-tx page-tx]
-          (when-not closed-value-page-uuids? closed-value-blocks-tx))))
+  "Property related util fns. Fns used in both DB and file graphs should go here"
+  (:require [logseq.db.frontend.property :as db-property]
+            [datascript.core :as d]
+            [logseq.db.sqlite.util :as sqlite-util]))
+
+(defn get-pid
+  "Get a built-in property's id (keyword name for file graph and db-ident for db
+  graph) given its db-ident. No need to use this fn in a db graph only context"
+  [repo db-ident]
+  (if (sqlite-util/db-based-graph? repo)
+    db-ident
+    (get-in db-property/built-in-properties [db-ident :name])))
+
+(defn lookup
+  "Get the value of coll by db-ident. For file and db graphs"
+  [repo coll db-ident]
+  (get coll (get-pid repo db-ident)))
+
+(defn get-block-property-value
+  "Get the value of built-in block's property by its db-ident"
+  [repo db block db-ident]
+  (when db
+    (let [block (or (d/entity db (:db/id block)) block)]
+      (if (sqlite-util/db-based-graph? repo)
+        (get block db-ident)
+        (lookup repo (:block/properties block) db-ident)))))
+
+(defn shape-block?
+  [repo db block]
+  (= :whiteboard-shape (get-block-property-value repo db block :logseq.property/ls-type)))

deps/db/src/logseq/db/frontend/rules.cljc

@@ -107,7 +107,7 @@
    :between
    '[(between ?b ?start ?end)
      [?b :block/page ?p]
-     [?p :block/journal? true]
+     [?p :block/type "journal"]
      [?p :block/journal-day ?d]
      [(>= ?d ?start)]
      [(<= ?d ?end)]]
@@ -154,52 +154,66 @@
   (merge
    query-dsl-rules
    {:page-tags
-   '[(page-tags ?p ?tags)
-     [?p :block/tags ?t]
-     [?t :block/name ?tag]
-     [(missing? $ ?p :block/link)]
-     [(contains? ?tags ?tag)]]
+    '[(page-tags ?p ?tags)
+      [?p :block/tags ?t]
+      [?t :block/name ?tag]
+      [(missing? $ ?p :block/link)]
+      [(contains? ?tags ?tag)]]
 
     :has-page-property
     '[(has-page-property ?p ?prop)
       [?p :block/name]
-      [?p :block/properties ?bp]
-      [(name ?prop) ?prop-name-str]
-      [?prop-b :block/name ?prop-name-str]
-      [?prop-b :block/type "property"]
-      [?prop-b :block/uuid ?prop-uuid]
-      [(get ?bp ?prop-uuid) ?v]
+      [?p ?prop ?v]
+      [?prop-e :db/ident ?prop]
+      [?prop-e :block/type "property"]
       ;; Some deleted properties leave #{} which this rule shouldn't match on
       [(not= #{} ?v)]]
+    ;; TODO: Delete when DB_GRAPH query-dsl tests are passing
+    #_'[(has-page-property ?p ?prop)
+        [?p :block/name]
+        [?p :block/properties ?bp]
+        [(name ?prop) ?prop-name-str]
+        [?prop-b :block/name ?prop-name-str]
+        [?prop-b :block/type "property"]
+        [?prop-b :block/uuid ?prop-uuid]
+        [(get ?bp ?prop-uuid) ?v]
+      ;; Some deleted properties leave #{} which this rule shouldn't match on
+        [(not= #{} ?v)]]
 
     :page-property
-    '[;; Clause 1: Match non-ref values
-      [(page-property ?p ?key ?val)
+    '[[(page-property ?p ?prop ?val)
        [?p :block/name]
-       [?p :block/properties ?prop]
-       [(name ?key) ?key-str]
-       [?prop-b :block/name ?key-str]
-       [?prop-b :block/type "property"]
-       [?prop-b :block/uuid ?prop-uuid]
-       [(get ?prop ?prop-uuid) ?v]
-       (or ([= ?v ?val])
-           [(contains? ?v ?val)])]
+       [?p ?prop ?val]
+       [?prop-e :db/ident ?prop]
+       [?prop-e :block/type "property"]]
+      ;; TODO: Delete when DB_GRAPH query-dsl tests are passing
+      ;; Clause 1: Match non-ref values
+      #_[(page-property ?p ?key ?val)
+         [?p :block/name]
+         [?p :block/properties ?prop]
+         [(name ?key) ?key-str]
+         [?prop-b :block/name ?key-str]
+         [?prop-b :block/type "property"]
+         [?prop-b :block/uuid ?prop-uuid]
+         [(get ?prop ?prop-uuid) ?v]
+         (or ([= ?v ?val])
+             [(contains? ?v ?val)])]
 
       ;; Clause 2: Match values joined by refs
-      [(page-property ?p ?key ?val)
-       [?p :block/name]
-       [?p :block/properties ?prop]
-       [(name ?key) ?key-str]
-       [?prop-b :block/name ?key-str]
-       [?prop-b :block/type "property"]
-       [?prop-b :block/uuid ?prop-uuid]
-       [(get ?prop ?prop-uuid) ?v]
-       [(str ?val) ?str-val]
+      #_[(page-property ?p ?key ?val)
+         [?p :block/name]
+         [?p :block/properties ?prop]
+         [(name ?key) ?key-str]
+         [?prop-b :block/name ?key-str]
+         [?prop-b :block/type "property"]
+         [?prop-b :block/uuid ?prop-uuid]
+         [(get ?prop ?prop-uuid) ?v]
+         [(str ?val) ?str-val]
       ;; str-val is for integer pages that aren't strings
-       [?prop-val-b :block/original-name ?str-val]
-       [?prop-val-b :block/uuid ?val-uuid]
-       (or ([= ?v ?val-uuid])
-           [(contains? ?v ?val-uuid)])]]
+         [?prop-val-b :block/original-name ?str-val]
+         [?prop-val-b :block/uuid ?val-uuid]
+         (or ([= ?v ?val-uuid])
+             [(contains? ?v ?val-uuid)])]]
 
     :has-property
     '[(has-property ?b ?prop)

deps/db/src/logseq/db/frontend/schema.cljs

@@ -5,9 +5,7 @@
 (defonce version 2)
 ;; A page is a special block, a page can corresponds to multiple files with the same ":block/name".
 (def ^:large-vars/data-var schema
-  {:schema/version  {}
-   :db/type         {}
-   :db/ident        {:db/unique :db.unique/identity}
+  {:db/ident        {:db/unique :db.unique/identity}
 
    :recent/pages {}
 
@@ -22,9 +20,8 @@
    :block/uuid {:db/unique :db.unique/identity}
    :block/parent {:db/valueType :db.type/ref
                   :db/index true}
-   :block/left   {:db/valueType :db.type/ref
-                  :db/index true}
-   :block/collapsed? {:db/index true}
+   :block/order {:db/index true}
+   :block/collapsed? {}
    :block/collapsed-properties {:db/valueType :db.type/ref
                                 :db/cardinality :db.cardinality/many}
 
@@ -49,6 +46,9 @@
    :block/link {:db/valueType :db.type/ref
                 :db/index true}
 
+   ;; page's namespace
+   :block/namespace {:db/valueType :db.type/ref}
+
    ;; for pages
    :block/alias {:db/valueType :db.type/ref
                  :db/cardinality :db.cardinality/many}
@@ -81,23 +81,23 @@
    ;; whether blocks is a repeated block (usually a task)
    :block/repeated? {}
 
-   :block/created-at {}
-   :block/updated-at {}
+   :block/created-at {:db/index true}
+   :block/updated-at {:db/index true}
 
    ;; page additional attributes
    ;; page's name, lowercase
    :block/name {:db/unique :db.unique/identity}
 
    ;; page's original name
-   :block/original-name {:db/unique :db.unique/identity}
-   ;; whether page's is a journal
-   :block/journal? {}
+   :block/original-name {:db/index true}
+
+   ;; page's journal day
    :block/journal-day {}
-   ;; page's namespace
-   :block/namespace {:db/valueType :db.type/ref}
+
    ;; macros in block
    :block/macros {:db/valueType :db.type/ref
                   :db/cardinality :db.cardinality/many}
+
    ;; block's file
    :block/file {:db/valueType :db.type/ref}
 
@@ -119,17 +119,22 @@
 (def schema-for-db-based-graph
   (merge
    (dissoc schema
-           :block/properties-text-values :block/pre-block? :recent/pages :file/handle :block/file
-           :block/properties-order)
-   {:class/parent {:db/valueType :db.type/ref
+           :block/namespace :block/properties-text-values :block/pre-block? :recent/pages :file/handle :block/file
+           :block/properties :block/properties-order :block/repeated? :block/deadline :block/scheduled :block/priority :block/marker)
+   {:block/name {:db/index true}        ; remove db/unique for :block/name
+    ;; class properties
+    :class/parent {:db/valueType :db.type/ref
                    :db/index true}
+    :class/schema.properties {:db/valueType :db.type/ref
+                              :db/cardinality :db.cardinality/many
+                              :db/index true}
+
     :file/last-modified-at {}
     :asset/uuid {:db/unique :db.unique/identity}
     :asset/meta {}}))
 
 (def retract-attributes
-  #{
-    :block/refs
+  #{:block/refs
     :block/tags
     :block/alias
     :block/marker
@@ -143,9 +148,7 @@
     :block/properties-text-values
     :block/macros
     :block/invalid-properties
-    :block/warning
-    }
-  )
+    :block/warning})
 
 ;; If only block/content changes
 (def db-version-retract-attributes
@@ -170,19 +173,21 @@
     :block/tags})
 
 
+;; DB graph helpers
+;; ================
 (def ref-type-attributes
   (into #{}
         (keep (fn [[attr-name attr-body-map]]
                 (when (= :db.type/ref (:db/valueType attr-body-map))
                   attr-name)))
-        schema))
+        schema-for-db-based-graph))
 
 (def card-many-attributes
   (into #{}
         (keep (fn [[attr-name attr-body-map]]
                 (when (= :db.cardinality/many (:db/cardinality attr-body-map))
                   attr-name)))
-        schema))
+        schema-for-db-based-graph))
 
 (def card-many-ref-type-attributes
   (set/intersection card-many-attributes ref-type-attributes))

deps/db/src/logseq/db/frontend/validate.cljs

@@ -22,31 +22,42 @@
   boolean indicating if db is valid"
   [{:keys [db-after tx-data tx-meta]} validate-options]
   (let [changed-ids (->> tx-data (map :e) distinct)
-        ent-maps* (->> changed-ids (mapcat #(d/datoms db-after :eavt %)) db-malli-schema/datoms->entity-maps vals)
-        ent-maps (vec (db-malli-schema/update-properties-in-ents ent-maps*))
+        ent-maps* (-> (mapcat #(d/datoms db-after :eavt %) changed-ids)
+                      (db-malli-schema/datoms->entity-maps {:entity-fn #(d/entity db-after %)})
+                      vals)
+        ent-maps (db-malli-schema/update-properties-in-ents db-after ent-maps*)
         db-schema (update-schema db-malli-schema/DB db-after validate-options)
-        explain-result (m/explain db-schema ent-maps)]
+        invalid-ent-maps (remove #(m/validate db-schema [%]) ent-maps)]
     (js/console.log "changed eids:" changed-ids tx-meta)
-    (if (:errors explain-result)
-      (do (js/console.error "Invalid datascript entities detected amongst changed entity ids:" changed-ids)
-          (pprint/pprint {:errors (me/humanize explain-result)})
-          (pprint/pprint {:entity-maps ent-maps})
-          false)
+    (if (seq invalid-ent-maps)
+      (do
+        (js/console.error "Invalid datascript entities detected amongst changed entity ids:" changed-ids)
+        (doseq [m invalid-ent-maps]
+          (pprint/pprint {:entity-map m
+                          :errors (me/humanize (m/explain db-schema [m]))}))
+        false)
       true)))
 
 (defn group-errors-by-entity
   "Groups malli errors by entities. db is used for providing more debugging info"
   [db ent-maps errors]
+  (assert (vector? ent-maps) "Must be a vec for grouping to work")
   (->> errors
        (group-by #(-> % :in first))
        (map (fn [[idx errors']]
-              {:entity (cond-> (get ent-maps idx)
-                         ;; Provide additional page info for debugging
-                         (:block/page (get ent-maps idx))
-                         (update :block/page
-                                 (fn [id] (select-keys (d/entity db id)
-                                                       [:block/name :block/type :db/id :block/created-at]))))
+              {:entity (let [ent (get ent-maps idx)
+                             db-id (:db/id (meta ent))]
+                         (cond-> ent
+                           db-id
+                           (assoc :db/id db-id)
+                           ;; Provide additional page info for debugging
+                           (:block/page ent)
+                           (update :block/page
+                                   (fn [id] (select-keys (d/entity db id)
+                                                         [:block/name :block/type :db/id :block/created-at])))))
+               :errors errors'
                ;; Group by type to reduce verbosity
+               ;; TODO: Move/remove this to another fn if unused
                :errors-by-type
                (->> (group-by :type errors')
                     (map (fn [[type' type-errors]]
@@ -67,9 +78,9 @@
   :errors and grouped by entity"
   [db]
   (let [datoms (d/datoms db :eavt)
-        ent-maps* (db-malli-schema/datoms->entity-maps datoms)
-        ent-maps (vec (db-malli-schema/update-properties-in-ents (vals ent-maps*)))
+        ent-maps* (db-malli-schema/datoms->entities datoms)
         schema (update-schema db-malli-schema/DB db {:closed-schema? true})
+        ent-maps (db-malli-schema/update-properties-in-ents db ent-maps*)
         errors (->> ent-maps (m/explain schema) :errors)]
     (cond-> {:datom-count (count datoms)
              :entities ent-maps}

deps/db/src/logseq/db/sqlite/common_db.cljs

@@ -6,7 +6,27 @@
             [logseq.db.sqlite.util :as sqlite-util]
             [logseq.common.util.date-time :as date-time-util]
             [logseq.common.util :as common-util]
-            [logseq.common.config :as common-config]))
+            [logseq.common.config :as common-config]
+            [logseq.db.frontend.entity-plus :as entity-plus]
+            [clojure.set :as set]
+            [logseq.db.frontend.order :as db-order]))
+
+(defn- get-pages-by-name
+  [db page-name]
+  (d/datoms db :avet :block/name (common-util/page-name-sanity-lc page-name)))
+
+(defn get-first-page-by-name
+  "Return the oldest page's db id for :block/name"
+  [db page-name]
+  (when (and db (string? page-name))
+    (first (sort (map :e (get-pages-by-name db page-name))))))
+
+(defn get-first-page-by-original-name
+  "Return the oldest page's db id for :block/original-name"
+  [db page-name]
+  {:pre [(string? page-name)]}
+  (first (sort (map :e
+                    (d/datoms db :avet :block/original-name page-name)))))
 
 (comment
   (defn- get-built-in-files
@@ -17,39 +37,34 @@
       (map #(d/pull db '[*] [:file/path %]) files))))
 
 (defn get-all-pages
-  [db]
+  [db exclude-page-ids]
   (->> (d/datoms db :avet :block/name)
-       (map (fn [e]
-              (d/pull db '[*] (:e e))))))
+       (keep (fn [e]
+               (when-not (contains? exclude-page-ids (:e e))
+                 (d/pull db '[:db/id :db/ident :block/uuid :block/name :block/original-name :block/alias :block/type
+                              :block/created-at :block/updated-at]
+                         (:e e)))))))
 
 (defn get-all-files
   [db]
   (->> (d/datoms db :avet :file/path)
-       (map (fn [e]
-              {:db/id (:e e)
-               :file/path (:v e)
-               :file/content (:file/content (d/entity db (:e e)))}))))
+       (mapcat (fn [e] (d/datoms db :eavt (:e e))))))
 
 (defn- with-block-refs
   [db block]
   (update block :block/refs (fn [refs] (map (fn [ref] (d/pull db '[*] (:db/id ref))) refs))))
 
-(defn with-parent-and-left
+(defn with-parent
   [db block]
   (cond
-    (:block/name block)
-    block
     (:block/page block)
-    (let [left (when-let [e (d/entity db (:db/id (:block/left block)))]
-                 (select-keys e [:db/id :block/uuid]))
-          parent (when-let [e (d/entity db (:db/id (:block/parent block)))]
+    (let [parent (when-let [e (d/entity db (:db/id (:block/parent block)))]
                    (select-keys e [:db/id :block/uuid]))]
       (->>
-       (assoc block
-              :block/left left
-              :block/parent parent)
+       (assoc block :block/parent parent)
        (common-util/remove-nils-non-nested)
        (with-block-refs db)))
+
     :else
     block))
 
@@ -61,121 +76,184 @@
   [b]
   (assoc b :block.temp/fully-loaded? true))
 
+(comment
+  (defn- property-without-db-attrs
+    [property]
+    (dissoc property :db/index :db/valueType :db/cardinality)))
+
+(defn- property-with-values
+  [db block]
+  (when (entity-plus/db-based-graph? db)
+    (let [block (d/entity db (:db/id block))
+          block-properties (when (seq (:block/properties block))
+                             (mapcat
+                              (fn [[_property-id property-values]]
+                                (let [values (if (and (coll? property-values)
+                                                      (map? (first property-values)))
+                                               property-values
+                                               #{property-values})
+                                      value-ids (when (every? map? values)
+                                                  (->> (map :db/id values)
+                                                       (filter (fn [id] (or (int? id) (keyword? id))))))
+                                      value-blocks (->>
+                                                    (when (seq value-ids)
+                                                      (map
+                                                       (fn [id] (d/pull db '[*] id))
+                                                       value-ids))
+                                                    ;; FIXME: why d/pull returns {:db/id db-ident} instead of {:db/id number-eid}?
+                                                    (map (fn [block]
+                                                           (let [from-property-id (get-in block [:logseq.property/created-from-property :db/id])]
+                                                             (if (keyword? from-property-id)
+                                                               (assoc-in block [:logseq.property/created-from-property :db/id] (:db/id (d/entity db from-property-id)))
+                                                               block)))))
+                                      page (when (seq values)
+                                             (when-let [page-id (:db/id (:block/page (d/entity db (:db/id (first values)))))]
+                                               (d/pull db '[*] page-id)))]
+                                  (remove nil? (concat [page] value-blocks))))
+                              (:block/properties block)))]
+      block-properties)))
+
+(defn get-block-children-ids
+  "Returns children UUIDs"
+  [db block-uuid]
+  (when-let [eid (:db/id (d/entity db [:block/uuid block-uuid]))]
+    (let [seen   (volatile! [])]
+      (loop [steps          100      ;check result every 100 steps
+             eids-to-expand [eid]]
+        (when (seq eids-to-expand)
+          (let [eids-to-expand*
+                (mapcat (fn [eid] (map first (d/datoms db :avet :block/parent eid))) eids-to-expand)
+                uuids-to-add (remove nil? (map #(:block/uuid (d/entity db %)) eids-to-expand*))]
+            (when (and (zero? steps)
+                       (seq (set/intersection (set @seen) (set uuids-to-add))))
+              (throw (ex-info "bad outliner data, need to re-index to fix"
+                              {:seen @seen :eids-to-expand eids-to-expand})))
+            (vswap! seen (partial apply conj) uuids-to-add)
+            (recur (if (zero? steps) 100 (dec steps)) eids-to-expand*))))
+      @seen)))
+
+(defn get-block-children
+  "Including nested children."
+  [db block-uuid]
+  (let [ids (get-block-children-ids db block-uuid)]
+    (when (seq ids)
+      (let [ids' (map (fn [id] [:block/uuid id]) ids)]
+        (d/pull-many db '[*] ids')))))
+
 (defn get-block-and-children
-  [db name children?]
-  (let [uuid? (common-util/uuid-string? name)
-        block (when uuid?
-                (let [id (uuid name)]
-                  (d/entity db [:block/uuid id])))
-        get-children (fn [children]
-                       (let [long-page? (> (count children) 500)]
+  [db id {:keys [children? nested-children?]}]
+  (let [block (d/entity db (if (uuid? id)
+                             [:block/uuid id]
+                             id))
+        get-children (fn [block children]
+                       (let [long-page? (and (> (count children) 500) (not (contains? (:block/type block) "whiteboard")))]
                          (if long-page?
                            (map (fn [e]
-                                  (select-keys e [:db/id :block/uuid :block/page :block/left :block/parent :block/collapsed?]))
+                                  (select-keys e [:db/id :block/uuid :block/page :block/order :block/parent :block/collapsed?]))
                                 children)
                            (->> (d/pull-many db '[*] (map :db/id children))
                                 (map #(with-block-refs db %))
-                                (map mark-block-fully-loaded)))))]
-    (if (and block (not (:block/name block))) ; not a page
-      (let [block' (->> (d/pull db '[*] (:db/id block))
-                        (with-parent-and-left db)
-                        (with-block-refs db)
-                        mark-block-fully-loaded)]
-        (cond->
-         {:block block'}
-          children?
-          (assoc :children (get-children (:block/_parent block)))))
-      (when-let [block (or block (d/entity db [:block/name name]))]
+                                (map mark-block-fully-loaded)
+                                (mapcat (fn [block]
+                                          (let [e (d/entity db (:db/id block))]
+                                            (conj
+                                             (if (seq (:block/properties e))
+                                               (vec (property-with-values db e))
+                                               [])
+                                             block))))))))]
+    (when block
+      (if (:block/page block) ; not a page
+        (let [block' (->> (d/pull db '[*] (:db/id block))
+                          (with-parent db)
+                          (with-block-refs db)
+                          mark-block-fully-loaded)]
+          (cond->
+           {:block block'
+            :properties (property-with-values db block)}
+            children?
+            (assoc :children (get-children block
+                                           (if nested-children?
+                                             (get-block-children db (:block/uuid block))
+                                             (:block/_parent block))))))
         (cond->
          {:block (->> (d/pull db '[*] (:db/id block))
                       (with-tags db)
-                      mark-block-fully-loaded)}
+                      mark-block-fully-loaded)
+          :properties (property-with-values db block)}
           children?
           (assoc :children
-                 (if (contains? (:block/type block) "whiteboard")
-                   (->> (d/pull-many db '[*] (map :db/id (:block/_page block)))
-                        (map #(with-block-refs db %))
-                        (map mark-block-fully-loaded))
-                   (get-children (:block/_page block)))))))))
+                 (get-children block (:block/_page block))))))))
 
 (defn get-latest-journals
   [db n]
   (let [today (date-time-util/date->int (js/Date.))]
     (->>
-     (d/q '[:find [(pull ?page [*]) ...]
+     (d/q '[:find [(pull ?page [:db/id :block/journal-day]) ...]
             :in $ ?today
             :where
             [?page :block/name ?page-name]
-            [?page :block/journal? true]
+            ;; [?page :block/type "journal"]
             [?page :block/journal-day ?journal-day]
             [(<= ?journal-day ?today)]]
           db
           today)
      (sort-by :block/journal-day)
      (reverse)
-     (take n))))
+     (take n)
+     (mapcat (fn [p]
+               (d/datoms db :eavt (:db/id p)))))))
 
-(defn get-structured-blocks
+(defn get-structured-datoms
   [db]
-  (let [special-pages (map #(d/pull db '[*] %) #{:logseq.property/tags})
-        structured-blocks (->> (d/datoms db :avet :block/type)
-                               (keep (fn [e]
-                                       (when (contains? #{"closed value" "property" "class"} (:v e))
-                                         (d/pull db '[*] (:e e))))))]
-    (concat special-pages structured-blocks)))
+  (mapcat (fn [type]
+            (->> (d/datoms db :avet :block/type type)
+                 (mapcat (fn [d]
+                           (d/datoms db :eavt (:e d))))))
+          ["property" "class" "closed value"]))
 
 (defn get-favorites
   "Favorites page and its blocks"
   [db]
-  (let [{:keys [block children]} (get-block-and-children db common-config/favorites-page-name true)]
+  (let [page-id (get-first-page-by-name db common-config/favorites-page-name)
+        {:keys [block children]} (get-block-and-children db page-id {:children? true})]
     (when block
-      (concat [block]
+      (concat (d/datoms db :eavt (:db/id block))
               (->> (keep :block/link children)
-                   (map (fn [l]
-                          (d/pull db '[*] (:db/id l)))))
-              children))))
-
-(defn get-full-page-and-blocks
-  [db page-name]
-  (let [data (get-block-and-children db (common-util/page-name-sanity-lc page-name) true)
-        result (first (tree-seq map? :children data))]
-    (cons (:block result)
-          (map #(dissoc % :children) (:children result)))))
-
-(defn get-home-page
-  [db files]
-  (let [config (->> (some (fn [m] (when (= (:file/path m) "logseq/config.edn")
-                                    (:file/content m))) files)
-                    (common-util/safe-read-string {}))
-        home-page (get-in config [:default-home :page])]
-    (when home-page
-      (get-full-page-and-blocks db (common-util/page-name-sanity-lc home-page)))))
+                   (mapcat (fn [l]
+                             (d/datoms db :eavt (:db/id l)))))
+              (mapcat (fn [child]
+                        (d/datoms db :eavt (:db/id child)))
+                      children)))))
 
 (defn get-initial-data
-  "Returns current database schema and initial data"
+  "Returns current database schema and initial data.
+   NOTE: This fn is called by DB and file graphs"
   [db]
-  (let [schema (:schema db)
-        idents (remove nil?
-                       (let [e (d/entity db :logseq.kv/graph-uuid)
-                             id (:graph/uuid e)]
-                         (when id
-                           [{:db/id (:db/id e)
-                             :db/ident :logseq.kv/graph-uuid
-                             :graph/uuid id}])))
-        favorites (get-favorites db)
+  (let [db-graph? (entity-plus/db-based-graph? db)
+        _ (when db-graph?
+            (db-order/reset-max-key! (db-order/get-max-order db)))
+        schema (:schema db)
+        idents (mapcat (fn [id]
+                         (when-let [e (d/entity db id)]
+                           (d/datoms db :eavt (:db/id e))))
+                       [:logseq.kv/db-type :logseq.kv/graph-uuid])
+        favorites (when db-graph? (get-favorites db))
         latest-journals (get-latest-journals db 3)
         all-files (get-all-files db)
-        home-page-data (get-home-page db all-files)
-        structured-blocks (get-structured-blocks db)]
+        structured-datoms (when db-graph?
+                            (get-structured-datoms db))
+        data (concat idents
+                     structured-datoms
+                     favorites
+                     latest-journals
+                     all-files)]
     {:schema schema
-     :initial-data (concat idents favorites latest-journals all-files home-page-data structured-blocks)}))
+     :initial-data data}))
 
 (defn restore-initial-data
-  "Given initial sqlite data and schema, returns a datascript connection"
+  "Given initial Datascript datoms and schema, returns a datascript connection"
   [data schema]
-  (let [conn (d/create-conn schema)]
-    (d/transact! conn data)
-    conn))
+  (d/conn-from-datoms data schema))
 
 (defn create-kvs-table!
   "Creates a sqlite table for use with datascript.storage if one doesn't exist"

deps/db/src/logseq/db/sqlite/create_graph.cljs

@@ -4,41 +4,32 @@
             [logseq.db.frontend.schema :as db-schema]
             [logseq.db.frontend.property :as db-property]
             [logseq.db.frontend.class :as db-class]
-            [logseq.db.frontend.property.util :as db-property-util]
+            [logseq.db.frontend.property.build :as db-property-build]
             [logseq.common.util :as common-util]
-            [datascript.core :as d]
-            [logseq.db.frontend.default :as default-db]))
+            [datascript.core :as d]))
 
 (defn- build-initial-properties
-  [db]
-  (let [;; Some uuids need to be pre-defined since they are referenced by other properties
-        default-property-uuids {:logseq.property/icon (d/squuid)}]
-    (mapcat
-     (fn [[db-ident {:keys [schema original-name closed-values] :as m}]]
-       (let [id (if (contains? db-property/first-stage-properties db-ident)
-                  (let [id (:block/uuid (d/entity db db-ident))]
-                    (assert (uuid? id) "First stage properties are not created yet")
-                    id)
-                  (d/squuid))
-             prop-name (or original-name (name (:name m)))
-             blocks (if closed-values
-                      (db-property-util/build-closed-values
-                       db
-                       prop-name
-                       {:block/schema schema :block/uuid id :closed-values closed-values}
-                       {:icon-id (get default-property-uuids :logseq.property/icon)
-                        :db-ident db-ident})
-                      [(sqlite-util/build-new-property
-                        prop-name
-                        schema
-                        (get default-property-uuids db-ident id)
-                        {:db-ident db-ident})])]
-         (update blocks 0 #(default-db/mark-block-as-built-in db %))))
-     db-property/built-in-properties)))
+  []
+  (mapcat
+   (fn [[db-ident {:keys [schema original-name closed-values] :as m}]]
+     (let [prop-name (or original-name (name (:name m)))
+           blocks (if closed-values
+                    (db-property-build/build-closed-values
+                     db-ident
+                     prop-name
+                     {:db/ident db-ident :block/schema schema :closed-values closed-values}
+                     {})
+                    [(sqlite-util/build-new-property
+                      db-ident
+                      schema
+                      {:original-name prop-name})])]
+       (update blocks 0 sqlite-util/mark-block-as-built-in)))
+   db-property/built-in-properties))
+
 
 (defn kv
   "Creates a key-value pair tx with the key under the :db/ident namespace :logseq.kv.
-   For example, the :db/type key is stored under an entity with ident :logseq.kv.db/type"
+   For example, the :db/type key is stored under an entity with ident :logseq.kv/db-type"
   [key value]
   {:db/ident (keyword "logseq.kv" (str (namespace key) "-" (name key)))
    key value})
@@ -46,15 +37,24 @@
 (def built-in-pages-names
   #{"Contents"})
 
+(defn- validate-tx-for-duplicate-idents [tx]
+  (when-let [conflicting-idents
+             (->> (keep :db/ident tx)
+                  frequencies
+                  (keep (fn [[k v]] (when (> v 1) k)))
+                  seq)]
+    (throw (ex-info (str "The following :db/idents are not unique and clobbered each other: "
+                         (vec conflicting-idents))
+                    {:idents conflicting-idents}))))
+
 (defn build-db-initial-data
-  [db* config-content]
-  (let [db (d/db-with db*
-                      (map (fn [ident]
-                             {:db/ident ident
-                              :block/name (name ident)
-                              :block/uuid (random-uuid)}) db-property/first-stage-properties))
-        initial-data [(kv :db/type "db")
-                      (kv :schema/version db-schema/version)]
+  "Builds tx of initial data for a new graph including key values, initial files,
+   built-in properties and built-in classes"
+  [config-content]
+  (let [initial-data [(kv :db/type "db")
+                      (kv :schema/version db-schema/version)
+                      ;; Empty property value used by db.type/ref properties
+                      {:db/ident :logseq.property/empty-placeholder}]
         initial-files [{:block/uuid (d/squuid)
                         :file/path (str "logseq/" "config.edn")
                         :file/content config-content
@@ -68,29 +68,31 @@
                         :file/content ""
                         :file/last-modified-at (js/Date.)}]
         default-pages (->> (map sqlite-util/build-new-page built-in-pages-names)
-                           (map #(assoc % :block/format :markdown))
-                           (map #(default-db/mark-block-as-built-in db %)))
-        default-properties (build-initial-properties db)
+                           (map sqlite-util/mark-block-as-built-in))
+        default-properties (build-initial-properties)
         db-ident->properties (zipmap
                               (map :db/ident default-properties)
                               default-properties)
         default-classes (map
                          (fn [[db-ident {:keys [schema original-name]}]]
-                           (default-db/mark-block-as-built-in
-                            db
-                            (sqlite-util/build-new-class
-                             (let [properties (mapv
-                                               (fn [db-ident]
-                                                 (let [id (:block/uuid (get db-ident->properties db-ident))]
-                                                   (assert id (str "Built-in property " db-ident " is not defined yet"))
-                                                   id))
-                                               (:properties schema))]
-                               (cond->
-                                {:block/original-name (or original-name (name db-ident))
-                                 :block/name (common-util/page-name-sanity-lc (name db-ident))
-                                 :db/ident db-ident
-                                 :block/uuid (d/squuid)}
-                                 (seq properties)
-                                 (assoc-in [:block/schema :properties] properties))))))
-                         db-class/built-in-classes)]
-    (vec (concat initial-data initial-files default-pages default-classes default-properties))))
+                           (let [original-name' (or original-name (name db-ident))]
+                             (sqlite-util/mark-block-as-built-in
+                              (sqlite-util/build-new-class
+                               (let [properties (mapv
+                                                 (fn [db-ident]
+                                                   (let [property (get db-ident->properties db-ident)]
+                                                     (assert property (str "Built-in property " db-ident " is not defined yet"))
+                                                     db-ident))
+                                                 (:properties schema))]
+                                 (cond->
+                                  {:block/original-name original-name'
+                                   :block/name (common-util/page-name-sanity-lc original-name')
+                                   :db/ident db-ident
+                                   :block/uuid (d/squuid)}
+                                   (seq properties)
+                                   (assoc :class/schema.properties properties)))))))
+                         db-class/built-in-classes)
+        tx (vec (concat default-properties default-classes
+                        initial-data initial-files default-pages))]
+    (validate-tx-for-duplicate-idents tx)
+    tx))

deps/db/src/logseq/db/sqlite/util.cljs

@@ -7,7 +7,10 @@
             [datascript.transit :as dt]
             [datascript.impl.entity :as de]
             [datascript.core :as d]
-            [cljs-bean.transit]))
+            [cljs-bean.transit]
+            [logseq.db.frontend.property.type :as db-property-type]
+            [logseq.db.frontend.property :as db-property]
+            [logseq.db.frontend.order :as db-order]))
 
 (defonce db-version-prefix "logseq_db_")
 (defonce file-version-prefix "logseq_local_")
@@ -68,19 +71,36 @@
     block))
 
 (defn build-new-property
-  "Build a standard new property so that it is is consistent across contexts"
-  [prop-name prop-schema prop-uuid & {:keys [db-ident]}]
-  (block-with-timestamps
-   (cond->
-    {:block/type "property"
-     :block/journal? false
-     :block/format :markdown
-     :block/uuid prop-uuid
-     :block/schema (merge {:type :default} prop-schema)
-     :block/original-name (name prop-name)
-     :block/name (common-util/page-name-sanity-lc (name prop-name))}
-     (and db-ident (keyword? db-ident))
-     (assoc :db/ident db-ident))))
+  "Build a standard new property so that it is is consistent across contexts. Takes
+   an optional map with following keys:
+   * :original-name - Case sensitive property name. Defaults to deriving this from db-ident
+   * :block-uuid - :block/uuid for property
+   * :from-ui-thread? - whether calls from the UI thread"
+  ([db-ident prop-schema] (build-new-property db-ident prop-schema {}))
+  ([db-ident prop-schema {:keys [original-name block-uuid ref-type? from-ui-thread?]}]
+   (assert (keyword? db-ident))
+   (let [db-ident' (if (qualified-keyword? db-ident)
+                     db-ident
+                     (db-property/create-user-property-ident-from-name (name db-ident)))
+         prop-name (or original-name (name db-ident'))
+         block-order (when-not from-ui-thread? (db-order/gen-key nil))]
+     (block-with-timestamps
+      (cond->
+       {:db/ident db-ident'
+        :block/type "property"
+        :block/format :markdown
+        :block/schema (merge {:type :default} prop-schema)
+        :block/name (common-util/page-name-sanity-lc (name prop-name))
+        :block/uuid (or block-uuid (d/squuid))
+        :block/original-name (name prop-name)
+        :db/index true
+        :db/cardinality (if (= :many (:cardinality prop-schema))
+                          :db.cardinality/many
+                          :db.cardinality/one)}
+        block-order
+        (assoc :block/order block-order)
+        (or ref-type? (contains? (conj db-property-type/ref-property-types :entity) (:type prop-schema)))
+        (assoc :db/valueType :db.type/ref))))))
 
 
 (defn build-new-class
@@ -88,7 +108,6 @@
   [block]
   (block-with-timestamps
    (merge {:block/type "class"
-           :block/journal? false
            :block/format :markdown}
           block)))
 
@@ -98,5 +117,15 @@
   (block-with-timestamps
    {:block/name (common-util/page-name-sanity-lc page-name)
     :block/original-name page-name
-    :block/journal? false
-    :block/uuid (d/squuid)}))
+    :block/uuid (d/squuid)
+    :block/format :markdown}))
+
+(defn page?
+  [block]
+  (and (:block/name block)
+       (nil? (:block/page block))))
+
+(defn mark-block-as-built-in
+  "Marks built-in blocks as built-in? including pages, classes, properties and closed values"
+  [block]
+  (assoc block :logseq.property/built-in? true))

deps/db/test/logseq/db/frontend/rules_test.cljs

@@ -0,0 +1,134 @@
+(ns logseq.db.frontend.rules-test
+  (:require [cljs.test :refer [deftest is testing]]
+            [datascript.core :as d]
+            [logseq.db.frontend.schema :as db-schema]
+            [logseq.db.frontend.rules :as rules]
+            [logseq.db.sqlite.create-graph :as sqlite-create-graph]
+            [logseq.db.sqlite.util :as sqlite-util]
+            [logseq.db :as ldb]))
+
+(defn- new-db-conn []
+  (let [conn (d/create-conn db-schema/schema-for-db-based-graph)
+        _ (d/transact! conn (sqlite-create-graph/build-db-initial-data "{}"))]
+    conn))
+
+(defn q-with-rules [query db]
+  ;; query assumes no :in given
+  (d/q (into query [:in '$ '%])
+       db
+       (rules/extract-rules rules/db-query-dsl-rules)))
+
+(deftest has-page-property-rule
+  (let [conn (new-db-conn)
+        page (sqlite-util/build-new-page "Page")
+        _ (d/transact! conn [(sqlite-util/build-new-property :user.property/foo {})
+                             (sqlite-util/build-new-property :user.property/foo2 {})
+                             page
+                             {:block/uuid (:block/uuid page) :user.property/foo "bar"}])]
+    (is (= ["Page"]
+           (->> (q-with-rules '[:find (pull ?b [:block/original-name]) :where (has-page-property ?b :user.property/foo)]
+                              @conn)
+                (map (comp :block/original-name first))))
+        "has-page-property returns result when page has property")
+    (is (= []
+           (->> (q-with-rules '[:find (pull ?b [:block/original-name]) :where (has-page-property ?b :user.property/foo2)]
+                              @conn)
+                (map (comp :block/original-name first))))
+        "has-page-property returns no result when page doesn't have property")
+    (is (= [:user.property/foo]
+           (q-with-rules '[:find [?p ...]
+                           :where (has-page-property ?b ?p) [?b :block/original-name "Page"]]
+                         @conn))
+        "has-page-property can bind to property arg")))
+
+(deftest page-property-rule
+  (let [conn (new-db-conn)
+        page (sqlite-util/build-new-page "Page")
+        page-a (sqlite-util/build-new-page "Page A")
+        _ (d/transact! conn [(sqlite-util/build-new-property :user.property/foo {})
+                             (sqlite-util/build-new-property :user.property/foo2 {})
+                             (sqlite-util/build-new-property :user.property/number-many {:type :number :cardinality :many})
+                             (sqlite-util/build-new-property :user.property/page-many {:type :page :cardinality :many})
+                             page
+                             page-a
+                             {:block/uuid (:block/uuid page) :user.property/foo "bar"}
+                             {:block/uuid (:block/uuid page) :user.property/number-many #{5 10}}
+                             {:block/uuid (:block/uuid page) :user.property/page-many #{[:block/uuid (:block/uuid page-a)]}}
+                             {:block/uuid (:block/uuid page-a) :user.property/foo "bar A"}])]
+
+    (testing "cardinality :one property"
+      (is (= ["Page"]
+             (->> (q-with-rules '[:find (pull ?b [:block/original-name]) :where (page-property ?b :user.property/foo "bar")]
+                                @conn)
+                  (map (comp :block/original-name first))))
+          "page-property returns result when page has property")
+      (is (= []
+             (->> (q-with-rules '[:find (pull ?b [:block/original-name]) :where (page-property ?b :user.property/foo "baz")]
+                                @conn)
+                  (map (comp :block/original-name first))))
+          "page-property returns no result when page doesn't have property value")
+      (is (= #{:user.property/foo}
+             (->> (q-with-rules '[:find [?p ...]
+                                  :where (page-property ?b ?p "bar") [?b :block/original-name "Page"]]
+                                @conn)
+                  set))
+          "page-property can bind to property arg with bound property value"))
+
+    (testing "cardinality :many property"
+      (is (= ["Page"]
+             (->> (q-with-rules '[:find (pull ?b [:block/original-name]) :where (page-property ?b :user.property/number-many 5)]
+                                @conn)
+                  (map (comp :block/original-name first))))
+          "page-property returns result when page has property")
+      (is (= []
+             (->> (q-with-rules '[:find (pull ?b [:block/original-name]) :where (page-property ?b :user.property/number-many 20)]
+                                @conn)
+                  (map (comp :block/original-name first))))
+          "page-property returns no result when page doesn't have property value")
+      (is (= #{:user.property/number-many}
+             (->> (q-with-rules '[:find [?p ...]
+                                  :where (page-property ?b ?p 5) [?b :block/original-name "Page"]]
+                                @conn)
+                  set))
+          "page-property can bind to property arg with bound property value"))
+
+    ;; NOTE: Querying a ref's name is different than before and requires more than just the rule
+    (testing ":ref property"
+      (is (= ["Page"]
+             (->> (q-with-rules '[:find (pull ?b [:block/original-name])
+                                  :where (page-property ?b :user.property/page-many ?pv) [?pv :block/original-name "Page A"]]
+                                @conn)
+                  (map (comp :block/original-name first))))
+          "page-property returns result when page has property")
+      (is (= []
+             (->> (q-with-rules '[:find (pull ?b [:block/original-name])
+                                  :where (page-property ?b :user.property/page-many ?pv) [?pv :block/original-name "Page B"]]
+                                @conn)
+                  (map (comp :block/original-name first))))
+          "page-property returns no result when page doesn't have property value"))
+
+    (testing "bindings with property value"
+      (is (= #{:user.property/foo :user.property/number-many :user.property/page-many}
+             (->> (q-with-rules '[:find [?p ...]
+                                  :where (page-property ?b ?p _) [?b :block/original-name "Page"]]
+                                @conn)
+                  set))
+          "page-property can bind to property arg with unbound property value")
+      (is (= #{[:user.property/number-many 10]
+               [:user.property/number-many 5]
+               [:user.property/foo "bar"]
+               [:user.property/page-many (:db/id (ldb/get-page @conn "Page A"))]}
+             (->> (q-with-rules '[:find ?p ?v
+                                  :where (page-property ?b ?p ?v) [?b :block/original-name "Page"]]
+                                @conn)
+                  set))
+          "page-property can bind to property and property value args")
+      (is (= #{"Page"}
+             (->> (q-with-rules '[:find (pull ?b [:block/original-name])
+                                  :where
+                                  (page-property ?b :user.property/page-many ?pv)
+                                  (page-property ?pv :user.property/foo "bar A")]
+                                @conn)
+                  (map (comp :block/original-name first))
+                  set))
+          "page-property can be used multiple times to query a property value's property"))))

deps/db/test/logseq/db/sqlite/common_db_test.cljs

@@ -51,7 +51,6 @@
           date-title (date-time-util/int->journal-title date-int "MMM do, yyyy")
           blocks [{:db/id 100001
                    :block/uuid page-uuid
-                   :block/journal? true
                    :block/journal-day date-int
                    :block/name (string/lower-case date-title)
                    :block/original-name date-title

deps/db/test/logseq/db/sqlite/create_graph_test.cljs

@@ -6,12 +6,13 @@
             [logseq.db.frontend.schema :as db-schema]
             [logseq.db.sqlite.create-graph :as sqlite-create-graph]
             [logseq.db.frontend.validate :as db-validate]
+            [logseq.db.frontend.property :as db-property]
             [logseq.db :as ldb]))
 
 (deftest new-graph-db-idents
   (testing "a new graph follows :db/ident conventions for"
     (let [conn (d/create-conn db-schema/schema-for-db-based-graph)
-          _ (d/transact! conn (sqlite-create-graph/build-db-initial-data @conn "{}"))
+          _ (d/transact! conn (sqlite-create-graph/build-db-initial-data "{}"))
           ident-ents (->> (d/q '[:find (pull ?b [:db/ident :block/type])
                                  :where [?b :db/ident]]
                                @conn)
@@ -24,7 +25,8 @@
         (is (= '() (remove namespace default-idents))
             "All default :db/ident's have namespaces")
         (is (= []
-               (->> (keep namespace default-idents)
+               (->> (remove db-property/db-attribute-properties default-idents)
+                    (keep namespace)
                     (remove #(string/starts-with? % "logseq."))))
             "All default :db/ident namespaces start with logseq."))
 
@@ -43,31 +45,36 @@
 
 (deftest new-graph-marks-built-ins
   (let [conn (d/create-conn db-schema/schema-for-db-based-graph)
-        _ (d/transact! conn (sqlite-create-graph/build-db-initial-data @conn "{}"))
-        idents (->> (d/q '[:find [(pull ?b [:db/ident :block/properties]) ...]
+        _ (d/transact! conn (sqlite-create-graph/build-db-initial-data "{}"))
+        idents (->> (d/q '[:find [(pull ?b [:db/ident :logseq.property/built-in?]) ...]
                            :where [?b :db/ident]]
                          @conn)
-                    ;; only kv's don't have built-in property
-                    (remove #(= "logseq.kv" (namespace (:db/ident %)))))]
+                    ;; only kv's and empty property value aren't marked because
+                    ;; they aren't user facing
+                    (remove #(or (= "logseq.kv" (namespace (:db/ident %)))
+                                 (= :logseq.property/empty-placeholder (:db/ident %)))))]
     (is (= []
-           (remove #(ldb/built-in? @conn %) idents))
+           (remove ldb/built-in? idents))
         "All entities with :db/ident have built-in property (except for kv idents)")))
 
 (deftest new-graph-creates-class
   (let [conn (d/create-conn db-schema/schema-for-db-based-graph)
-        _ (d/transact! conn (sqlite-create-graph/build-db-initial-data @conn "{}"))
+        _ (d/transact! conn (sqlite-create-graph/build-db-initial-data "{}"))
         task (d/entity @conn :logseq.class/task)]
     (is (contains? (:block/type task) "class")
         "Task class has correct type")
-    (is (= 4 (count (get-in task [:block/schema :properties])))
+    (is (= 4 (count (:class/schema.properties task)))
         "Has correct number of task properties")
-    (is (every? #(contains? (:block/type (d/entity @conn [:block/uuid %])) "property")
-                (get-in task [:schema :properties]))
+    (is (every? #(contains? (:block/type %) "property")
+                (:class/schema.properties task))
         "Each task property has correct type")))
 
 (deftest new-graph-is-valid
   (let [conn (d/create-conn db-schema/schema-for-db-based-graph)
-        _ (d/transact! conn (sqlite-create-graph/build-db-initial-data @conn "{}"))
+        _ (d/transact! conn (sqlite-create-graph/build-db-initial-data "{}"))
         validation (db-validate/validate-db! @conn)]
+    ;; For debugging
+    ;; (cljs.pprint/pprint (map :entity (:errors validation)))
+    ;; (println (count (:errors validation)) "errors of" (count (:entities validation)))
     (is (nil? (:errors validation))
         "New graph has no validation errors")))

deps/db/yarn.lock

@@ -2,9 +2,9 @@
 # yarn lockfile v1
 
 
-"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v10":
-  version "1.2.173-feat-db-v10"
-  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/8a4bfaae55d85292476ac8f976516b86e6ae4036"
+"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v12":
+  version "1.2.173-feat-db-v12"
+  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/8274d8b40b7fec84532c54e16b5a8c6f78bf673b"
   dependencies:
     import-meta-resolve "^2.1.0"
 

deps/graph-parser/deps.edn

@@ -17,7 +17,7 @@
  ;; with karma, shadow-cljs.edn and headless mode on CI
  {:test {:extra-paths ["test"]
          :extra-deps  {olical/cljs-test-runner   {:mvn/version "3.8.0"}
-                       org.clojure/clojurescript {:mvn/version "1.11.54"}}
+                       org.clojure/clojurescript {:mvn/version "1.11.132"}}
          :main-opts   ["-m" "cljs-test-runner.main"]}
 
   :clj-kondo {:replace-deps {clj-kondo/clj-kondo {:mvn/version "2023.05.26"}}

deps/graph-parser/package.json

@@ -3,7 +3,7 @@
   "version": "1.0.0",
   "private": true,
   "devDependencies": {
-    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v10",
+    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v12",
     "better-sqlite3": "9.3.0"
   },
   "dependencies": {

deps/graph-parser/src/logseq/graph_parser.cljs

@@ -23,12 +23,11 @@
   [db file-path]
   (ffirst
    (d/q
-    '[:find ?page-name
+    '[:find ?page
       :in $ ?path
       :where
       [?file :file/path ?path]
-      [?page :block/file ?file]
-      [?page :block/original-name ?page-name]]
+      [?page :block/file ?file]]
     db
     file-path)))
 
@@ -50,9 +49,9 @@
   UUIDs."
   [db file-page file-path retain-uuid-blocks]
   (let [existing-file-page (get-file-page db file-path)
-        pages-to-clear (distinct (filter some? [existing-file-page (:block/name file-page)]))
-        blocks (mapcat (fn [page]
-                         (ldb/get-page-blocks db page {:pull-keys [:db/id :block/uuid]}))
+        pages-to-clear (distinct (filter some? [existing-file-page (:db/id file-page)]))
+        blocks (mapcat (fn [page-id]
+                         (ldb/get-page-blocks db page-id {:pull-keys [:db/id :block/uuid]}))
                        pages-to-clear)
         retain-uuids (set (keep :block/uuid retain-uuid-blocks))]
     (retract-blocks-tx (distinct blocks) retain-uuids)))

deps/graph-parser/src/logseq/graph_parser/block.cljs

@@ -13,7 +13,9 @@
             [logseq.common.util :as common-util]
             [logseq.common.util.block-ref :as block-ref]
             [logseq.common.util.page-ref :as page-ref]
-            [datascript.impl.entity :as de]))
+            [datascript.impl.entity :as de]
+            [logseq.db :as ldb]
+            [logseq.db.frontend.order :as db-order]))
 
 (defn heading-block?
   [block]
@@ -301,45 +303,50 @@
     Useful when creating new pages from references or namespaces,
     as there's no chance to introduce timestamps via editing in page"
   [original-page-name with-id? db with-timestamp? date-formatter
-   & {:keys [from-page]}]
-  (cond
-    (and original-page-name (string? original-page-name))
-    (let [original-page-name (common-util/remove-boundary-slashes original-page-name)
-          [original-page-name page-name journal-day] (convert-page-if-journal original-page-name date-formatter)
-          namespace? (and (not (boolean (text/get-nested-page-name original-page-name)))
-                          (text/namespace-page? original-page-name))
-          page-entity (some-> db (d/entity [:block/name page-name]))
-          original-page-name (or from-page (:block/original-name page-entity) original-page-name)]
-      (merge
-       {:block/name page-name
-        :block/original-name original-page-name}
-       (when with-id?
-         (let [new-uuid (or
-                         (cond page-entity      (:block/uuid page-entity)
-                               (uuid? with-id?) with-id?)
-                         (d/squuid))]
-           {:block/uuid new-uuid}))
-       (when namespace?
-         (let [namespace (first (common-util/split-last "/" original-page-name))]
-           (when-not (string/blank? namespace)
-             {:block/namespace {:block/name (common-util/page-name-sanity-lc namespace)}})))
-       (when (and with-timestamp? (not page-entity)) ;; Only assign timestamp on creating new entity
-         (let [current-ms (common-util/time-ms)]
-           {:block/created-at current-ms
-            :block/updated-at current-ms}))
-       (if journal-day
-         {:block/journal? true
-          :block/journal-day journal-day}
-         {:block/journal? false})))
-
-    (and (map? original-page-name) (:block/uuid original-page-name))
-    original-page-name
-
-    (and (map? original-page-name) with-id?)
-    (assoc original-page-name :block/uuid (d/squuid))
-
-    :else
-    nil))
+   & {:keys [from-page class?]}]
+  (let [db-based? (ldb/db-based-graph? db)]
+    (cond
+     (and original-page-name (string? original-page-name))
+     (let [original-page-name (common-util/remove-boundary-slashes original-page-name)
+           [original-page-name page-name journal-day] (convert-page-if-journal original-page-name date-formatter)
+           namespace? (and (not db-based?)
+                           (not (boolean (text/get-nested-page-name original-page-name)))
+                           (text/namespace-page? original-page-name))
+           page-entity (when db
+                         (if class?
+                           (ldb/get-case-page db original-page-name)
+                           (ldb/get-page db original-page-name)))
+           original-page-name (or from-page (:block/original-name page-entity) original-page-name)]
+       (merge
+        {:block/name page-name
+         :block/original-name original-page-name}
+        (when with-id?
+          (let [new-uuid (or
+                          (cond page-entity      (:block/uuid page-entity)
+                                (uuid? with-id?) with-id?)
+                          (d/squuid))]
+            {:block/uuid new-uuid}))
+        (when namespace?
+          (let [namespace (first (common-util/split-last "/" original-page-name))]
+            (when-not (string/blank? namespace)
+              {:block/namespace {:block/name (common-util/page-name-sanity-lc namespace)}})))
+        (when (and with-timestamp? (not page-entity)) ;; Only assign timestamp on creating new entity
+          (let [current-ms (common-util/time-ms)]
+            {:block/created-at current-ms
+             :block/updated-at current-ms}))
+        (if journal-day
+          {:block/type "journal"
+           :block/journal-day journal-day}
+          {})))
+
+     (and (map? original-page-name) (:block/uuid original-page-name))
+     original-page-name
+
+     (and (map? original-page-name) with-id?)
+     (assoc original-page-name :block/uuid (d/squuid))
+
+     :else
+     nil)))
 
 (defn- with-page-refs-and-tags
   [{:keys [title body tags refs marker priority] :as block} with-id? db date-formatter]
@@ -347,7 +354,8 @@
                   (remove string/blank?)
                   (distinct))
         *refs (atom refs)
-        *structured-tags (atom #{})]
+        *structured-tags (atom #{})
+        db-based? (ldb/db-based-graph? db)]
     (walk/prewalk
      (fn [form]
        ;; skip custom queries
@@ -367,17 +375,18 @@
     (let [*name->id (atom {})
           ref->map-fn (fn [*col _tag?]
                         (let [col (remove string/blank? @*col)
-                              children-pages (->> (mapcat (fn [p]
-                                                            (let [p (if (map? p)
-                                                                      (:block/original-name p)
-                                                                      p)]
-                                                              (when (string? p)
-                                                                (let [p (or (text/get-nested-page-name p) p)]
-                                                                  (when (text/namespace-page? p)
-                                                                    (common-util/split-namespace-pages p))))))
-                                                          col)
-                                                  (remove string/blank?)
-                                                  (distinct))
+                              children-pages (when-not db-based?
+                                               (->> (mapcat (fn [p]
+                                                              (let [p (if (map? p)
+                                                                        (:block/original-name p)
+                                                                        p)]
+                                                                (when (string? p)
+                                                                  (let [p (or (text/get-nested-page-name p) p)]
+                                                                    (when (text/namespace-page? p)
+                                                                      (common-util/split-namespace-pages p))))))
+                                                            col)
+                                                    (remove string/blank?)
+                                                    (distinct)))
                               col (->> (distinct (concat col children-pages))
                                        (remove nil?))]
                           (map
@@ -461,57 +470,10 @@
           with-block-refs
           (update :refs (fn [col] (remove nil? col)))))
 
-(defn- with-path-refs
-  [blocks]
-  (loop [blocks blocks
-         acc []
-         parents []]
-    (if (empty? blocks)
-      acc
-      (let [block (first blocks)
-            cur-level (:block/level block)
-            level-diff (- cur-level
-                          (get (last parents) :block/level 0))
-            [path-refs parents]
-            (cond
-              (zero? level-diff)            ; sibling
-              (let [path-refs (mapcat :block/refs (drop-last parents))
-                    parents (conj (vec (butlast parents)) block)]
-                [path-refs parents])
-
-              (> level-diff 0)              ; child
-              (let [path-refs (mapcat :block/refs parents)]
-                [path-refs (conj parents block)])
-
-              (< level-diff 0)              ; new parent
-              (let [parents (vec (take-while (fn [p] (< (:block/level p) cur-level)) parents))
-                    path-refs (mapcat :block/refs parents)]
-                [path-refs (conj parents block)]))
-            path-ref-pages (->> path-refs
-                                (concat (:block/refs block))
-                                (map (fn [ref]
-                                       (cond
-                                         (map? ref)
-                                         (:block/name ref)
-
-                                         :else
-                                         ref)))
-                                (remove string/blank?)
-                                (map (fn [ref]
-                                       (if (string? ref)
-                                         {:block/name (common-util/page-name-sanity-lc ref)}
-                                         ref)))
-                                (remove vector?)
-                                (remove nil?)
-                                (distinct))]
-        (recur (rest blocks)
-               (conj acc (assoc block :block/path-refs path-ref-pages))
-               parents)))))
-
 (defn- macro->block
   "macro: {:name \"\" arguments [\"\"]}"
   [macro]
-  {:db/ident (str (:name macro) " " (string/join " " (:arguments macro)))
+  {:block/uuid (random-uuid)
    :block/type "macro"
    :block/properties {:logseq.macro-name (:name macro)
                       :logseq.macro-arguments (:arguments macro)}})
@@ -567,7 +529,7 @@
                    (select-keys first-block [:block/format :block/page]))
                   blocks)
                  blocks)]
-    (with-path-refs blocks)))
+    blocks))
 
 (defn- with-heading-property
   [properties markdown-heading? size]
@@ -714,7 +676,7 @@
         result (with-pre-block-if-exists blocks body pre-block-properties encoded-content options)]
     (map #(dissoc % :block/meta) result)))
 
-(defn with-parent-and-left
+(defn with-parent-and-order
   [page-id blocks]
   (let [[blocks other-blocks] (split-with
                                (fn [b]
@@ -736,7 +698,6 @@
                            (= level-spaces parent-spaces)        ; sibling
                            (let [block (assoc block
                                               :block/parent parent
-                                              :block/left [:block/uuid uuid]
                                               :block/level level)
                                  parents' (conj (vec (butlast parents)) block)
                                  result' (conj result block)]
@@ -745,9 +706,8 @@
                            (> level-spaces parent-spaces)         ; child
                            (let [parent (if uuid [:block/uuid uuid] (:page/id last-parent))
                                  block (cond->
-                                         (assoc block
-                                                :block/parent parent
-                                                :block/left parent)
+                                        (assoc block
+                                               :block/parent parent)
                                          ;; fix block levels with wrong order
                                          ;; For example:
                                          ;;   - a
@@ -763,10 +723,8 @@
                            (cond
                              (some #(= (:block/level-spaces %) (:block/level-spaces block)) parents) ; outdent
                              (let [parents' (vec (filter (fn [p] (<= (:block/level-spaces p) level-spaces)) parents))
-                                   left (last parents')
                                    blocks (cons (assoc (first blocks)
-                                                       :block/level (dec level)
-                                                       :block/left [:block/uuid (:block/uuid left)])
+                                                       :block/level (dec level))
                                                 (rest blocks))]
                                [blocks parents' result])
 
@@ -776,18 +734,17 @@
                                    parent-id (if-let [block-id (:block/uuid (last f))]
                                                [:block/uuid block-id]
                                                page-id)
-                                   block (cond->
-                                           (assoc block
-                                                  :block/parent parent-id
-                                                  :block/left [:block/uuid (:block/uuid left)]
-                                                  :block/level (:block/level left)
-                                                  :block/level-spaces (:block/level-spaces left)))
+                                   block (assoc block
+                                                :block/parent parent-id
+                                                :block/level (:block/level left)
+                                                :block/level-spaces (:block/level-spaces left))
 
                                    parents' (->> (concat f [block]) vec)
                                    result' (conj result block)]
                                [others parents' result'])))]
-                     (recur blocks parents result))))]
-    (concat result other-blocks)))
+                     (recur blocks parents result))))
+        result' (map (fn [block] (assoc block :block/order (db-order/gen-key))) result)]
+    (concat result' other-blocks)))
 
 (defn extract-plain
   "Extract plain elements including page refs"

deps/graph-parser/src/logseq/graph_parser/db.cljs

@@ -23,7 +23,6 @@
   [title]
   {:block/name (string/lower-case title)
    :block/original-name title
-   :block/journal? false
    :block/uuid (random-uuid)})
 
 (def built-in-pages
@@ -43,7 +42,7 @@
   "Creates default pages if one of the default pages does not exist. This
    fn is idempotent"
   [db-conn]
-  (when-not (d/entity @db-conn [:block/name "card"])
+  (when-not (ldb/get-page @db-conn "card")
     (let [built-in-pages (build-pages-tx built-in-pages)]
       (ldb/transact! db-conn built-in-pages))))
 
@@ -56,6 +55,5 @@
 
 (defn get-page-file
   [db page-name]
-  (some-> (or (d/entity db [:block/name page-name])
-              (d/entity db [:block/original-name page-name]))
+  (some-> (ldb/get-page db page-name)
           :block/file))

deps/graph-parser/src/logseq/graph_parser/exporter.cljs

@@ -19,12 +19,13 @@
             [logseq.db.frontend.rules :as rules]
             [logseq.db.frontend.class :as db-class]
             [logseq.common.util.page-ref :as page-ref]
-            [promesa.core :as p]))
+            [promesa.core :as p]
+            [logseq.db.frontend.order :as db-order]))
 
 (defn- get-pid
   "Get a property's id (name or uuid) given its name. For db graphs"
   [db property-name]
-  (:block/uuid (d/entity db [:block/name (common-util/page-name-sanity-lc (name property-name))])))
+  (:block/uuid (ldb/get-page db property-name)))
 
 (defn- add-missing-timestamps
   "Add updated-at or created-at timestamps if they doesn't exist"
@@ -49,8 +50,7 @@
       (-> tag-block
           add-missing-timestamps
           ;; don't use build-new-class b/c of timestamps
-          (merge {:block/journal? false
-                  :block/format :markdown
+          (merge {:block/format :markdown
                   :block/type "class"})))))
 
 (defn- update-page-tags
@@ -182,9 +182,8 @@
                             ;; don't create different uuids and thus an invalid page
                             (assoc (sqlite-util/build-new-page
                                     (date-time-util/int->journal-title deadline (common-config/get-date-formatter user-config)))
-                                   :block/journal? true
-                                   :block/journal-day deadline
-                                   :block/format :markdown))]
+                                   :block/type "journal"
+                                   :block/journal-day deadline))]
       (-> block
           (update :block/properties assoc deadline-prop (:block/uuid deadline-page))
           (update :block/refs (fnil into []) [:logseq.task/deadline deadline-page])
@@ -203,9 +202,8 @@
                                          db scheduled))
                             (assoc (sqlite-util/build-new-page
                                     (date-time-util/int->journal-title scheduled (common-config/get-date-formatter user-config)))
-                                   :block/journal? true
-                                   :block/journal-day scheduled
-                                   :block/format :markdown))]
+                                   :block/type "journal"
+                                   :block/journal-day scheduled))]
       (-> block
           (update :block/properties assoc scheduled-prop (:block/uuid scheduled-page))
           (update :block/refs (fnil into []) [:logseq.task/scheduled scheduled-page])
@@ -246,7 +244,8 @@
   (let [prop-type (cond (and (coll? prop-val)
                              (seq prop-val)
                              (set/subset? prop-val
-                                          (set (keep #(when (:block/journal? %) (:block/original-name %)) refs))))
+                                          (set (keep #(when (contains? (:block/type %) "journal")
+                                                        (:block/original-name %)) refs))))
                         :date
                         (and (coll? prop-val) (seq prop-val) (text-with-refs? prop-val prop-val-text))
                         :default
@@ -546,10 +545,8 @@
     block))
 
 (defn- fix-pre-block-references
-  [{:block/keys [left parent page] :as block} pre-blocks]
+  [{:block/keys [parent page] :as block} pre-blocks]
   (cond-> block
-    (and (vector? left) (contains? pre-blocks (second left)))
-    (assoc :block/left page)
     ;; Children blocks of pre-blocks get lifted up to the next level which can cause conflicts
     ;; TODO: Detect sibling blocks to avoid parent-left conflicts
     (and (vector? parent) (contains? pre-blocks (second parent)))
@@ -577,7 +574,7 @@
 
 (defn- build-new-page
   [m new-property-schemas tag-classes page-names-to-uuids page-tags-uuid]
-  (-> (merge {:block/journal? false} m)
+  (-> m
       ;; Fix pages missing :block/original-name. Shouldn't happen
       ((fn [m']
          (if-not (:block/original-name m')
@@ -602,7 +599,7 @@
                                      (not (:block/file %))))
                        ;; remove file path relative
                        (map #(dissoc % :block/file)))
-        existing-pages (keep #(d/entity @conn [:block/name (:block/name %)]) all-pages)
+        existing-pages (keep #(ldb/get-page @conn (:block/name %)) all-pages)
         existing-page-names (set (map :block/name existing-pages))
         new-pages (remove #(contains? existing-page-names (:block/name %)) all-pages)
         page-names-to-uuids (into {}
@@ -616,9 +613,9 @@
                           (let [schema (get new-property-schemas (keyword (:block/name %)))
                                 ;; These attributes are not allowed to be transacted because they must not change across files
                                 ;; block/uuid was particularly bad as it actually changed the page's identity across files
-                                disallowed-attributes [:block/name :block/uuid :block/format :block/journal? :block/original-name :block/journal-day
+                                disallowed-attributes [:block/name :block/uuid :block/format :block/original-name :block/journal-day
                                                        :block/created-at :block/updated-at]
-                                allowed-attributes [:block/properties :block/tags :block/alias :block/namespace :class/parent :block/type]
+                                allowed-attributes [:block/properties :block/tags :block/alias :class/parent :block/type :block/namespace]
                                 block-changes (select-keys % allowed-attributes)]
                             (when-let [ignored-attrs (not-empty (apply dissoc % (into disallowed-attributes allowed-attributes)))]
                               (notify-user {:msg (str "Import ignored the following attributes on page " (pr-str (:block/original-name %)) ": "
@@ -748,8 +745,7 @@
                               (filter #(#{"whiteboard" ["whiteboard"]} (:block/type %)))
                               (map (fn [page-block]
                                      (-> page-block
-                                         (assoc :block/journal? false
-                                                :block/format :markdown
+                                         (assoc :block/format :markdown
                                                  ;; fixme: missing properties
                                                 :block/properties {(get-pid @conn :ls-type) :whiteboard-page})))))
         pre-blocks (->> blocks (keep #(when (:block/pre-block? %) (:block/uuid %))) set)
@@ -880,7 +876,7 @@
                     [(get ?bp ?prop-uuid) ?_v]]
                   @conn
                   (set (map :block/name user-classes)))
-             (remove #(ldb/built-in? @conn (d/entity @conn [:block/name (second %)])))
+             (remove #(ldb/built-in? (ldb/get-page @conn (second %))))
              (reduce (fn [acc [class-id _prop-name prop-uuid]]
                        (update acc class-id (fnil conj #{}) prop-uuid))
                      {}))
@@ -925,7 +921,7 @@
                             (merge (ldb/build-favorite-tx favorite-id)
                                    {:block/uuid (d/squuid)
                                     :db/id (or (some-> (:db/id (last acc)) dec) -1)
-                                    :block/left {:db/id (or (:db/id (last acc)) page-id)}
+                                    :block/order (db-order/gen-key nil)
                                     :block/parent page-id
                                     :block/page page-id}))))
                    []
@@ -939,10 +935,10 @@
      (ldb/create-favorites-page repo)
      (if-let [favorited-ids
               (keep (fn [page-name]
-                      (some-> (d/entity @conn [:block/name (common-util/page-name-sanity-lc page-name)])
+                      (some-> (ldb/get-page @conn page-name)
                               :block/uuid))
                     favorites)]
-       (let [page-entity (d/entity @conn [:block/name common-config/favorites-page-name])]
+       (let [page-entity (ldb/get-page @conn common-config/favorites-page-name)]
          (insert-favorites repo favorited-ids (:db/id page-entity)))
        (log-fn :no-favorites-found {:favorites favorites})))))
 
@@ -975,7 +971,7 @@
    * :<save-config-file - fn which saves a config file
    * :<save-logseq-file - fn which saves a logseq file
    * :<copy-asset - fn which copies asset file
-   
+
    Note: See export-doc-files for additional options that are only for it"
   [repo-or-conn conn config-file *files {:keys [<read-file <copy-asset rpath-key log-fn]
                                          :or {rpath-key :path log-fn println}
@@ -1002,4 +998,4 @@
        (export-favorites-from-config-edn conn repo-or-conn config {})
        (export-class-properties conn repo-or-conn)
        {:import-state (:import-state doc-options)
-        :files files}))))
+        :files files}))))

deps/graph-parser/src/logseq/graph_parser/extract.cljc

@@ -16,7 +16,8 @@
             [logseq.common.config :as common-config]
             #?(:org.babashka/nbb [logseq.common.log :as log]
                :default [lambdaisland.glogi :as log])
-            [logseq.graph-parser.whiteboard :as gp-whiteboard]))
+            [logseq.graph-parser.whiteboard :as gp-whiteboard]
+            [logseq.db :as ldb]))
 
 (defn- filepath->page-name
   [filepath]
@@ -216,8 +217,10 @@
                                        :or {extracted-block-ids (atom #{})
                                             resolve-uuid-fn (constantly nil)}
                                        :as options}]
+  (assert db "Datascript DB is required")
   (try
-    (let [page (get-page-name file ast false filename-format)
+    (let [db-based? (ldb/db-based-graph? db)
+          page (get-page-name file ast false filename-format)
           [page page-name _journal-day] (gp-block/convert-page-if-journal page date-formatter)
           options' (assoc options :page-name page-name)
           ;; In case of diff-merge (2way) triggered, use the uuids to override the ones extracted from the AST
@@ -225,24 +228,21 @@
           blocks (->> (gp-block/extract-blocks ast content false format options')
                       (attach-block-ids-if-match override-uuids)
                       (mapv #(gp-block/fix-block-id-if-duplicated! db page-name extracted-block-ids %))
-                      (gp-block/with-parent-and-left {:block/name page-name})
+                      ;; FIXME: use page uuid
+                      (gp-block/with-parent-and-order {:block/name page-name})
                       (vec))
           ref-pages (atom #{})
           blocks (map (fn [block]
                         (if (contains? (:block/type block) "macro")
                           block
-                          (let [block-ref-pages (seq (:block/refs block))
-                                page-lookup-ref [:block/name page-name]
-                                block-path-ref-pages (->> (cons page-lookup-ref (seq (:block/path-refs block)))
-                                                          (remove nil?))]
+                          (let [block-ref-pages (seq (:block/refs block))]
                             (when block-ref-pages
                               (swap! ref-pages set/union (set block-ref-pages)))
                             (-> block
                                 (dissoc :ref-pages)
                                 (assoc :block/format format
                                        :block/page [:block/name page-name]
-                                       :block/refs block-ref-pages
-                                       :block/path-refs block-path-ref-pages)))))
+                                       :block/refs block-ref-pages)))))
                       blocks)
           [properties invalid-properties properties-text-values]
           (if (:block/pre-block? (first blocks))
@@ -251,12 +251,13 @@
              (:block/properties-text-values (first blocks))]
             [properties [] {}])
           page-map (build-page-map properties invalid-properties properties-text-values file page page-name (assoc options' :from-page page))
-          namespace-pages (let [page (:block/original-name page-map)]
-                            (when (text/namespace-page? page)
-                              (->> (common-util/split-namespace-pages page)
-                                   (map (fn [page]
-                                          (-> (gp-block/page-name->map page true db true date-formatter)
-                                              (assoc :block/format format)))))))
+          namespace-pages (when-not db-based?
+                            (let [page (:block/original-name page-map)]
+                              (when (text/namespace-page? page)
+                                (->> (common-util/split-namespace-pages page)
+                                     (map (fn [page]
+                                            (-> (gp-block/page-name->map page true db true date-formatter)
+                                                (assoc :block/format format))))))))
           pages (->> (concat
                       [page-map]
                       @ref-pages
@@ -317,6 +318,7 @@
                 (fn [block]
                   (-> block
                       (common-util/dissoc-in [:block/parent :block/name])
+                      ;; :block/left here for backward compatiblity
                       (common-util/dissoc-in [:block/left :block/name])))
                 blocks)
         serialized-page (first pages)
@@ -334,7 +336,7 @@
         page-block (gp-whiteboard/migrate-page-block page-block)
         blocks (->> blocks
                     (map gp-whiteboard/migrate-shape-block)
-                    (map #(merge % (gp-whiteboard/with-whiteboard-block-props % page-name))))
+                    (map #(merge % (gp-whiteboard/with-whiteboard-block-props % [:block/uuid (:block/uuid page-block)]))))
         _ (when verbose (println "Parsing finished: " file))]
     {:pages (list page-block)
      :blocks blocks}))

deps/graph-parser/src/logseq/graph_parser/property.cljs

@@ -76,7 +76,7 @@
      :id :background-color :heading :collapsed
      :created-at :updated-at :last-modified-at
      :query-table :query-properties :query-sort-by :query-sort-desc :ls-type
-     :hl-type :hl-page :hl-stamp :hl-color :logseq.macro-name :logseq.macro-arguments
+     :hl-type :hl-page :hl-stamp :hl-color
      :logseq.order-list-type :logseq.tldraw.page :logseq.tldraw.shape
      ; task markers
      :todo :doing :now :later :done}

deps/graph-parser/src/logseq/graph_parser/test/docs_graph_helper.cljs

@@ -62,7 +62,7 @@
 (defn- get-journal-page-count [db]
   (->> (d/q '[:find (count ?b)
               :where
-              [?b :block/journal? true]
+              [?b :block/type "journal"]
               [?b :block/name]
               [?b :block/file]]
             db)
@@ -109,12 +109,13 @@
     (is (= {:markdown 6106 :org 509} (get-block-format-counts db))
         "Block format counts")
 
-    (is (= {:description 81, :updated-at 46, :tags 5, :logseq.macro-arguments 104
+    (is (= {:description 81, :updated-at 46, :tags 5,
             :logseq.tldraw.shape 79, :card-last-score 6, :card-repeats 6,
             :card-next-schedule 6, :ls-type 79, :card-last-interval 6, :type 107,
             :template 5, :title 114, :alias 41, :supports 5, :id 145, :url 5,
-            :card-ease-factor 6, :logseq.macro-name 104, :created-at 46,
-            :card-last-reviewed 6, :platforms 51, :initial-version 8, :heading 226}
+            :card-ease-factor 6, :created-at 46,
+            :card-last-reviewed 6, :platforms 51, :initial-version 8, :heading 226
+            :logseq.macro-arguments 109, :logseq.macro-name 109}
            (get-top-block-properties db))
         "Counts for top block properties")
 
@@ -161,13 +162,6 @@
   ;; only increase over time as the docs graph rarely has deletions
   (testing "Counts"
     (is (= 303 (count files)) "Correct file count")
-    (is (= 64375 (count (d/datoms db :eavt))) "Correct datoms count")
-
-    (is (= 5866
-           (ffirst
-            (d/q '[:find (count ?b)
-                   :where [?b :block/path-refs ?bp] [?bp :block/name]] db)))
-        "Correct referenced blocks count")
     (is (= 23
            (ffirst
             (d/q '[:find (count ?b)

deps/graph-parser/src/logseq/graph_parser/text.cljs

@@ -47,14 +47,6 @@
        :else
        (remove-level-space-aux! text block-pattern space? trim-left?)))))
 
-(defn namespace-page?
-  [page-name]
-  (and (string? page-name)
-       (string/includes? page-name "/")
-       (not (string/starts-with? page-name "../"))
-       (not (string/starts-with? page-name "./"))
-       (not (common-util/url? page-name))))
-
 (defn parse-non-string-property-value
   "Return parsed non-string property value or nil if none is found"
   [v]
@@ -156,3 +148,11 @@
           (if-some [new-val (parse-non-string-property-value v')]
             new-val
             v'))))))
+
+(defn namespace-page?
+  [page-name]
+  (and (string? page-name)
+       (string/includes? page-name "/")
+       (not (string/starts-with? page-name "../"))
+       (not (string/starts-with? page-name "./"))
+       (not (common-util/url? page-name))))

deps/graph-parser/src/logseq/graph_parser/whiteboard.cljs

@@ -1,9 +1,7 @@
 (ns logseq.graph-parser.whiteboard
   "Whiteboard related parser utilities"
-  (:require [logseq.common.util :as common-util]
-            [logseq.common.util.block-ref :as block-ref]
-            [logseq.common.util.page-ref :as page-ref]
-            [logseq.db.frontend.property :as db-property]))
+  (:require [logseq.db.frontend.property.util :as db-property-util]
+            [logseq.db.sqlite.util :as sqlite-util]))
 
 (defn block->shape [block]
   (get-in block [:block/properties :logseq.tldraw.shape]))
@@ -42,22 +40,19 @@
 
 (defn- get-shape-refs [shape]
   (let [portal-refs (when (= "logseq-portal" (:type shape))
-                      [(if (= (:blockType shape) "P")
-                         {:block/name (common-util/page-name-sanity-lc (:pageId shape))}
-                         {:block/uuid (uuid (:pageId shape))})])
+                      [{:block/uuid (uuid (:pageId shape))}])
         shape-link-refs (->> (:refs shape)
                              (filter (complement empty?))
-                             (map (fn [ref] (if (parse-uuid ref)
-                                              {:block/uuid (parse-uuid ref)}
-                                              {:block/name (common-util/page-name-sanity-lc ref)}))))]
+                             (keep (fn [ref] (when (parse-uuid ref)
+                                              {:block/uuid (parse-uuid ref)}))))]
     (concat portal-refs shape-link-refs)))
 
 (defn- with-whiteboard-block-refs
-  [shape page-name]
+  [shape page-id]
   (let [refs (or (get-shape-refs shape) [])]
     (merge {:block/refs (if (seq refs) refs [])
             :block/path-refs (if (seq refs)
-                               (conj refs {:block/name page-name})
+                               (conj refs page-id)
                                [])})))
 
 (defn- with-whiteboard-content
@@ -65,37 +60,35 @@
   [shape]
   {:block/content (case (:type shape)
                     "text" (:text shape)
-                    "logseq-portal" (if (= (:blockType shape) "P")
-                                      (page-ref/->page-ref (:pageId shape))
-                                      (block-ref/->block-ref (:pageId shape)))
+                    "logseq-portal" ""
                     "line" (str "whiteboard arrow" (when-let [label (:label shape)] (str ": " label)))
                     (str "whiteboard " (:type shape)))})
 
 (defn with-whiteboard-block-props
   "Builds additional block attributes for a whiteboard block. Expects :block/properties
    to be in file graph format"
-  [block page-name]
+  [block page-id]
   (let [shape? (shape-block? block)
-        shape (block->shape block)
-        default-page-ref {:block/name page-name}]
+        shape (block->shape block)]
     (merge (when shape?
              (merge
               {:block/uuid (uuid (:id shape))}
-              (with-whiteboard-block-refs shape page-name)
+              (with-whiteboard-block-refs shape page-id)
               (with-whiteboard-content shape)))
-           (when (nil? (:block/parent block)) {:block/parent default-page-ref})
+           (when (nil? (:block/parent block)) {:block/parent page-id})
            (when (nil? (:block/format block)) {:block/format :markdown}) ;; TODO: read from config
-           {:block/page default-page-ref})))
+           {:block/page page-id})))
 
-(defn shape->block [repo db shape page-name]
-  (let [properties {(db-property/get-pid repo db :logseq.property/ls-type) :whiteboard-shape
-                    (db-property/get-pid repo db :logseq.property.tldraw/shape) shape}
-        page-name (common-util/page-name-sanity-lc page-name)
-        block {:block/uuid (if (uuid? (:id shape)) (:id shape) (uuid (:id shape)))
-               :block/page {:block/name page-name}
-               :block/parent {:block/name page-name}
-               :block/properties properties}
-        additional-props (with-whiteboard-block-props
-                           (assoc block :block/properties {:ls-type :whiteboard-shape :logseq.tldraw.shape shape})
-                           page-name)]
-    (merge block additional-props)))
+(defn shape->block [repo shape page-id]
+  (let [block-uuid (if (uuid? (:id shape)) (:id shape) (uuid (:id shape)))
+        properties {(db-property-util/get-pid repo :logseq.property/ls-type) :whiteboard-shape
+                    (db-property-util/get-pid repo :logseq.property.tldraw/shape) shape}
+        block {:block/uuid block-uuid
+               :block/content ""
+               :block/page page-id
+               :block/parent page-id}
+        block' (if (sqlite-util/db-based-graph? repo)
+                 (merge block properties)
+                 (assoc block :block/properties properties))
+        additional-props (with-whiteboard-block-props block' page-id)]
+    (merge block' additional-props)))

deps/graph-parser/test/logseq/graph_parser/cli_test.cljs

@@ -9,7 +9,8 @@
             [clojure.set :as set]
             ["fs" :as fs]
             ["process" :as process]
-            ["path" :as path]))
+            ["path" :as path]
+            [logseq.db.frontend.order :as db-order]))
 
 (use-fixtures
   :each
@@ -30,6 +31,9 @@
 
     (docs-graph-helper/docs-graph-assertions @conn graph-dir files)
 
+    (testing "Additional counts"
+      (is (= 48225 (count (d/datoms @conn :eavt))) "Correct datoms count"))
+
     (testing "Asts"
       (is (seq asts) "Asts returned are non-zero")
       (is (= files (map :file asts))
@@ -84,8 +88,8 @@
   (fs/mkdirSync (path/join graph-dir "pages") #js {:recursive true})
   (fs/mkdirSync (path/join graph-dir "journals"))
   (doseq [[page blocks] pages-to-blocks]
-    (fs/writeFileSync (if (:block/journal? page)
-                                ;; Hardcode journal name until more are added
+    (fs/writeFileSync (if (contains? (:block/type page) "journal")
+                        ;; Hardcode journal name until more are added
                         (path/join graph-dir "journals" "2023_07_20.md")
                         (path/join graph-dir "pages" (str (:block/name page) ".md")))
                       (string/join "\n" (map #(str "- " (:block/content %)) blocks))))
@@ -113,9 +117,8 @@
                               {:db/id (new-db-id)
                                :block/uuid (random-uuid)
                                :block/format :markdown
-                               :block/path-refs [{:db/id page-id}]
                                :block/page {:db/id page-id}
-                               :block/left {:db/id page-id}
+                               :block/order (db-order/gen-key nil)
                                :block/parent {:db/id page-id}
                                :block/created-at created-at
                                :block/updated-at created-at})
@@ -147,9 +150,9 @@
   (let [graph-dir "tmp/file-and-db-graph"
         ;; pages and their blocks which are being tested
         pages-to-blocks
-        {{:block/name "page1" :block/journal? false}
+        {{:block/name "page1"}
          [{:block/content "block 1"} {:block/content "block 2"}]
-         {:block/name "jul 20th, 2023" :block/journal? true :block/journal-day 20230720}
+         {:block/name "jul 20th, 2023" :block/type #{"journal"} :block/journal-day 20230720}
          [{:block/content "b1"} {:block/content "b2"}]}
         file-db (create-file-db graph-dir pages-to-blocks)
         graph-db (create-graph-db "tmp" "file-and-db-graph" pages-to-blocks)

deps/graph-parser/test/logseq/graph_parser/extract_test.cljs

@@ -1,7 +1,8 @@
 (ns logseq.graph-parser.extract-test
   (:require [cljs.test :refer [deftest is are]]
             [logseq.graph-parser.extract :as extract]
-            [clojure.pprint :as pprint]))
+            [datascript.core :as d]
+            [logseq.db.frontend.schema :as db-schema]))
 
 ;; This is a copy of frontend.util.fs/multiplatform-reserved-chars for reserved chars testing
 (def multiplatform-reserved-chars ":\\*\\?\"<>|\\#\\\\")
@@ -41,28 +42,29 @@
   (is (= "asldk lakls" (#'extract/path->file-body "file://data/app/asldk lakls.as")))
   (is (= "中文asldk lakls" (#'extract/path->file-body "file://中文data/app/中文asldk lakls.as"))))
 
-(defn- extract
+(defn- extract [file content & [options]]
+  (extract/extract file
+                   content
+                   (merge {:block-pattern "-" :db (d/empty-db db-schema/schema)}
+                          options)))
+
+(defn- extract-block-content
   [text]
-  (let [{:keys [blocks]} (extract/extract "a.md" text {:block-pattern "-"})
-        lefts (map (juxt :block/parent :block/left) blocks)]
-    (if (not= (count lefts) (count (distinct lefts)))
-      (do
-        (pprint/pprint (map (fn [x] (select-keys x [:block/uuid :block/level :block/content :block/left])) blocks))
-        (throw (js/Error. ":block/parent && :block/left conflicts")))
-      (mapv :block/content blocks))))
+  (let [{:keys [blocks]} (extract "a.md" text)]
+    (mapv :block/content blocks)))
 
 (defn- extract-title [file text]
-  (-> (extract/extract file text {}) :pages first :block/properties :title))
+  (-> (extract file text) :pages first :block/properties :title))
 
 (deftest extract-blocks-for-headings
   (is (= ["a" "b" "c"]
-         (extract
+         (extract-block-content
           "- a
   - b
     - c")))
 
   (is (= ["## hello" "world" "nice" "nice" "bingo" "world"]
-         (extract "## hello
+         (extract-block-content "## hello
     - world
       - nice
         - nice
@@ -70,7 +72,7 @@
       - world")))
 
   (is (= ["# a" "## b" "### c" "#### d" "### e" "f" "g" "h" "i" "j"]
-       (extract "# a
+         (extract-block-content "# a
 ## b
 ### c
 #### d
@@ -99,26 +101,25 @@
          (extract-title "foo.org" ":PROPERTIES:
 :ID:       72289d9a-eb2f-427b-ad97-b605a4b8c59b
 :END:
-#+title: diagram/abcdef")))
-)
+#+title: diagram/abcdef"))))
 
 (deftest extract-blocks-with-property-pages-config
   (are [extract-args expected-refs]
        (= expected-refs
-          (->> (apply extract/extract extract-args)
+          (->> (apply extract extract-args)
                :blocks
                (mapcat #(->> % :block/refs (map :block/name)))
                set))
 
-       ["a.md" "foo:: #bar\nbaz:: #bing" {:block-pattern "-" :user-config {:property-pages/enabled? true}}]
-       #{"bar" "bing" "foo" "baz"}
+    ["a.md" "foo:: #bar\nbaz:: #bing" {:user-config {:property-pages/enabled? true}}]
+    #{"bar" "bing" "foo" "baz"}
 
-       ["a.md" "foo:: #bar\nbaz:: #bing" {:block-pattern "-" :user-config {:property-pages/enabled? false}}]
-       #{"bar" "bing"}))
+    ["a.md" "foo:: #bar\nbaz:: #bing" {:user-config {:property-pages/enabled? false}}]
+    #{"bar" "bing"}))
 
 (deftest test-regression-1902
   (is (= ["line1" "line2" "line3" "line4"]
-         (extract
+         (extract-block-content
           "- line1
     - line2
       - line3
@@ -134,6 +135,7 @@
     :pages
     ({:block/format :markdown,
       :block/original-name "Foo"
+      :block/uuid #uuid "a846e3b4-c41d-4251-80e1-be6978c36d8c"
       :block/properties {:title "my whiteboard foo"}})})
 
 (deftest test-extract-whiteboard-edn
@@ -143,4 +145,4 @@
     (is (= (:block/name page) "foo"))
     (is (= (:block/type page) "whiteboard"))
     (is (= (:block/original-name page) "Foo"))
-    (is (every? #(= (:block/parent %) {:block/name "foo"}) blocks))))
+    (is (every? #(= (:block/parent %) [:block/uuid #uuid "a846e3b4-c41d-4251-80e1-be6978c36d8c"]) blocks))))

deps/graph-parser/test/logseq/graph_parser_test.cljs

@@ -5,7 +5,8 @@
             [logseq.graph-parser.db :as gp-db]
             [logseq.graph-parser.block :as gp-block]
             [logseq.graph-parser.property :as gp-property]
-            [datascript.core :as d]))
+            [datascript.core :as d]
+            [logseq.db :as ldb]))
 
 (def foo-edn
   "Example exported whiteboard page as an edn exportable."
@@ -414,7 +415,7 @@ id:: 63f199bc-c737-459f-983d-84acfcda14fe
 id:: 63f199bc-c737-459f-983d-84acfcda14fe
 "
                                parse-opts)
-      (let [blocks (:block/_parent (d/entity @conn [:block/name "foo"]))]
+      (let [blocks (:block/_parent (ldb/get-page @conn "foo"))]
         (is (= 2 (count blocks)))
         (is (= 1 (count (filter #(= (:block/uuid %) block-id) blocks)))))))
 
@@ -442,6 +443,6 @@ bar
              (-> (d/entity @conn [:block/uuid block-id])
                  :block/page
                  :block/name)))
-      (let [bar-block (first (:block/_parent (d/entity @conn [:block/name "bar"])))]
+      (let [bar-block (first (:block/_parent (ldb/get-page @conn "bar")))]
         (is (some? (:block/uuid bar-block)))
         (is (not= (:block/uuid bar-block) block-id))))))

deps/graph-parser/yarn.lock

@@ -2,9 +2,9 @@
 # yarn lockfile v1
 
 
-"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v10":
-  version "1.2.173-feat-db-v10"
-  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/8a4bfaae55d85292476ac8f976516b86e6ae4036"
+"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v12":
+  version "1.2.173-feat-db-v12"
+  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/8274d8b40b7fec84532c54e16b5a8c6f78bf673b"
   dependencies:
     import-meta-resolve "^2.1.0"
 

deps/outliner/.carve/ignore

@@ -1,5 +1,5 @@
 ;; API fn
-logseq.outliner.cli.pipeline/add-listener
+logseq.outliner.db-pipeline/add-listener
 ;; private
 logseq.outliner.core/*transaction-opts*
 ;; API fn

deps/outliner/deps.edn

@@ -1,11 +1,11 @@
 {:deps
  ;; External deps should be kept in sync with https://github.com/logseq/nbb-logseq/blob/main/bb.edn
  {datascript/datascript {:git/url "https://github.com/logseq/datascript" ;; fork
-                         :sha     "fc3645fa4a6b8160e087e2ad1ce1c242675680cb"}
+                         :sha     "aa2f963be5d507b55d05c4d9a6ee3ce9454c4415"}
   logseq/db             {:local/root "../db"}
   logseq/graph-parser   {:local/root "../db"}
   com.cognitect/transit-cljs {:mvn/version "0.8.280"}
-  metosin/malli {:mvn/version "0.10.0"}}
+  metosin/malli {:mvn/version "0.16.1"}}
  :aliases
  {:clj-kondo
   {:replace-deps {clj-kondo/clj-kondo {:mvn/version "2023.05.26"}}

deps/outliner/nbb.edn

@@ -5,6 +5,6 @@
   logseq/graph-parser
   {:local/root "../graph-parser"}
   metosin/malli
-  {:mvn/version "0.10.0"}
+  {:mvn/version "0.16.1"}
   io.github.nextjournal/nbb-test-runner
   {:git/sha "60ed57aa04bca8d604f5ba6b28848bd887109347"}}}

deps/outliner/package.json

@@ -3,7 +3,7 @@
   "version": "1.0.0",
   "private": true,
   "devDependencies": {
-    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v10"
+    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v12"
   },
   "dependencies": {
     "better-sqlite3": "9.3.0",

deps/outliner/script/transact.cljs

@@ -1,6 +1,6 @@
 (ns transact
   "This script generically runs transactions against the queried blocks"
-  (:require [logseq.outliner.cli.pipeline :as cli-pipeline]
+  (:require [logseq.outliner.db-pipeline :as db-pipeline]
             [logseq.db.sqlite.db :as sqlite-db]
             [logseq.db.frontend.rules :as rules]
             [datascript.core :as d]
@@ -33,9 +33,9 @@
           (println "With the following blocks updated:")
           (prn (map #(select-keys (d/entity @conn %) [:block/name :block/content]) blocks-to-update)))
       (do
-        (cli-pipeline/add-listener conn)
+        (db-pipeline/add-listener conn)
         (d/transact! conn update-tx)
         (println "Updated" (count update-tx) "block(s) for graph" (str db-name "!"))))))
 
 (when (= nbb/*file* (:file (meta #'-main)))
-  (-main *command-line-args*))
+  (-main *command-line-args*))

deps/outliner/src/logseq/outliner/cli/pipeline.cljs

@@ -1,23 +0,0 @@
-(ns ^:node-only logseq.outliner.cli.pipeline
-  "This ns provides a datascript listener for DB graphs to add additional changes
-   that the frontend also adds per transact.
-   Known limitations:
-   * Deleted blocks don't update effected :block/tx-id"
-  (:require [datascript.core :as d]
-            [logseq.outliner.datascript-report :as ds-report]
-            [logseq.outliner.pipeline :as outliner-pipeline]))
-
-(defn- invoke-hooks
-  "Modified copy of frontend.modules.outliner.pipeline/invoke-hooks that doesn't
-  handle :block/tx-id"
-  [conn tx-report]
-  (when (not (get-in tx-report [:tx-meta :replace?]))
-    (let [{:keys [blocks]} (ds-report/get-blocks-and-pages tx-report)
-          block-path-refs-tx (outliner-pipeline/compute-block-path-refs-tx tx-report blocks)]
-      (d/transact! conn block-path-refs-tx {:replace? true}))))
-
-(defn add-listener
-  "Adds a listener to the datascript connection to add additional changes from outliner.pipeline"
-  [conn]
-  (d/listen! conn :pipeline-updates (fn pipeline-updates [tx-report]
-                                      (invoke-hooks conn tx-report))))

deps/outliner/src/logseq/outliner/datascript_report.cljs

@@ -42,17 +42,4 @@
                            (set))]
     (if (seq tx-meta-pages)
       (update result :pages set/union tx-meta-pages)
-      result)))
-
-(defn get-blocks
-  [{:keys [db-before db-after tx-data] :as _tx-report}]
-  (let [updated-db-ids (-> (mapv first tx-data) (set))]
-    (reduce
-      (fn [acc x]
-        (let [block-entity
-              (get-entity-from-db-after-or-before db-before db-after x)]
-          (cond-> acc
-            (some? block-entity)
-            (conj block-entity))))
-      []
-      updated-db-ids)))
+      result)))

deps/outliner/src/logseq/outliner/db_pipeline.cljs

@@ -0,0 +1,25 @@
+(ns ^:node-only logseq.outliner.db-pipeline
+  "This ns provides a datascript listener for DB graphs to add additional changes
+   that the frontend also adds per transact.
+   Missing features from frontend.worker.pipeline including:
+   * Deleted blocks don't update effected :block/tx-id
+   * Delete empty property parent"
+  (:require [datascript.core :as d]
+            [logseq.outliner.pipeline :as outliner-pipeline]
+            [logseq.outliner.datascript-report :as ds-report]))
+
+(defn- invoke-hooks
+  "Modified copy of frontend.worker.pipeline/invoke-hooks that doesn't
+  handle :block/tx-id"
+  [conn tx-report]
+  (when (not (get-in tx-report [:tx-meta :pipeline-replace?]))
+    (let [{:keys [blocks]} (ds-report/get-blocks-and-pages tx-report)
+          block-path-refs-tx (distinct (outliner-pipeline/compute-block-path-refs-tx tx-report blocks))]
+      (when (seq block-path-refs-tx)
+        (d/transact! conn block-path-refs-tx {:pipeline-replace? true})))))
+
+(defn add-listener
+  "Adds a listener to the datascript connection to add additional changes from outliner.pipeline"
+  [conn]
+  (d/listen! conn :pipeline-updates (fn pipeline-updates [tx-report]
+                                      (invoke-hooks conn tx-report))))

deps/outliner/src/logseq/outliner/pipeline.cljs

@@ -1,5 +1,5 @@
 (ns logseq.outliner.pipeline
-  "Core fns for use with frontend.modules.outliner.pipeline"
+  "Core fns for use with frontend worker and node"
   (:require [datascript.core :as d]
             [clojure.set :as set]
             [logseq.db :as ldb]))
@@ -77,7 +77,8 @@
             blocks)))
 
 (defn compute-block-path-refs-tx
+  "Main fn for computing path-refs"
   [tx-report blocks]
   (let [refs-tx (compute-block-path-refs tx-report blocks)
         truncate-refs-tx (map (fn [m] [:db/retract (:db/id m) :block/path-refs]) refs-tx)]
-    (concat truncate-refs-tx refs-tx)))
+    (concat truncate-refs-tx refs-tx)))

deps/outliner/src/logseq/outliner/tree.cljs

@@ -1,57 +1,55 @@
 (ns logseq.outliner.tree
   "Provides tree fns and INode protocol"
   (:require [logseq.db :as ldb]
-            [clojure.string :as string]
-            [logseq.db.frontend.property :as db-property]
+            [logseq.db.frontend.property.util :as db-property-util]
             [datascript.core :as d]))
 
 (defprotocol INode
-  (-get-id [this conn])
-  (-get-parent-id [this conn])
-  (-get-left-id [this conn])
-  (-set-left-id [this left-id conn])
-  (-get-parent [this conn])
-  (-get-left [this conn])
-  (-get-right [this conn])
-  (-get-down [this conn])
   (-save [this txs-state conn repo date-formatter opts])
-  (-del [this db conn])
-  (-get-children [this conn]))
-
-(defn satisfied-inode?
-  [node]
-  (satisfies? INode node))
+  (-del [this db conn]))
 
 (defn- blocks->vec-tree-aux
   [repo db blocks root]
   (let [root-id (:db/id root)
-        blocks (remove #(db-property/shape-block? repo db %) blocks)
+        blocks (remove #(db-property-util/shape-block? repo db %) blocks)
         parent-blocks (group-by #(get-in % [:block/parent :db/id]) blocks) ;; exclude whiteboard shapes
         sort-fn (fn [parent]
-                  (ldb/sort-by-left (get parent-blocks parent) {:db/id parent}))
+                  (when-let [children (get parent-blocks parent)]
+                    (ldb/sort-by-order children)))
         block-children (fn block-children [parent level]
                          (map (fn [m]
                                 (let [id (:db/id m)
                                       children (-> (block-children id (inc level))
-                                                   (ldb/sort-by-left m))]
+                                                   (ldb/sort-by-order))]
                                   (assoc m
                                          :block/level level
                                          :block/children children)))
-                           (sort-fn parent)))]
+                              (sort-fn parent)))]
     (block-children root-id 1)))
 
 (defn- get-root-and-page
   [db root-id]
-  (if (string? root-id)
+  (cond
+    (uuid? root-id)
+    (let [e (d/entity db [:block/uuid root-id])]
+      (if (ldb/page? e) [true e] [false e]))
+
+    (number? root-id)
+    (let [e (d/entity db root-id)]
+      (if (ldb/page? e) [true e] [false e]))
+
+    (string? root-id)
     (if-let [id (parse-uuid root-id)]
       [false (d/entity db [:block/uuid id])]
-      [true (d/entity db [:block/name (string/lower-case root-id)])])
+      [true (ldb/get-page db root-id)])
+
+    :else
     [false root-id]))
 
 (defn blocks->vec-tree
   "`blocks` need to be in the same page."
   [repo db blocks root-id]
-  (let [[page? root] (get-root-and-page db (str root-id))]
+  (let [[page? root] (get-root-and-page db root-id)]
     (if-not root ; custom query
       blocks
       (let [result (blocks->vec-tree-aux repo db blocks root)]
@@ -73,7 +71,7 @@
                             b')))
                       (let [parent {:db/id parent-id}]
                         (-> (get parent->children parent)
-                            (ldb/try-sort-by-left parent)))))
+                            (ldb/sort-by-order)))))
         children (nodes root-id 1)
         root' (assoc root :block/level (or default-level 1))]
     (if (seq children)
@@ -86,8 +84,6 @@
            :block/uuid (:block/uuid e)
            :block/parent {:db/id (:db/id (:block/parent e))}
            :block/page (:block/page e)}
-    (:db/id (:block/left e))
-    (assoc :block/left {:db/id (:db/id (:block/left e))})
     (:block/refs e)
     (assoc :block/refs (:block/refs e))
     (:block/children e)
@@ -107,7 +103,7 @@
   ([blocks default-level]
    (let [blocks (map block-entity->map blocks)
          top-level-blocks (filter-top-level-blocks blocks)
-         top-level-blocks' (ldb/try-sort-by-left top-level-blocks (:block/parent (first top-level-blocks)))
+         top-level-blocks' (ldb/sort-by-order top-level-blocks)
          parent->children (group-by :block/parent blocks)]
      (map #(tree parent->children % (or default-level 1)) top-level-blocks'))))
 
@@ -115,7 +111,7 @@
   [parents parent-groups]
   (mapv (fn [parent]
           (let [parent-id {:db/id (:db/id parent)}
-                children (ldb/sort-by-left (get @parent-groups parent-id) parent)
+                children (ldb/sort-by-order (get @parent-groups parent-id))
                 _ (swap! parent-groups #(dissoc % parent-id))
                 sorted-nested-children (when (not-empty children) (sort-blocks-aux children parent-groups))]
                     (if sorted-nested-children [parent sorted-nested-children] [parent])))

deps/outliner/src/logseq/outliner/util.cljs

@@ -1,42 +0,0 @@
-(ns logseq.outliner.util
-  "Util fns for outliner"
-  (:require
-   #_:clj-kondo/ignore
-   ;; db needs to load before entity for nbb
-   [datascript.db]
-   [datascript.impl.entity :as e]
-   [logseq.common.util :as common-util]))
-
-(defn- block-id?
-  [id]
-  (or
-   (number? id)
-   (string? id)
-   (uuid? id)))
-
-(defn check-block-id
-  [id]
-  (assert (block-id? id)
-          (common-util/format "The id should match block-id?: %s" (pr-str id))))
-
-(defn ->block-lookup-ref
-  "
-  string? or number?  -> [:block/uuid x]
-  [:block/uuid x] -> [:block/uuid x]
-  {:db/id x} -> {:db/id x}
-  :else -> nil
-  "
-  [id]
-  (cond
-    (and
-     (vector? id)
-     (= (first id) :block/uuid))
-    id
-
-    (block-id? id)
-    [:block/uuid id]
-
-    (or (e/entity? id) (map? id))
-    id
-
-    :else nil))

deps/outliner/yarn.lock

@@ -2,9 +2,9 @@
 # yarn lockfile v1
 
 
-"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v10":
-  version "1.2.173-feat-db-v10"
-  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/8a4bfaae55d85292476ac8f976516b86e6ae4036"
+"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v12":
+  version "1.2.173-feat-db-v12"
+  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/8274d8b40b7fec84532c54e16b5a8c6f78bf673b"
   dependencies:
     import-meta-resolve "^2.1.0"
 

deps/publishing/package.json

@@ -3,7 +3,7 @@
   "version": "1.0.0",
   "private": true,
   "devDependencies": {
-    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v10",
+    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v12",
     "mldoc": "^1.5.8"
   },
   "dependencies": {

deps/publishing/src/logseq/publishing/db.cljs

@@ -3,22 +3,25 @@
   (:require [datascript.core :as d]
             [logseq.db.frontend.rules :as rules]
             [clojure.set :as set]
-            [clojure.string :as string]))
+            [clojure.string :as string]
+            [logseq.db.frontend.entity-plus :as entity-plus]))
 
 (defn ^:api get-area-block-asset-url
   "Returns asset url for an area block used by pdf assets. This lives in this ns
   because it is used by this dep and needs to be independent from the frontend app"
-  ([block page] (get-area-block-asset-url block page {}))
-  ;; TODO: Add prop-lookup-fn support for db graphs and commandline publishing
-  ([block page {:keys [prop-lookup-fn] :or {prop-lookup-fn get}}]
-   (when-some [props (and block page (:block/properties block))]
-     (when-some [uuid (:block/uuid block)]
-       (when-some [stamp (prop-lookup-fn props :hl-stamp)]
-         (let [group-key      (string/replace-first (:block/original-name page) #"^hls__" "")
-               hl-page        (prop-lookup-fn props :hl-page)
-               encoded-chars? (boolean (re-find #"(?i)%[0-9a-f]{2}" group-key))
-               group-key      (if encoded-chars? (js/encodeURI group-key) group-key)]
-           (str "./assets/" group-key "/" (str hl-page "_" uuid "_" stamp ".png"))))))))
+  [db block page]
+  (when-some [props (and block page (:block/properties block))]
+    ;; Can't use db-property-util/lookup b/c repo isn't available
+    (let [prop-lookup-fn (if (entity-plus/db-based-graph? db)
+                           #(get %1 %2)
+                           #(get %1 (keyword (name %2))))]
+      (when-some [uuid (:block/uuid block)]
+        (when-some [stamp (prop-lookup-fn props :logseq.property/hl-stamp)]
+          (let [group-key      (string/replace-first (:block/original-name page) #"^hls__" "")
+                hl-page        (prop-lookup-fn props :logseq.property/hl-page)
+                encoded-chars? (boolean (re-find #"(?i)%[0-9a-f]{2}" group-key))
+                group-key      (if encoded-chars? (js/encodeURI group-key) group-key)]
+            (str "./assets/" group-key "/" (str hl-page "_" uuid "_" stamp ".png"))))))))
 
 (defn- clean-asset-path-prefix
   [path]
@@ -45,7 +48,7 @@
   (let [pages (->> (d/q
                     '[:find ?p
                       :in $ %
-                      :where (page-property ?p :public true)]
+                      :where (page-property ?p :logseq.property/public true)]
                     db
                     (rules/extract-rules rules/db-query-dsl-rules [:page-property]))
                    (map first)
@@ -55,13 +58,10 @@
         tag-pages (concat tag-pages*
                           ;; built-in property needs to be public to display tags
                           (when (seq tag-pages*)
-                            (some-> (d/entity db :logseq.property/tags) :db/id vector)))
+                            (some-> (d/entity db :block/tags) :db/id vector)))
         property-pages (mapcat (fn [ent]
-                                 (let [props (:block/properties ent)]
-                                   (->> (keys props)
-                                        (into (mapcat #(filter uuid? (if (coll? %) % [%]))
-                                                      (vals props)))
-                                        (map #(:db/id (d/entity db [:block/uuid %]))))))
+                                 (->> (keys (:block/properties ent))
+                                      (map #(:db/id (d/entity db %)))))
                                page-ents)]
     (concat pages tag-pages property-pages)))
 
@@ -70,7 +70,7 @@
   (->> (d/q
         '[:find ?p
           :in $ %
-          :where (page-property ?p :public false)]
+          :where (page-property ?p :logseq.property/public false)]
         db
         (rules/extract-rules rules/db-query-dsl-rules [:page-property]))
        (map first)
@@ -101,7 +101,6 @@
         db)
        (map first)))
 
-;; FIXME: store assets as blocks for db-based graphs
 (defn- get-assets
   [db datoms]
   (let [pull (fn [eid db]
@@ -111,7 +110,15 @@
                    (pull % db)
                    :block/page
                    :db/id
-                   (pull db)))]
+                   (pull db)))
+        hl-type-area? (if (entity-plus/db-based-graph? db)
+                        (fn [datom]
+                          (and (= :logseq.property/hl-type (:a datom))
+                               (= (keyword (:v datom)) :area)))
+                        (fn [datom]
+                          (and
+                           (= :block/properties (:a datom))
+                           (= (keyword (get (:v datom) :hl-type)) :area))))]
     (->>
      (keep
       (fn [datom]
@@ -124,12 +131,10 @@
                                    (not (string/ends-with? path ".js")))
                           path)))))
           ;; area image assets
-          ;; FIXME: Lookup by property uuid
-          (and
-           (= :block/properties (:a datom))
-           (= (keyword (get (:v datom) :hl-type)) :area))
+          (hl-type-area? datom)
           (#(let [path (some-> (pull (:e datom) db)
                                (get-area-block-asset-url
+                                db
                                 (get-page-by-eid (:e datom))))
                   path (clean-asset-path-prefix path)]
               (conj % path)))))
@@ -166,6 +171,7 @@
                                        (not (contains? non-public-datom-ids (:e datom)))))))
         datoms (d/datoms filtered-db :eavt)
         assets (get-assets db datoms)]
+    ;; (prn :datoms (count datoms) :assets (count assets))
     [@(d/conn-from-datoms datoms (:schema db)) assets]))
 
 (defn filter-only-public-pages-and-blocks
@@ -190,4 +196,5 @@
                                           (contains? public-pages (:db/id (:block/page (d/entity db (:e datom))))))))))))
         datoms (d/datoms filtered-db :eavt)
         assets (get-assets db datoms)]
+    ;; (prn :datoms (count datoms) :assets (count assets))
     [@(d/conn-from-datoms datoms (:schema db)) assets]))

deps/publishing/test/logseq/publishing/db_test.cljs

@@ -4,7 +4,8 @@
             [logseq.publishing.db :as publish-db]
             [logseq.graph-parser :as graph-parser]
             [datascript.core :as d]
-            [logseq.graph-parser.db :as gp-db]))
+            [logseq.graph-parser.db :as gp-db]
+            [logseq.db :as ldb]))
 
 (deftest clean-export!
   (let [conn (gp-db/start-conn)
@@ -56,8 +57,8 @@
         "Contains all pages that have been marked public")
     (is (not (contains? exported-pages "page1"))
         "Doesn't contain private page")
-    (is (seq (d/entity filtered-db [:block/name "page2-alias"]))
-          "Alias of public page is exported")
+    (is (seq (ldb/get-page filtered-db "page2-alias"))
+        "Alias of public page is exported")
     (is (= #{"page2" "page3"} exported-block-pages)
         "Only exports blocks from public pages")
     (is (= ["thumb-on-fire_1648822523866_0.PNG"] assets)

deps/publishing/yarn.lock

@@ -2,9 +2,9 @@
 # yarn lockfile v1
 
 
-"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v10":
-  version "1.2.173-feat-db-v10"
-  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/8a4bfaae55d85292476ac8f976516b86e6ae4036"
+"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v12":
+  version "1.2.173-feat-db-v12"
+  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/8274d8b40b7fec84532c54e16b5a8c6f78bf673b"
   dependencies:
     import-meta-resolve "^2.1.0"
 

deps/shui/deps.edn

@@ -1,7 +1,7 @@
 {:paths ["src"]
  :deps
  {org.clojure/clojure                   {:mvn/version "1.11.1"}
-  org.clojure/clojurescript             {:mvn/version "1.11.54"}
+  org.clojure/clojurescript             {:mvn/version "1.11.132"}
   funcool/promesa                       {:mvn/version "4.0.2"}
   rum/rum                               {:mvn/version "0.12.9"}
   medley/medley                         {:mvn/version "1.4.0"}

deps/shui/shui-graph/logseq/config.edn

@@ -177,7 +177,6 @@
             [?h :block/marker ?marker]
             [(contains? #{"NOW" "DOING"} ?marker)]
             [?h :block/page ?p]
-            [?p :block/journal? true]
             [?p :block/journal-day ?d]
             [(>= ?d ?start)]
             [(<= ?d ?today)]]
@@ -193,7 +192,6 @@
             [?h :block/marker ?marker]
             [(contains? #{"NOW" "LATER" "TODO"} ?marker)]
             [?h :block/page ?p]
-            [?p :block/journal? true]
             [?p :block/journal-day ?d]
             [(> ?d ?start)]
             [(< ?d ?next)]]

docs/dev-practices.md

@@ -365,7 +365,7 @@ These tasks are specific to database graphs. For these tasks there is a one time
   ```sh
   $ bb dev:db-query woot '[:find (pull ?b [*]) :where (block-content ?b "Dogma")]'
   DB contains 833 datoms
-  [{:block/tx-id 536870923, :block/link #:db{:id 100065}, :block/uuid #uuid "65565c26-f972-4400-bce4-a15df488784d", :block/updated-at 1700158508564, :block/left #:db{:id 100051}, :block/refs [#:db{:id 100064}], :block/created-at 1700158502056, :block/format :markdown, :block/tags [#:db{:id 100064}], :block/content "Dogma #~^65565c2a-b1c5-4dc8-a0f0-81b786bc5c6d", :db/id 100090, :block/path-refs [#:db{:id 100051} #:db{:id 100064}], :block/parent #:db{:id 100051}, :block/page #:db{:id 100051}}]
+  [{:block/tx-id 536870923, :block/link #:db{:id 100065}, :block/uuid #uuid "65565c26-f972-4400-bce4-a15df488784d", :block/updated-at 1700158508564, :block/order "a0", :block/refs [#:db{:id 100064}], :block/created-at 1700158502056, :block/format :markdown, :block/tags [#:db{:id 100064}], :block/content "Dogma #~^65565c2a-b1c5-4dc8-a0f0-81b786bc5c6d", :db/id 100090, :block/path-refs [#:db{:id 100051} #:db{:id 100064}], :block/parent #:db{:id 100051}, :block/page #:db{:id 100051}}]
   ```
 
 * `dev:db-transact` - Run a `d/transact!` against the queried results of a DB graph
@@ -420,7 +420,7 @@ These tasks are specific to database graphs. For these tasks there is a one time
     #uuid "6581c8db-a2a2-4e09-b30d-cdea6ad69512"
     536871037
     true]]]
-  
+
   # By default this task ignores commonly changing datascript attributes.
   # To see all changed attributes, tell the task to ignore a nonexistent attribute:
   $ bb dev:diff-datoms w2.edn w3.edn -i a
@@ -439,7 +439,7 @@ These tasks are specific to database graphs. For these tasks there is a one time
     [162 :block/content "b7" 536871039 true]
     [162 :block/created-at 1703004379103 536871037 true]
     [162 :block/format :markdown 536871037 true]
-    [162 :block/left 149 536871037 true]
+    [162 :block/order "a0" 536871037 true]
     [162 :block/page 149 536871037 true]
     [162 :block/parent 149 536871037 true]
     [162 :block/path-refs 108 536871044 true]

scripts/README.md

@@ -65,4 +65,4 @@ Created graph schema!
 #### Update graph scripts
 
 For database graphs, it is recommended to use
-`logseq.outliner.cli.pipeline/add-listener!` when updating graphs.  TODO
+`logseq.outliner.db-pipeline/add-listener!` when updating graphs.  TODO

scripts/package.json

@@ -3,7 +3,7 @@
   "version": "1.0.0",
   "private": true,
   "devDependencies": {
-    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v10"
+    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v12"
   },
   "dependencies": {
     "better-sqlite3": "9.3.0",

scripts/src/logseq/tasks/db_graph/create_graph.cljs

@@ -6,15 +6,17 @@
   (:require [logseq.db.sqlite.db :as sqlite-db]
             [logseq.db.sqlite.util :as sqlite-util]
             [logseq.db.sqlite.create-graph :as sqlite-create-graph]
-            [logseq.db.frontend.property.util :as db-property-util]
-            [logseq.outliner.cli.pipeline :as cli-pipeline]
+            [logseq.db.frontend.property.build :as db-property-build]
+            [logseq.outliner.db-pipeline :as db-pipeline]
             [logseq.common.util :as common-util]
-            [logseq.db :as ldb]
             [clojure.string :as string]
+            [clojure.set :as set]
             [datascript.core :as d]
             ["fs" :as fs]
             ["path" :as node-path]
-            [nbb.classpath :as cp]))
+            [nbb.classpath :as cp]
+            [logseq.db.frontend.property :as db-property]
+            [logseq.db.frontend.order :as db-order]))
 
 (defn- find-on-classpath [rel-path]
   (some (fn [dir]
@@ -34,7 +36,7 @@
           (string/replace-first #"(:file/name-format :triple-lowbar)"
                                 (str "$1 "
                                      (string/replace-first (str additional-config) #"^\{(.*)\}$" "$1"))))]
-    (d/transact! conn (sqlite-create-graph/build-db-initial-data @conn config-content))))
+    (d/transact! conn (sqlite-create-graph/build-db-initial-data config-content))))
 
 (defn init-conn
   "Create sqlite DB, initialize datascript connection and sync listener and then
@@ -43,51 +45,50 @@
   (fs/mkdirSync (node-path/join dir db-name) #js {:recursive true})
   ;; Same order as frontend.db.conn/start!
   (let [conn (sqlite-db/open-db! dir db-name)]
-    (cli-pipeline/add-listener conn)
+    (db-pipeline/add-listener conn)
     (setup-init-data conn additional-config)
     conn))
 
 (defn- translate-property-value
-  "Translates a property value as needed. A value wrapped in vector indicates a reference type
-   e.g. [:page \"some page\"]"
+  "Translates a property value for create-graph edn. A value wrapped in vector
+  may indicate a reference type e.g. [:page \"some page\"]"
   [val {:keys [page-uuids block-uuids]}]
   (if (vector? val)
     (case (first val)
+      ;; Converts a page name to block/uuid
       :page
-      (or (page-uuids (second val))
-          (throw (ex-info (str "No uuid for page '" (second val) "'") {:name (second val)})))
+      (if-let [page-uuid (page-uuids (second val))]
+        [:block/uuid page-uuid]
+        (throw (ex-info (str "No uuid for page '" (second val) "'") {:name (second val)})))
+      :block/uuid
+      val
+      ;; TODO: If not used by :default and replace uuid-maps with just page-uuids everywhere
       :block
       (or (block-uuids (second val))
           (throw (ex-info (str "No uuid for block '" (second val) "'") {:name (second val)})))
       (throw (ex-info "Invalid property value type. Valid values are :block and :page" {})))
     val))
 
-(defn- ->block-properties-tx [properties {:keys [property-uuids] :as uuid-maps}]
-  (->> properties
-       (map
-        (fn [[prop-name val]]
-          [(or (property-uuids prop-name)
-               (throw (ex-info "No uuid for property" {:name prop-name})))
-            ;; set indicates a :many value
-           (if (set? val)
-             (set (map #(translate-property-value % uuid-maps) val))
-             (translate-property-value val uuid-maps))]))
-       (into {})))
+(defn- get-ident [all-idents kw]
+  (or (get all-idents kw)
+      (throw (ex-info (str "No ident found for " kw) {}))))
+
+(defn- ->block-properties [properties uuid-maps all-idents]
+  (->>
+   (map
+    (fn [[prop-name val]]
+      [(get-ident all-idents prop-name)
+       ;; set indicates a :many value
+       (if (set? val)
+         (set (map #(translate-property-value % uuid-maps) val))
+         (translate-property-value val uuid-maps))])
+    properties)
+   (into {})))
 
 (defn- create-uuid-maps
   "Creates maps of unique page names, block contents and property names to their uuids"
-  [pages-and-blocks properties]
-  (let [property-uuids (->> pages-and-blocks
-                            (map #(-> (:blocks %) vec (conj (:page %))))
-                            (mapcat #(->> % (map :properties) (mapcat keys)))
-                            set
-                            (map #(vector % (random-uuid)))
-                            ;; TODO: Dedupe with above to avoid squashing a previous definition
-                            (concat (map (fn [[k v]]
-                                           [k (or (:block/uuid v) (random-uuid))])
-                                         properties))
-                            (into {}))
-        page-uuids (->> pages-and-blocks
+  [pages-and-blocks]
+  (let [page-uuids (->> pages-and-blocks
                         (map :page)
                         (map (juxt #(or (:block/name %) (common-util/page-name-sanity-lc (:block/original-name %)))
                                    :block/uuid))
@@ -96,16 +97,13 @@
                          (mapcat :blocks)
                          (map (juxt :block/content :block/uuid))
                          (into {}))]
-    {:property-uuids property-uuids
-     :page-uuids page-uuids
+    {:page-uuids page-uuids
      :block-uuids block-uuids}))
 
-(defn- build-property-refs [properties property-db-ids]
+(defn- build-property-refs [properties all-idents]
   (mapv
    (fn [prop-name]
-     {:db/id
-      (or (property-db-ids (name prop-name))
-          (throw (ex-info (str "No :db/id for property '" prop-name "'") {:property prop-name})))})
+     {:db/ident (get-ident all-idents prop-name)})
    (keys properties)))
 
 (def current-db-id (atom 0))
@@ -113,17 +111,123 @@
   "Provides the next temp :db/id to use in a create-graph transact!"
   #(swap! current-db-id dec))
 
-(defn- ->block-tx [m uuid-maps property-db-ids page-id last-block]
+(defn- ->block-tx [m uuid-maps all-idents page-id]
   (merge (dissoc m :properties)
          (sqlite-util/block-with-timestamps
           {:db/id (new-db-id)
            :block/format :markdown
            :block/page {:db/id page-id}
-           :block/left {:db/id (or (:db/id last-block) page-id)}
+           :block/order (db-order/gen-key nil)
            :block/parent {:db/id page-id}})
          (when (seq (:properties m))
-           {:block/properties (->block-properties-tx (:properties m) uuid-maps)
-            :block/refs (build-property-refs (:properties m) property-db-ids)})))
+           (merge (->block-properties (:properties m) uuid-maps all-idents)
+                  {:block/refs (build-property-refs (:properties m) all-idents)}))))
+
+(defn- build-properties-tx [properties uuid-maps all-idents]
+  (let [property-db-ids (->> (keys properties)
+                             (map #(vector (name %) (new-db-id)))
+                             (into {}))
+        new-properties-tx (vec
+                           (mapcat
+                            (fn [[prop-name prop-m]]
+                              (if (:closed-values prop-m)
+                                (let [db-ident (get-ident all-idents prop-name)]
+                                  (db-property-build/build-closed-values
+                                   db-ident
+                                   prop-name
+                                   (assoc prop-m :db/ident db-ident)
+                                   {:property-attributes
+                                    {:db/id (or (property-db-ids (name prop-name))
+                                                (throw (ex-info "No :db/id for property" {:property prop-name})))}}))
+                                [(merge
+                                  (sqlite-util/build-new-property (get-ident all-idents prop-name)
+                                                                  (:block/schema prop-m)
+                                                                  {:block-uuid (:block/uuid prop-m)})
+                                  {:db/id (or (property-db-ids (name prop-name))
+                                              (throw (ex-info "No :db/id for property" {:property prop-name})))}
+                                  (when-let [props (not-empty (:properties prop-m))]
+                                    (merge
+                                     (->block-properties props uuid-maps all-idents)
+                                     {:block/refs (build-property-refs props all-idents)})))]))
+                            properties))]
+    new-properties-tx))
+
+(defn- build-classes-tx [classes uuid-maps all-idents]
+  (let [class-db-ids (->> (keys classes)
+                          (map #(vector (name %) (new-db-id)))
+                          (into {}))
+        classes-tx (mapv
+                    (fn [[class-name {:keys [class-parent schema-properties] :as class-m}]]
+                      (merge
+                       (sqlite-util/build-new-class
+                        {:block/name (common-util/page-name-sanity-lc (name class-name))
+                         :block/original-name (name class-name)
+                         :block/uuid (d/squuid)
+                         :db/ident (get-ident all-idents class-name)
+                         :db/id (or (class-db-ids (name class-name))
+                                    (throw (ex-info "No :db/id for class" {:class class-name})))})
+                       (dissoc class-m :properties :class-parent :schema-properties)
+                       (when-let [props (not-empty (:properties class-m))]
+                         (merge
+                          (->block-properties props uuid-maps all-idents)
+                          {:block/refs (build-property-refs props all-idents)}))
+                       (when class-parent
+                         {:class/parent
+                          (or (class-db-ids class-parent)
+                              (throw (ex-info (str "No :db/id for " class-parent) {})))})
+                       (when schema-properties
+                         {:class/schema.properties
+                          (mapv #(hash-map :db/ident (get-ident all-idents (keyword %)))
+                                schema-properties)})))
+                    classes)]
+    classes-tx))
+
+
+(defn- validate-options
+  [{:keys [pages-and-blocks properties classes]}]
+  (let [page-block-properties (->> pages-and-blocks
+                                   (map #(-> (:blocks %) vec (conj (:page %))))
+                                   (mapcat #(->> % (map :properties) (mapcat keys)))
+                                   set)
+        property-class-properties (->> (vals properties)
+                                       (concat (vals classes))
+                                       (mapcat #(keys (:properties %)))
+                                       set)
+        undeclared-properties (-> page-block-properties
+                                  (into property-class-properties)
+                                  (set/difference (set (keys properties))))
+        invalid-pages (remove #(or (:block/original-name %) (:block/name %))
+                              (map :page pages-and-blocks))]
+    (assert (empty? invalid-pages)
+            (str "The following pages did not have a name attribute: " invalid-pages))
+    (assert (every? :block/schema (vals properties))
+            "All properties must have :block/schema")
+    (assert (empty? undeclared-properties)
+            (str "The following properties used in EDN were not declared in :properties: " undeclared-properties))))
+
+;; TODO: How to detect these idents don't conflict with existing? :db/add?
+(defn- create-all-idents
+  [properties classes graph-namespace]
+  (let [property-idents (->> (keys properties)
+                             (map #(vector %
+                                           (if graph-namespace
+                                             (db-property/create-db-ident-from-name (str (name graph-namespace) ".property")
+                                                                                    (name %))
+                                             (db-property/create-user-property-ident-from-name (name %)))))
+                             (into {}))
+        _ (assert (= (count (set (vals property-idents))) (count properties)) "All property db-idents must be unique")
+        class-idents (->> (keys classes)
+                          (map #(vector %
+                                        (if graph-namespace
+                                          (db-property/create-db-ident-from-name (str (name graph-namespace) ".class")
+                                                                                 (name %))
+                                          (db-property/create-db-ident-from-name "user.class" (name %)))))
+                          (into {}))
+        _ (assert (= (count (set (vals class-idents))) (count classes)) "All class db-idents must be unique")
+        all-idents (merge property-idents class-idents)]
+    (assert (= (count all-idents) (+ (count property-idents) (count class-idents)))
+            "Class and property db-idents have no overlap")
+    all-idents))
 
 (defn create-blocks-tx
   "Given an EDN map for defining pages, blocks and properties, this creates a
@@ -131,7 +235,7 @@
    have the following limitations:
 
  * Only top level blocks can be easily defined. Other level blocks can be
-   defined but they require explicit setting of attributes like :block/left and :block/parent
+   defined but they require explicit setting of attributes like :block/order and :block/parent
  * Block content containing page refs or tags is not supported yet
 
    The EDN map has the following keys:
@@ -144,77 +248,69 @@
        :block/content is required and :properties can be passed to define block properties
    * :properties - This is a map to configure properties where the keys are property names
      and the values are maps of datascript attributes e.g. `{:block/schema {:type :checkbox}}`.
-     An additional key `:closed-values` is available to define closed values. The key takes
-     a vec of maps containing keys :uuid, :value and :icon.
-
-   The :properties for :pages-and-blocks is a map of property names to property
-   values.  Multiple property values for a many cardinality property are defined
-   as a set. The following property types are supported: :default, :url,
-   :checkbox, :number, :page and :date. :checkbox and :number values are written
-   as booleans and integers. :page and :block are references that are written as
-   vectors e.g. `[:page \"PAGE NAME\"]` and `[:block \"block content\"]`
-
-   This fn also takes an optional map arg which supports these keys:
-   * :property-uuids - A map of property keyword names to uuids to provide ids for built-in properties"
-  [db {:keys [pages-and-blocks properties]} & {:as options}]
-  (let [;; add uuids before tx for refs in :properties
+     Additional keys available:
+     * :closed-values - Define closed values with a vec of maps. A map contains keys :uuid, :value and :icon.
+     * :properties - Define properties on a property page.
+   * :classes - This is a map to configure classes where the keys are class names
+     and the values are maps of datascript attributes e.g. `{:block/original-name \"Foo\"}`.
+     Additional keys available:
+     * :properties - Define properties on a class page
+     * :class-parent - Add a class parent by its name
+     * :schema-properties - Vec of property names. Defines properties that a class gives to its objects
+  * :graph-namespace - namespace to use for db-ident creation. Useful when importing an ontology
+  * :page-id-fn - custom fn that returns ent lookup id for page refs e.g. `[:block/uuid X]`
+    Default is :db/id
+
+   The :properties in :pages-and-blocks, :properties and :classes is a map of
+   property names to property values.  Multiple property values for a many
+   cardinality property are defined as a set. The following property types are
+   supported: :default, :url, :checkbox, :number, :page and :date. :checkbox and
+   :number values are written as booleans and integers/floats. :page references
+   are written as vectors e.g. `[:page \"PAGE NAME\"]`"
+  [{:keys [pages-and-blocks properties classes graph-namespace page-id-fn]
+    :or {page-id-fn :db/id}
+    :as options}]
+  (let [_ (validate-options options)
+        ;; add uuids before tx for refs in :properties
         pages-and-blocks' (mapv (fn [{:keys [page blocks]}]
                                   (cond-> {:page (merge {:block/uuid (random-uuid)} page)}
                                     (seq blocks)
                                     (assoc :blocks (mapv #(merge {:block/uuid (random-uuid)} %) blocks))))
                                 pages-and-blocks)
-        {:keys [property-uuids] :as uuid-maps} (create-uuid-maps pages-and-blocks' properties)
-        property-db-ids (->> property-uuids
-                             (map #(vector (name (first %)) (new-db-id)))
-                             (into {}))
-        new-properties-tx (vec
-                           (mapcat
-                            (fn [[prop-name uuid]]
-                              (if (get-in properties [prop-name :closed-values])
-                                (db-property-util/build-closed-values
-                                 db
-                                 prop-name
-                                 (assoc (get properties prop-name) :block/uuid uuid)
-                                 {:icon-id
-                                  (get-in options [:property-uuids :icon])
-                                  :translate-closed-page-value-fn
-                                  #(hash-map :block/uuid (translate-property-value (:value %) uuid-maps))
-                                  :property-attributes
-                                  {:db/id (or (property-db-ids (name prop-name))
-                                              (throw (ex-info "No :db/id for property" {:property prop-name})))}})
-                                [(merge
-                                  (sqlite-util/build-new-property prop-name (get-in properties [prop-name :block/schema]) uuid)
-                                  {:db/id (or (property-db-ids (name prop-name))
-                                                      (throw (ex-info "No :db/id for property" {:property prop-name})))}
-                                  (when-let [props (not-empty (get-in properties [prop-name :properties]))]
-                                            {:block/properties (->block-properties-tx props uuid-maps)
-                                             :block/refs (build-property-refs props property-db-ids)}))]))
-                            property-uuids))
+        uuid-maps (create-uuid-maps pages-and-blocks')
+        all-idents (create-all-idents properties classes graph-namespace)
+        properties-tx (build-properties-tx properties uuid-maps all-idents)
+        classes-tx (build-classes-tx classes uuid-maps all-idents)
         pages-and-blocks-tx
         (vec
          (mapcat
           (fn [{:keys [page blocks]}]
-            (let [page-id (or (:db/id page) (new-db-id))]
+            (let [new-page (merge
+                            {:db/id (or (:db/id page) (new-db-id))
+                             :block/original-name (or (:block/original-name page) (string/capitalize (:block/name page)))
+                             :block/name (or (:block/name page) (common-util/page-name-sanity-lc (:block/original-name page)))
+                             :block/format :markdown}
+                            (dissoc page :properties :db/id :block/name :block/original-name))]
               (into
                ;; page tx
                [(sqlite-util/block-with-timestamps
                  (merge
-                  {:db/id page-id
-                   :block/original-name (or (:block/original-name page) (string/capitalize (:block/name page)))
-                   :block/name (or (:block/name page) (common-util/page-name-sanity-lc (:block/original-name page)))
-                   :block/journal? false
-                   :block/format :markdown}
-                  (dissoc page :properties)
+                  new-page
+                  (when (seq (:properties page))
+                    (->block-properties (:properties page) uuid-maps all-idents))
                   (when (seq (:properties page))
-                    {:block/properties (->block-properties-tx (:properties page) uuid-maps)
-                     :block/refs (build-property-refs (:properties page) property-db-ids)
-                          ;; app doesn't do this yet but it should to link property to page
-                     :block/path-refs (build-property-refs (:properties page) property-db-ids)})))]
+                    {:block/refs (build-property-refs (:properties page) all-idents)
+                     ;; app doesn't do this yet but it should to link property to page
+                     :block/path-refs (build-property-refs (:properties page) all-idents)})))]
                ;; blocks tx
                (reduce (fn [acc m]
                          (conj acc
-                               (->block-tx m uuid-maps property-db-ids page-id (last acc))))
+                               (->block-tx m uuid-maps all-idents (page-id-fn new-page))))
                        []
                        blocks))))
           pages-and-blocks'))]
-    (into pages-and-blocks-tx new-properties-tx)))
+    ;; Properties first b/c they have schema. Then pages b/c they can be referenced by blocks
+    (vec (concat properties-tx
+                 classes-tx
+                 (filter :block/name pages-and-blocks-tx)
+                 (remove :block/name pages-and-blocks-tx)))))

scripts/src/logseq/tasks/db_graph/create_graph_with_inferred_properties.cljs

@@ -63,7 +63,7 @@
                         ((juxt node-path/dirname node-path/basename) graph-dir)
                         [(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
         conn (create-graph/init-conn dir db-name)
-        blocks-tx (create-graph/create-blocks-tx @conn (create-init-data))]
+        blocks-tx (create-graph/create-blocks-tx (create-init-data))]
     (println "Generating" (count (filter :block/name blocks-tx)) "pages and"
              (count (filter :block/content blocks-tx)) "blocks ...")
     (d/transact! conn blocks-tx)

scripts/src/logseq/tasks/db_graph/create_graph_with_large_sizes.cljs

@@ -24,7 +24,7 @@
     (map-indexed
      (fn [idx id]
        {:block/uuid id
-        :block/name (str "page-" (+ start-idx idx))})
+        :block/original-name (str "Page-" (+ start-idx idx))})
      ids)))
 
 (defn build-blocks
@@ -40,7 +40,9 @@
   (let [pages (build-pages 0 (:pages options))]
     {:pages-and-blocks
      (mapv #(hash-map :page % :blocks (build-blocks (:blocks options)))
-           pages)}))
+           pages)
+     ;; Custom id fn because transaction chunks may separate blocks and pages from each other
+     :page-id-fn (fn [b] [:block/uuid (:block/uuid b)])}))
 
 (def spec
   "Options spec"
@@ -65,8 +67,8 @@
                         [(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
         conn (create-graph/init-conn dir db-name)
         _ (println "Building tx ...")
-        blocks-tx (create-graph/create-blocks-tx @conn (create-init-data options))]
-    (println "Built" (count blocks-tx) "tx," (count (filter :block/name blocks-tx)) "pages and"
+        blocks-tx (create-graph/create-blocks-tx (create-init-data options))]
+    (println "Built" (count blocks-tx) "tx," (count (filter :block/original-name blocks-tx)) "pages and"
              (count (filter :block/content blocks-tx)) "blocks ...")
     ;; Vary the chunking with page size up to a max to avoid OOM
     (let [tx-chunks (partition-all (min (:pages options) 30000) blocks-tx)]
@@ -74,7 +76,7 @@
              chunk-num 1]
         (when-let [chunk (first chunks)]
           (println "Transacting chunk" chunk-num  "of" (count tx-chunks)
-                   "starting with block:" (pr-str (select-keys (first chunk) [:block/content :block/name])))
+                   "starting with block:" (pr-str (select-keys (first chunk) [:block/content :block/original-name])))
           (d/transact! conn chunk)
           (recur (rest chunks) (inc chunk-num)))))
     #_(d/transact! conn blocks-tx)

scripts/src/logseq/tasks/db_graph/create_graph_with_properties.cljs

@@ -8,6 +8,7 @@
             [logseq.db.frontend.property.type :as db-property-type]
             [clojure.string :as string]
             [clojure.edn :as edn]
+            [clojure.set :as set]
             [datascript.core :as d]
             ["path" :as node-path]
             ["os" :as os]
@@ -25,7 +26,7 @@
 
 (defn- build-closed-values-config
   [{:keys [dates]}]
-  {:default-closed
+  {:string-closed
    (mapv #(hash-map :value %
                     :uuid (random-uuid)
                     :icon {:id % :name % :type :emoji})
@@ -39,130 +40,142 @@
                     :uuid (random-uuid))
          [10 42 (rand 100)])
    :page-closed
-   (mapv #(hash-map :value [:page %])
-         ["page 1" "page 2" "page 3"])
+   (mapv #(hash-map :value %
+                    :uuid (random-uuid))
+         ["Page 1" "Page 2" "Page 3"])
    :date-closed
-   (mapv #(hash-map :value [:page (date-journal-title %)])
+   ;; TODO: Consider capitalized journal name when confirming queries work
+   (mapv #(hash-map :value (date-journal-title %)
+                    :uuid (random-uuid))
          dates)})
 
+(defn- create-journal-page
+  [date journal-name-uuid-map]
+  (let [journal-name (date-journal-title date)]
+    {:block/name journal-name
+     :block/uuid (or (journal-name-uuid-map journal-name) (throw (ex-info "No uuid for journal name" {})))
+     :block/type "journal"
+     :block/journal-day (date-journal-day date)}))
+
 (defn- create-init-data
   []
   (let [today (new js/Date)
         yesterday (subtract-days today 1)
         two-days-ago (subtract-days today 2)
         closed-values-config (build-closed-values-config {:dates [today yesterday two-days-ago]})
+        page-values-tx (mapv #(hash-map :page
+                                        {:block/uuid (:uuid %) :block/original-name (:value %)})
+                             (:page-closed closed-values-config))
+        journal-name->uuid (into {} (map (juxt :value :uuid) (:date-closed closed-values-config)))
         ;; Stores random closed values for use with queries
         closed-values (atom {})
         random-closed-value #(let [val (-> closed-values-config % rand-nth)]
                                (swap! closed-values assoc % (:value val))
-                               (:uuid val))
-        random-page-closed-value #(let [val (-> closed-values-config % rand-nth :value)]
-                                    (swap! closed-values assoc % (second val))
-                                    val)
+                               [:block/uuid (:uuid val)])
         get-closed-value #(get @closed-values %)]
     {:pages-and-blocks
-     ;; Journals
-     [{:page
-       {:block/name (date-journal-title today) :block/journal? true :block/journal-day (date-journal-day today)}
-       :blocks
-       [{:block/content "[[Block Properties]]"}
-        {:block/content "[[Block Property Queries]]"}
-        {:block/content "[[Page Property Queries]]"}]}
-      {:page
-       {:block/name (date-journal-title yesterday) :block/journal? true :block/journal-day (date-journal-day yesterday)}}
-      {:page
-       {:block/name (date-journal-title two-days-ago) :block/journal? true :block/journal-day (date-journal-day two-days-ago)}}
-
-      ;; Block property blocks and queries
-      {:page {:block/original-name "Block Properties"}
-       :blocks
-       [{:block/content "default property block" :properties {:default "haha"}}
-        {:block/content "default-closed property block" :properties {:default-closed (random-closed-value :default-closed)}}
-        {:block/content "url property block" :properties {:url "https://logseq.com"}}
-        {:block/content "url-many property block" :properties {:url-many #{"https://logseq.com" "https://docs.logseq.com"}}}
-        {:block/content "url-closed property block" :properties {:url-closed (random-closed-value :url-closed)}}
-        {:block/content "checkbox property block" :properties {:checkbox true}}
-        {:block/content "number property block" :properties {:number 5}}
-        {:block/content "number-many property block" :properties {:number-many #{5 10}}}
-        {:block/content "number-closed property block" :properties {:number-closed (random-closed-value :number-closed)}}
-        {:block/content "page property block" :properties {:page [:page "page 1"]}}
-        {:block/content "page-many property block" :properties {:page-many #{[:page "page 1"] [:page "page 2"]}}}
-        {:block/content "page-closed property block" :properties {:page-closed (random-page-closed-value :page-closed)}}
-        {:block/content "date property block" :properties {:date [:page (date-journal-title today)]}}
-        {:block/content "date-many property block" :properties {:date-many #{[:page (date-journal-title today)]
-                                                                             [:page (date-journal-title yesterday)]}}}
-        {:block/content "date-closed property block" :properties {:date-closed (random-page-closed-value :date-closed)}}]}
-      {:page {:block/original-name "Block Property Queries"}
-       :blocks
-       [{:block/content "{{query (property :default \"haha\")}}"}
-        {:block/content (str "{{query (property :default-closed " (pr-str (get-closed-value :default-closed)) ")}}")}
-        {:block/content "{{query (property :url \"https://logseq.com\")}}"}
-        {:block/content "{{query (property :url-many \"https://logseq.com\")}}"}
-        {:block/content (str "{{query (property :url-closed " (pr-str (get-closed-value :url-closed)) ")}}")}
-        {:block/content "{{query (property :checkbox true)}}"}
-        {:block/content "{{query (property :number 5)}}"}
-        {:block/content "{{query (property :number-many 10)}}"}
-        {:block/content (str "{{query (property :number-closed " (pr-str (get-closed-value :number-closed)) ")}}")}
-        {:block/content "{{query (property :page [[Page 1]])}}"}
-        {:block/content "{{query (property :page-many [[Page 2]])}}"}
-        {:block/content (str "{{query (property :page-closed " (page-ref/->page-ref (string/capitalize (get-closed-value :page-closed))) ")}}")}
-        {:block/content (str "{{query (property :date " (page-ref/->page-ref (string/capitalize (date-journal-title today))) ")}}")}
-        {:block/content (str "{{query (property :date-many " (page-ref/->page-ref (string/capitalize (date-journal-title yesterday))) ")}}")}
-        {:block/content (str "{{query (property :date-closed " (page-ref/->page-ref (string/capitalize (get-closed-value :date-closed))) ")}}")}]}
+     (into
+      ;; Page property values needs to be before b/c they are referenced by everything else
+      page-values-tx
+      ;; Journals
+      [{:page
+        (create-journal-page today journal-name->uuid)
+        :blocks
+        [{:block/content "[[Block Properties]]"}
+         {:block/content "[[Block Property Queries]]"}
+         {:block/content "[[Page Property Queries]]"}]}
+       {:page (create-journal-page yesterday journal-name->uuid)}
+       {:page (create-journal-page two-days-ago journal-name->uuid)}
 
-      ;; Page property pages and queries
-      {:page {:block/name "default page" :properties {:default "yolo"}}}
-      {:page {:block/name "default-closed page" :properties {:default-closed (random-closed-value :default-closed)}}}
-      {:page {:block/name "url page" :properties {:url "https://logseq.com"}}}
-      {:page {:block/name "url-many page" :properties {:url-many #{"https://logseq.com" "https://docs.logseq.com"}}}}
-      {:page {:block/name "url-closed page" :properties {:url-closed (random-closed-value :url-closed)}}}
-      {:page {:block/name "checkbox page" :properties {:checkbox true}}}
-      {:page {:block/name "number page" :properties {:number 5}}}
-      {:page {:block/name "number-many page" :properties {:number-many #{5 10}}}}
-      {:page {:block/name "number-closed page" :properties {:number-closed (random-closed-value :number-closed)}}}
-      {:page {:block/name "page page" :properties {:page [:page "page 1"]}}}
-      {:page {:block/name "page-many page" :properties {:page-many #{[:page "page 1"] [:page "page 2"]}}}}
-      {:page {:block/name "page-closed page" :properties {:page-closed (random-page-closed-value :page-closed)}}}
-      {:page {:block/name "date page" :properties {:date [:page (date-journal-title today)]}}}
-      {:page {:block/name "date-many page" :properties {:date-many #{[:page (date-journal-title today)]
-                                                                     [:page (date-journal-title yesterday)]}}}}
-      {:page {:block/name "date-closed page" :properties {:date-closed (random-page-closed-value :date-closed)}}}
-      {:page {:block/original-name "Page Property Queries"}
-       :blocks
-       [{:block/content "{{query (page-property :default \"yolo\")}}"}
-        {:block/content (str "{{query (page-property :default-closed " (pr-str (get-closed-value :default-closed)) ")}}")}
-        {:block/content "{{query (page-property :url \"https://logseq.com\")}}"}
-        {:block/content "{{query (page-property :url-many \"https://logseq.com\")}}"}
-        {:block/content (str "{{query (page-property :url-closed " (pr-str (get-closed-value :url-closed)) ")}}")}
-        {:block/content "{{query (page-property :checkbox true)}}"}
-        {:block/content "{{query (page-property :number 5)}}"}
-        {:block/content "{{query (page-property :number-many 10)}}"}
-        {:block/content (str "{{query (page-property :number-closed " (pr-str (get-closed-value :number-closed)) ")}}")}
-        {:block/content "{{query (page-property :page [[Page 1]])}}"}
-        {:block/content "{{query (page-property :page-many [[Page 2]])}}"}
-        {:block/content (str "{{query (page-property :page-closed " (page-ref/->page-ref (string/capitalize (get-closed-value :page-closed))) ")}}")}
-        {:block/content (str "{{query (page-property :date " (page-ref/->page-ref (string/capitalize (date-journal-title today))) ")}}")}
-        {:block/content (str "{{query (page-property :date-many " (page-ref/->page-ref (string/capitalize (date-journal-title yesterday))) ")}}")}
-        {:block/content (str "{{query (page-property :date-closed " (page-ref/->page-ref (string/capitalize (get-closed-value :date-closed))) ")}}")}]}
+       ;; Block property blocks and queries
+       {:page {:block/original-name "Block Properties"}
+        :blocks
+        ;; TODO: Fix :default
+        [#_{:block/content "default property block" :properties {:default "haha block"}}
+         {:block/content "string property block" :properties {:string "haha"}}
+         {:block/content "string-many property block" :properties {:string-many #{"yee" "haw" "sir"}}}
+         {:block/content "string-closed property block" :properties {:string-closed (random-closed-value :string-closed)}}
+         {:block/content "url property block" :properties {:url "https://logseq.com"}}
+         {:block/content "url-many property block" :properties {:url-many #{"https://logseq.com" "https://docs.logseq.com"}}}
+         {:block/content "url-closed property block" :properties {:url-closed (random-closed-value :url-closed)}}
+         {:block/content "checkbox property block" :properties {:checkbox true}}
+         {:block/content "number property block" :properties {:number 5}}
+         {:block/content "number-many property block" :properties {:number-many #{5 10}}}
+         {:block/content "number-closed property block" :properties {:number-closed (random-closed-value :number-closed)}}
+         {:block/content "page property block" :properties {:page [:page "page 1"]}}
+         {:block/content "page-many property block" :properties {:page-many #{[:page "page 1"] [:page "page 2"]}}}
+         {:block/content "page-closed property block" :properties {:page-closed (random-closed-value :page-closed)}}
+         {:block/content "date property block" :properties {:date [:page (date-journal-title today)]}}
+         {:block/content "date-many property block" :properties {:date-many #{[:page (date-journal-title today)]
+                                                                              [:page (date-journal-title yesterday)]}}}
+         {:block/content "date-closed property block" :properties {:date-closed (random-closed-value :date-closed)}}]}
+       {:page {:block/original-name "Block Property Queries"}
+        :blocks
+        [{:block/content "{{query (property :string \"haha\")}}"}
+         {:block/content "{{query (property :string-many \"haw\")}}"}
+         {:block/content (str "{{query (property :string-closed " (pr-str (get-closed-value :string-closed)) ")}}")}
+         {:block/content "{{query (property :url \"https://logseq.com\")}}"}
+         {:block/content "{{query (property :url-many \"https://logseq.com\")}}"}
+         {:block/content (str "{{query (property :url-closed " (pr-str (get-closed-value :url-closed)) ")}}")}
+         {:block/content "{{query (property :checkbox true)}}"}
+         {:block/content "{{query (property :number 5)}}"}
+         {:block/content "{{query (property :number-many 10)}}"}
+         {:block/content (str "{{query (property :number-closed " (pr-str (get-closed-value :number-closed)) ")}}")}
+         {:block/content "{{query (property :page [[Page 1]])}}"}
+         {:block/content "{{query (property :page-many [[Page 2]])}}"}
+         {:block/content (str "{{query (property :page-closed " (page-ref/->page-ref (string/capitalize (get-closed-value :page-closed))) ")}}")}
+         {:block/content (str "{{query (property :date " (page-ref/->page-ref (string/capitalize (date-journal-title today))) ")}}")}
+         {:block/content (str "{{query (property :date-many " (page-ref/->page-ref (string/capitalize (date-journal-title yesterday))) ")}}")}
+         {:block/content (str "{{query (property :date-closed " (page-ref/->page-ref (string/capitalize (get-closed-value :date-closed))) ")}}")}]}
 
-      ;; Property values
-      {:page {:block/name "page 1"}
-       :blocks
-       [{:block/content "yee"}
-        {:block/content "haw"}]}
-      {:page {:block/name "page 2"}}
-      {:page {:block/name "page 3"}}]
+       ;; Page property pages and queries
+       ;; {:page {:block/name "default page" :properties {:default "yolo block"}}}
+       {:page {:block/name "string page" :properties {:string "yolo"}}}
+       {:page {:block/name "string-many page" :properties {:string-many #{"yee" "haw" "sir"}}}}
+       {:page {:block/name "string-closed page" :properties {:string-closed (random-closed-value :string-closed)}}}
+       {:page {:block/name "url page" :properties {:url "https://logseq.com"}}}
+       {:page {:block/name "url-many page" :properties {:url-many #{"https://logseq.com" "https://docs.logseq.com"}}}}
+       {:page {:block/name "url-closed page" :properties {:url-closed (random-closed-value :url-closed)}}}
+       {:page {:block/name "checkbox page" :properties {:checkbox true}}}
+       {:page {:block/name "number page" :properties {:number 5}}}
+       {:page {:block/name "number-many page" :properties {:number-many #{5 10}}}}
+       {:page {:block/name "number-closed page" :properties {:number-closed (random-closed-value :number-closed)}}}
+       {:page {:block/name "page page" :properties {:page [:page "page 1"]}}}
+       {:page {:block/name "page-many page" :properties {:page-many #{[:page "page 1"] [:page "page 2"]}}}}
+       {:page {:block/name "page-closed page" :properties {:page-closed (random-closed-value :page-closed)}}}
+       {:page {:block/name "date page" :properties {:date [:page (date-journal-title today)]}}}
+       {:page {:block/name "date-many page" :properties {:date-many #{[:page (date-journal-title today)]
+                                                                      [:page (date-journal-title yesterday)]}}}}
+       {:page {:block/name "date-closed page" :properties {:date-closed (random-closed-value :date-closed)}}}
+       {:page {:block/original-name "Page Property Queries"}
+        :blocks
+        [{:block/content "{{query (page-property :string \"yolo\")}}"}
+         {:block/content "{{query (page-property :string-many \"haw\")}}"}
+         {:block/content (str "{{query (page-property :string-closed " (pr-str (get-closed-value :string-closed)) ")}}")}
+         {:block/content "{{query (page-property :url \"https://logseq.com\")}}"}
+         {:block/content "{{query (page-property :url-many \"https://logseq.com\")}}"}
+         {:block/content (str "{{query (page-property :url-closed " (pr-str (get-closed-value :url-closed)) ")}}")}
+         {:block/content "{{query (page-property :checkbox true)}}"}
+         {:block/content "{{query (page-property :number 5)}}"}
+         {:block/content "{{query (page-property :number-many 10)}}"}
+         {:block/content (str "{{query (page-property :number-closed " (pr-str (get-closed-value :number-closed)) ")}}")}
+         {:block/content "{{query (page-property :page [[Page 1]])}}"}
+         {:block/content "{{query (page-property :page-many [[Page 2]])}}"}
+         {:block/content (str "{{query (page-property :page-closed " (page-ref/->page-ref (string/capitalize (get-closed-value :page-closed))) ")}}")}
+         {:block/content (str "{{query (page-property :date " (page-ref/->page-ref (string/capitalize (date-journal-title today))) ")}}")}
+         {:block/content (str "{{query (page-property :date-many " (page-ref/->page-ref (string/capitalize (date-journal-title yesterday))) ")}}")}
+         {:block/content (str "{{query (page-property :date-closed " (page-ref/->page-ref (string/capitalize (get-closed-value :date-closed))) ")}}")}]}])
 
      ;; Properties
      :properties
-     (->> [:default :url :checkbox :number :page :date]
+     (->> [:default :string :url :checkbox :number :page :date]
           (mapcat #(cond-> [[% {:block/schema {:type %}}]]
                      (db-property-type/property-type-allows-schema-attribute? % :cardinality)
                      (conj [(keyword (str (name %) "-many")) {:block/schema {:type % :cardinality :many}}])))
           (into (mapv #(vector (keyword (str (name %) "-closed"))
                                {:closed-values (closed-values-config (keyword (str (name %) "-closed")))
                                 :block/schema {:type %}})
-                      [:default :url :number :page :date]))
+                      [:string :url :number :page :date]))
           (into {}))}))
 
 (def spec
@@ -184,10 +197,13 @@
                         ((juxt node-path/dirname node-path/basename) graph-dir)
                         [(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
         conn (create-graph/init-conn dir db-name {:additional-config (:config options)})
-        blocks-tx (create-graph/create-blocks-tx
-                   @conn
-                   (create-init-data)
-                   {:property-uuids {:icon (:block/uuid (d/entity @conn :logseq.property/icon))}})]
+        blocks-tx (create-graph/create-blocks-tx (create-init-data))
+        existing-names (set (map :v (d/datoms @conn :avet :block/original-name)))
+        conflicting-names (set/intersection existing-names (set (keep :block/original-name blocks-tx)))]
+    (when (seq conflicting-names)
+      (println "Error: Following names conflict -" (string/join "," conflicting-names))
+      (js/process.exit 1))
+    (println "DB dir: " (node-path/join dir db-name))
     (println "Generating" (count (filter :block/name blocks-tx)) "pages and"
              (count (filter :block/content blocks-tx)) "blocks ...")
     (d/transact! conn blocks-tx)

scripts/src/logseq/tasks/db_graph/create_graph_with_schema_org.cljs

@@ -12,6 +12,7 @@
      * schema.org assumes no cardinality. For now, only :page properties are given a :cardinality :many"
   (:require [logseq.tasks.db-graph.create-graph :as create-graph]
             [logseq.common.util :as common-util]
+            [logseq.db.frontend.property :as db-property]
             [clojure.string :as string]
             [datascript.core :as d]
             ["path" :as node-path]
@@ -20,13 +21,8 @@
             [nbb.core :as nbb]
             [clojure.set :as set]
             [clojure.walk :as w]
-            [babashka.cli :as cli]))
-
-(defn- get-class-db-id [class-db-ids class-id]
-  (or (class-db-ids class-id)
-      ;; Map of owl:equivalentClass exceptions
-      (class-db-ids ({"rdfs:Class" "schema:Class"} class-id))
-      (throw (ex-info (str "No :db/id for " class-id) {}))))
+            [babashka.cli :as cli]
+            [logseq.db.frontend.malli-schema :as db-malli-schema]))
 
 (defn- get-class-uuid [class-uuids class-id]
   (or (class-uuids class-id)
@@ -46,7 +42,7 @@
 (defn- strip-schema-prefix [s]
   (string/replace-first s "schema:" ""))
 
-(defn- ->class-page [class-m class-db-ids class-uuids class-properties property-uuids {:keys [verbose renamed-classes renamed-pages]}]
+(defn- ->class-page [class-m class-uuids class-properties {:keys [verbose renamed-classes renamed-pages]}]
   (let [parent-class* (class-m "rdfs:subClassOf")
         parent-class (cond
                        (map? parent-class*)
@@ -63,28 +59,29 @@
                                          (if (string? type') [type'] type')))
                                   "schema:DataType")
                        "schema:DataType")
-        properties (sort (class-properties (class-m "@id")))
+        ;; Map of owl:equivalentClass exceptions
+        parent-class' (get {"rdfs:Class" "Class"} parent-class parent-class)
+        properties (class-properties (class-m "@id"))
         inverted-renamed-classes (set/map-invert renamed-classes)
         class-name (strip-schema-prefix (class-m "@id"))
         url (str "https://schema.org/" (get inverted-renamed-classes class-name class-name))]
     (cond-> {:block/original-name class-name
              :block/type "class"
              :block/uuid (get-class-uuid class-uuids (class-m "@id"))
-             :db/id (get-class-db-id class-db-ids (class-m "@id"))
              :properties (cond-> {:url url}
                            (class-m "rdfs:comment")
                            (assoc :description (get-comment-string (class-m "rdfs:comment") renamed-pages)))}
-      parent-class
-      (assoc :class/parent {:db/id (get-class-db-id class-db-ids parent-class)})
+      parent-class'
+      (assoc :class-parent (strip-schema-prefix parent-class'))
       (seq properties)
-      (assoc :block/schema {:properties (mapv property-uuids properties)}))))
+      (assoc :schema-properties (map strip-schema-prefix properties)))))
 
 (def schema->logseq-data-types
   "Schema datatypes, https://schema.org/DataType, mapped to their Logseq equivalents"
   {"schema:Integer" :number
    "schema:Float" :number
    "schema:Number" :number
-   "schema:Text_Class" :default
+   "schema:Text_Class" :string
    "schema:URL_Class" :url
    "schema:Boolean" :checkbox
    "schema:Date" :date})
@@ -105,7 +102,7 @@
              (when (class-map %) :page))
         range-includes))
 
-(defn- ->property-page [property-m prop-uuid class-map class-uuids {:keys [verbose renamed-pages renamed-properties]}]
+(defn- ->property-page [property-m class-map class-uuids {:keys [verbose renamed-pages renamed-properties]}]
   (let [range-includes (get-range-includes property-m)
         schema-type (get-schema-type range-includes class-map)
         ;; Pick first range to determine type as only one range is supported currently
@@ -132,8 +129,7 @@
                                                                 " has DataType class values which aren't supported: " datatype-classes) {})))]
                                    (set (map class-uuids range-includes)))))]
     {(keyword (strip-schema-prefix (property-m "@id")))
-     {:block/uuid prop-uuid
-      :block/schema schema
+     {:block/schema schema
       :properties {:url url}}}))
 
 (defn- get-class-to-properties
@@ -243,23 +239,20 @@
             (println "Skipping" (count unsupported-properties) "properties with unsupported data types"))]
     all-properties))
 
-(defn- generate-pages
-  [select-classes class-uuids class-to-properties property-uuids options]
-  (let [;; Build db-ids for all classes as they are needed for refs later, across class maps
-        class-db-ids (->> select-classes
-                          (map #(vector (% "@id") (create-graph/new-db-id)))
-                          (into {}))
-        pages (mapv #(hash-map :page
-                               (->class-page % class-db-ids class-uuids class-to-properties property-uuids options))
-                    select-classes)]
-    (assert (= ["Thing"] (keep #(when-not (:class/parent (:page %))
-                                  (:block/original-name (:page %)))
-                               pages))
+(defn- generate-classes
+  [select-classes class-uuids class-to-properties options]
+  (let [classes (->> select-classes
+                     (map #(vector (strip-schema-prefix (get % "@id"))
+                                   (->class-page % class-uuids class-to-properties options)))
+                     (into {}))]
+    (assert (= ["Thing"] (keep #(when-not (:class-parent %)
+                                  (:block/original-name %))
+                               (vals classes)))
             "Thing is the only class that doesn't have a parent class")
-    pages))
+    classes))
 
 (defn- generate-properties
-  [select-properties property-uuids class-map class-uuids options]
+  [select-properties class-map class-uuids options]
   (when (:verbose options)
     (println "Properties by type:"
              (->> select-properties
@@ -268,7 +261,7 @@
                   frequencies)
              "\n"))
   (apply merge
-         (mapv #(->property-page % (property-uuids (% "@id")) class-map class-uuids options)
+         (mapv #(->property-page % class-map class-uuids options)
                select-properties)))
 
 (defn- get-all-classes-and-properties
@@ -318,20 +311,17 @@
         class-map (->> all-classes
                        (map #(vector (% "@id") %))
                        (into {}))
-        select-class-ids (keys class-map)
-        ;; Debug: Uncomment to generate a narrower graph of classes
-        ;; select-class-ids ["schema:Person" "schema:CreativeWorkSeries"
-        ;;                   "schema:Movie" "schema:CreativeWork" "schema:Thing"]
+        select-class-ids
+        (if (:subset options)
+          ["schema:Person" "schema:CreativeWorkSeries" "schema:Organization"
+           "schema:Movie" "schema:CreativeWork" "schema:Thing"]
+          (keys class-map))
         ;; Generate class uuids as they are needed for properties (:page) and pages
         class-uuids (->> all-classes
                          (map #(vector (% "@id") (random-uuid)))
                          (into {}))
         class-to-properties (get-class-to-properties select-class-ids all-properties)
         select-properties (set (mapcat val class-to-properties))
-        ;; Generate property uuids as they are needed for properties and pages (:schema properties)
-        property-uuids (->> select-properties
-                            (map #(vector % (random-uuid)))
-                            (into {}))
         options' (assoc options
                         :renamed-classes renamed-classes
                         :renamed-properties renamed-properties
@@ -339,12 +329,23 @@
         ;; Generate pages and properties
         properties (generate-properties
                     (filter #(contains? select-properties (% "@id")) all-properties)
-                    property-uuids class-map class-uuids options')
-        pages (generate-pages
-               (map #(class-map %) select-class-ids)
-               class-uuids class-to-properties property-uuids options')]
-    {:pages-and-blocks pages
-     :properties properties}))
+                    class-map class-uuids options')
+        properties'
+        (if (:subset options)
+          ;; only keep classes that are in subset to keep graph valid
+          (let [select-class-uuids (->> select-class-ids (map class-uuids) set)]
+            (-> properties
+                (update-vals (fn [m]
+                               (if (get-in m [:block/schema :classes])
+                                 (update-in m [:block/schema :classes] (fn [cs] (set (filterv #(contains? select-class-uuids %) cs))))
+                                 m)))))
+          properties)
+        classes (generate-classes
+                 (map #(class-map %) select-class-ids)
+                 class-uuids class-to-properties options')]
+    {:graph-namespace :schema
+     :classes classes
+     :properties properties'}))
 
 (def spec
   "Options spec"
@@ -352,30 +353,39 @@
           :desc "Print help"}
    :debug {:alias :d
            :desc "Prints additional debug info and a schema.edn for debugging"}
+   :subset {:alias :s
+            :desc "Only generate a subset of data for testing purposes"}
    :verbose {:alias :v
              :desc "Verbose mode"}})
 
-(defn- write-debug-file [blocks-tx db]
-  (let [block-uuid->name* (->> (d/q '[:find (pull ?b [:block/name :block/uuid]) :where [?b :block/name]] db)
+(defn- write-debug-file [db]
+  (let [ents (remove #(db-malli-schema/internal-ident? (:db/ident %))
+                     (d/q '[:find [(pull ?b [*
+                                             {:class/schema.properties [:block/original-name]}
+                                             {:class/parent [:block/original-name]}]) ...]
+                            :in $
+                            :where [?b :db/ident ?ident]]
+                          db))
+        block-uuid->name* (->> (d/q '[:find (pull ?b [:block/original-name :block/uuid]) :where [?b :block/original-name]] db)
                                (map first)
-                               (map (juxt :block/uuid :block/name))
+                               (map (juxt :block/uuid :block/original-name))
                                (into {}))
-        block-uuid->name #(or (block-uuid->name* %) (throw (ex-info (str "No entity found for " %) {})))
-        ;; TODO: Figure out why some Thing's properties don't exist
-        block-uuid->name-please-fixme
-        #(or (block-uuid->name* %2) (println "WARNING: Page" (pr-str (:block/original-name %1)) "skipped uuid" %2))]
+        block-uuid->name #(or (block-uuid->name* %) (throw (ex-info (str "No entity found for " %) {})))]
     (fs/writeFileSync "schema-org.edn"
                       (pr-str
-                       (->> blocks-tx
+                       (->> ents
                             (map (fn [m]
-                                   (cond-> (select-keys m [:block/name :block/type :block/original-name
-                                                           :block/properties :block/schema])
-                                     (seq (:block/properties m))
-                                     (update :block/properties #(update-keys % block-uuid->name))
-                                     (seq (get-in m [:block/schema :properties]))
-                                     (update-in [:block/schema :properties] #(mapv (partial block-uuid->name-please-fixme m) %))
-                                     (seq (get-in m [:block/schema :classes]))
-                                     (update-in [:block/schema :classes] #(mapv block-uuid->name %)))))
+                                   (let [props (db-property/properties m)]
+                                     (cond-> (select-keys m [:block/name :block/type :block/original-name :block/schema :db/ident
+                                                             :class/schema.properties :class/parent])
+                                       (seq props)
+                                       (assoc :block/properties (update-keys props name))
+                                       (seq (:class/schema.properties m))
+                                       (update :class/schema.properties #(set (map :block/original-name %)))
+                                       (some? (:class/parent m))
+                                       (update :class/parent :block/original-name)
+                                       (seq (get-in m [:block/schema :classes]))
+                                       (update-in [:block/schema :classes] #(set (map block-uuid->name %)))))))
                             set)))))
 
 (defn -main [args]
@@ -391,14 +401,14 @@
         conn (create-graph/init-conn dir db-name)
         init-data (create-init-data (d/q '[:find [?name ...] :where [?b :block/name ?name]] @conn)
                                     options)
-        blocks-tx (create-graph/create-blocks-tx @conn init-data)]
+        blocks-tx (create-graph/create-blocks-tx init-data)]
     (println "Generating" (str (count (filter :block/name blocks-tx)) " pages with "
-                               (count (:pages-and-blocks init-data)) " classes and "
+                               (count (:classes init-data)) " classes and "
                                (count (:properties init-data)) " properties ..."))
     (d/transact! conn blocks-tx)
     (when (:verbose options) (println "Transacted" (count (d/datoms @conn :eavt)) "datoms"))
-    (when (:debug options) (write-debug-file blocks-tx @conn))
+    (when (:debug options) (write-debug-file @conn))
     (println "Created graph" (str db-name "!"))))
 
 (when (= nbb/*file* (:file (meta #'-main)))
-  (-main *command-line-args*))
+  (-main *command-line-args*))

scripts/src/logseq/tasks/dev.clj

@@ -60,7 +60,7 @@
   [file1 file2 & args]
   (let [spec {:ignored-attributes
               ;; Ignores some attributes by default that are expected to change often
-              {:alias :i :coerce #{:keyword} :default #{:block/tx-id :block/left :block/updated-at}}}
+              {:alias :i :coerce #{:keyword} :default #{:block/tx-id :block/order :block/updated-at}}}
         {{:keys [ignored-attributes]} :opts} (cli/parse-args args {:spec spec})
         datom-filter (fn [[e a _ _ _]] (contains? ignored-attributes a))
         data-diff* (apply data/diff (map (fn [x] (->> x slurp edn/read-string (remove datom-filter))) [file1 file2]))

scripts/src/logseq/tasks/dev/db_and_file_graphs.clj

@@ -77,12 +77,14 @@
   []
   (let [file-concepts (->>
                        ;; from logseq.db.frontend.schema
-                       [:block/properties-text-values :block/pre-block :recent/pages :file/handle :block/file :block/properties-order]
+                       [:block/properties-text-values :block/pre-block :recent/pages :file/handle :block/file :block/properties-order
+                        :block/marker :block/priority :block/scheduled :block/deadline]
                        (map str)
                        (into [;; e.g. block/properties :title
                               "block/properties :"
                               ;; anything org mode
-                              "org"]))
+                              "org"
+                              "db/get-page"]))
         res (apply shell {:out :string :continue true}
                    "git grep -E" (str "(" (string/join "|" file-concepts) ")")
                    db-graph-paths)]

scripts/yarn.lock

@@ -2,9 +2,9 @@
 # yarn lockfile v1
 
 
-"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v10":
-  version "1.2.173-feat-db-v10"
-  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/8a4bfaae55d85292476ac8f976516b86e6ae4036"
+"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v12":
+  version "1.2.173-feat-db-v12"
+  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/8274d8b40b7fec84532c54e16b5a8c6f78bf673b"
   dependencies:
     import-meta-resolve "^2.1.0"
 

shadow-cljs.edn

@@ -52,7 +52,6 @@
                           frontend.config/ENABLE-PLUGINS #shadow/env ["ENABLE_PLUGINS" :as :bool :default true]
                           ;; Set to switch file sync server to dev, set this to false in `yarn watch`
                           frontend.config/ENABLE-FILE-SYNC-PRODUCTION #shadow/env ["ENABLE_FILE_SYNC_PRODUCTION" :as :bool :default true]
-                          frontend.config/TEST #shadow/env ["LOGSEQ_CI" :as :bool :default false]
                           frontend.config/REVISION #shadow/env ["LOGSEQ_REVISION" :default "dev"]} ;; set by git-revision-hook
 
         ;; NOTE: electron, browser/mobile-app use different asset-paths.

src/dev-cljs/shadow/build_large_graph.cljs

@@ -1,59 +0,0 @@
-(ns shadow.build-large-graph)
-
-(comment
-
-  (in-ns 'frontend.db-worker)
-  (def repo "logseq_db_large-db-demo")
-  (def conn (worker-state/get-datascript-conn repo))
-
-  (defonce *ids (atom (set (map :v (d/datoms @conn :avet :block/uuid)))))
-  (defn get-next-id
-    []
-    (let [id (random-uuid)]
-      (if (@*ids id)
-        (get-next-id)
-        (do
-          (swap! *ids conj id)
-          id))))
-
-  (defn pages
-    [start-idx n]
-    (let [ids (repeatedly n get-next-id)]
-      (map-indexed
-       (fn [idx id]
-         {:block/uuid id
-          :block/original-name (str "page-" (+ start-idx idx))
-          :block/name (str "page-" (+ start-idx idx))
-          :block/format :markdown})
-       ids)))
-
-  (defn blocks
-    [page-id size]
-    (let [page-id [:block/uuid page-id]
-          blocks (vec (repeatedly size (fn []
-                                         (let [id (get-next-id)]
-                                           {:block/uuid id
-                                            :block/content (str id)
-                                            :block/format :markdown
-                                            :block/page page-id
-                                            :block/parent page-id}))))]
-      (map-indexed
-       (fn [i b]
-         (if (zero? i)
-           (assoc b :block/left page-id)
-           (let [left (nth blocks (dec i))]
-             (assoc b :block/left [:block/uuid (:block/uuid left)]))))
-       blocks)))
-
-  (defn create-graph!
-    [conn page-size blocks-size start-idx]
-    (let [pages (pages start-idx page-size)
-          page-blocks (map (fn [p]
-                             (cons p
-                                   (blocks (:block/uuid p) blocks-size))) pages)]
-      (doseq [data (partition-all 1000 page-blocks)]
-        (let [tx-data (apply concat data)]
-          (prn :debug :progressing (:block/name (first tx-data)))
-          (d/transact! conn tx-data {:new-graph? true})))))
-
-  (create-graph! conn 30000 20 0))

src/main/electron/listener.cljs

@@ -87,13 +87,13 @@
                          ;; No error handling required, as a page name is always valid
                          ;; Open new page if the page does not exist
                          (if whiteboard?
-                           (route-handler/redirect-to-whiteboard! page-name {:block-id block-id})
+                           (route-handler/redirect-to-page! page-name {:block-id block-id})
                            (editor-handler/insert-first-page-block-if-not-exists! db-page-name)))
 
                        block-id
                        (if-let [block (db-model/get-block-by-uuid block-id)]
                          (if (pu/shape-block? block)
-                           (route-handler/redirect-to-whiteboard! (get-in block [:block/page :block/name]) {:block-id block-id})
+                           (route-handler/redirect-to-page! (get-in block [:block/page :block/uuid]) {:block-id block-id})
                            (route-handler/redirect-to-page! block-id))
                          (notification/show! (str "Open link failed. Block-id `" block-id "` doesn't exist in the graph.") :error false))
 

src/main/frontend/colors.cljs

@@ -12,21 +12,6 @@
   ([color value alpha?]
    (str "var(--rx-" (name color) "-" (cond-> value keyword? name) (if alpha? "-alpha" "") ")")))
 
-(defn linear-gradient [color-name color-stop gradient-level]
-  (let [color-index (.indexOf color-list (keyword color-name))
-        step (fn [dist]
-               (str "var(--rx-"
-                 (name (nth color-list (mod (+ color-index dist) (count color-list))))
-                 "-" (name color-stop) ")"))]
-    (case gradient-level
-      2 (str "linear-gradient(-45deg, " (step -1) " -50%, " (step 0) " 50%, " (step 1) " 150%)")
-      3 (str "linear-gradient(-45deg, " (step -1) " 0%, " (step 0) " 50%, " (step 1) " 100%)")
-      4 (str "linear-gradient(-45deg, " (step -2) " -16.66%, " (step -1) " 16.66%, " (step 0) " 50%, " (step 1) " 83.33%, " (step 2) " 116.66%)")
-      5 (str "linear-gradient(-45deg, " (step -2) " 0%, " (step -1) " 25%, " (step 0) " 50%, " (step 1) " 75%, " (step 2) " 100%)")
-      6 (str "linear-gradient(-45deg, " (step -3) " -10%, " (step -2) " 10%, " (step -1) " 30%, " (step 0) " 50%, " (step 1) " 70%, " (step 2) " 90%, " (step 3) " 110%)")
-      7 (str "linear-gradient(-45deg, " (step -3) " 0%, " (step -2) " 16.66%, " (step -1) " 33.33%, " (step 0) " 50%, " (step 1) " 66.66%, " (step 2) " 83.33%, " (step 3) " 100%)")
-      (str "linear-gradient(90deg, " (step 0) ", " (step 0) ")"))))
-
 (defn get-accent-color
   []
   (when-let [color (some-> js/document.documentElement

src/main/frontend/common_keywords.cljs

@@ -22,7 +22,7 @@
 (sr/defkeyword :block/parent
   "page blocks don't have this attr")
 
-(sr/defkeyword :block/left
+(sr/defkeyword :block/order
   "
 - page blocks don't have this attr
 - some no-order blocks don't have this attr too,

src/main/frontend/components/block.css

@@ -215,6 +215,10 @@
       @apply relative left-[-3px];
     }
   }
+
+  &.bullet-hidden {
+    padding-right: 2px;
+  }
 }
 
 .block-title-wrap {
@@ -403,7 +407,7 @@
 }
 
 .block-main-container {
-  @apply min-h-[26px];
+  @apply min-h-[24px];
 
   &:has(h1.as-heading) {
     .block-control-wrap {

src/main/frontend/components/block/macros.cljs

@@ -4,11 +4,8 @@
             [frontend.extensions.sci :as sci]
             [frontend.handler.common :as common-handler]
             [frontend.handler.property.util :as pu]
-            [frontend.handler.db-based.property.util :as db-pu]
-            [frontend.state :as state]
             [goog.string :as gstring]
-            [goog.string.format]
-            [frontend.config :as config]))
+            [goog.string.format]))
 
 (defn- normalize-query-function
   [ast result]
@@ -47,12 +44,7 @@
 
            (let [vals (map #(pu/lookup-by-name (:block/properties %) f) result)
                  int? (some integer? vals)
-                 repo (state/get-current-repo)
-                 prop-key (if (config/db-based-graph? repo)
-                            (or (db-pu/get-user-property-uuid repo f)
-                                ;; Fall back to the keyword for queries that set named properties through :result-transform
-                                f)
-                            f)]
+                 prop-key f]
              `(~'fn [~'b]
                     (~'let [~'result-str (~'get-in ~'b [:block/properties ~prop-key])
                             ~'result-num (~'parseFloat ~'result-str)
@@ -78,4 +70,4 @@
     (when (fn? f)
       (try (f query-result)
            (catch :default e
-             (js/console.error e))))))
+             (js/console.error e))))))

src/main/frontend/components/class.cljs

@@ -37,9 +37,9 @@
                {:on-pointer-down
                 (fn [e]
                   (when (util/meta-key? e)
-                    (if-let [page-name (:block/name (db/entity [:block/uuid (some-> (util/evalue e) uuid)]))]
+                    (if-let [page (db/entity [:block/uuid (some-> (util/evalue e) uuid)])]
                       (do
-                        (route-handler/redirect-to-page! page-name)
+                        (route-handler/redirect-to-page! (:block/uuid page))
                         (.preventDefault e))
                       (js/console.error "No selected option found to navigate to"))))})))
 
@@ -75,10 +75,9 @@
         (if config/publishing?
           [:div.col-span-3
            (if-let [parent-class (some-> (:db/id (:class/parent page))
-                                         db/entity
-                                         :block/original-name)]
-             [:a {:on-click #(route-handler/redirect-to-page! parent-class)}
-              parent-class]
+                                         db/entity)]
+             [:a {:on-click #(route-handler/redirect-to-page! (:block/uuid parent-class))}
+              (:block/original-name parent-class)]
              "None")]
           [:div.col-span-3
            (let [parent (some-> (:db/id (:class/parent page))
@@ -90,16 +89,16 @@
                                 (if-let [parent (:class/parent (last parents))]
                                   (recur (conj parents parent))
                                   parents))
-               class-ancestors (map :block/original-name (reverse ancestor-pages))]
+               class-ancestors (reverse ancestor-pages)]
            (when (> (count class-ancestors) 2)
              [:div.grid.grid-cols-5.gap-1.items-center.class-ancestors
               [:div.col-span-2 "Ancestor classes:"]
               [:div.col-span-3
                (interpose [:span.opacity-50.text-sm " > "]
-                          (map (fn [class-name]
+                          (map (fn [{class-name :block/original-name :as ancestor}]
                                  (if (= class-name (:block/original-name page))
                                    [:span class-name]
-                                   [:a {:on-click #(route-handler/redirect-to-page! class-name)} class-name]))
+                                   [:a {:on-click #(route-handler/redirect-to-page! (:block/uuid ancestor))} class-name]))
                                class-ancestors))]])))])))
 
 (defn class-children-aux

src/main/frontend/components/cmdk.cljs

@@ -108,7 +108,7 @@
                             :else
                             (take 5 items))))
         page-exists? (when-not (string/blank? input)
-                       (db/entity [:block/name (string/trim input)]))
+                       (db/get-page (string/trim input)))
         include-slash? (or (string/includes? input "/")
                            (string/starts-with? input "/"))
         order* (cond
@@ -218,15 +218,15 @@
                        (remove nil?)
                        (map
                         (fn [page]
-                          (let [entity (db/entity [:block/name (util/page-name-sanity-lc page)])
+                          (let [entity (db/get-page page)
                                 whiteboard? (= (:block/type entity) "whiteboard")
-                                source-page (model/get-alias-source-page repo page)]
+                                source-page (model/get-alias-source-page repo (:db/id entity))]
                             (hash-map :icon (if whiteboard? "whiteboard" "page")
                                       :icon-theme :gray
                                       :text page
                                       :source-page (if source-page
-                                              (:block/original-name source-page)
-                                              page))))))]
+                                                     (:block/original-name source-page)
+                                                     page))))))]
       (swap! !results update group        merge {:status :success :items items}))))
 
 (defmethod load-results :whiteboards [group state]
@@ -240,7 +240,7 @@
                        (remove nil?)
                        (keep
                         (fn [page]
-                          (let [entity (db/entity [:block/name (util/page-name-sanity-lc page)])
+                          (let [entity (db/get-page page)
                                 whiteboard? (= (:block/type entity) "whiteboard")]
                             (when whiteboard?
                               (hash-map :icon "whiteboard"
@@ -316,27 +316,6 @@
                    files)]
       (swap! !results update group        merge {:status :success :items items}))))
 
-;; FIXME: recent search
-;; (defmethod load-results :recents [group state]
-;;   (let [!input (::input state)
-;;         !results (::results state)
-;;         recent-searches (mapv (fn [q] {:type :search :data q}) (db/get-key-value :recent/search))
-;;         recent-pages (->> (filter string? (db/get-key-value :recent/pages))
-;;                           (keep (fn [page]
-;;                                   (when-let [page-entity (db/entity [:block/name (util/page-name-sanity-lc page)])]
-;;                                     {:type :page :data (:block/original-name page-entity)})))
-;;                           vec)]
-;;     (swap! !results assoc-in [group :status] :loading)
-;;     (let [items (->> (concat recent-searches recent-pages)
-;;                      (filter #(string/includes? (lower-case-str (:data %)) (lower-case-str @!input)))
-;;                      (map #(hash-map :icon (if (= :page (:type %)) "page" "history")
-;;                                      :icon-theme :gray
-;;                                      :text (:data %)
-;;                                      :source-recent %
-;;                                      :source-page (when (= :page (:type %)) (:data %))
-;;                                      :source-search (when (= :search (:type %)) (:data %)))))]
-;;       (swap! !results update group merge {:status :success :items items}))))
-
 (defn- get-filter-q
   [input]
   (or (when (string/starts-with? input "/")
@@ -400,45 +379,49 @@
 
 (defmulti handle-action (fn [action _state _event] action))
 
-(defn- get-highlighted-page-name
+(defn- get-highlighted-page-uuid-or-name
   [state]
   (let [highlighted-item (some-> state state->highlighted-item)]
-    (or (when-let [id (:block/uuid (:source-block highlighted-item))]
-          (:block/name (db/entity [:block/uuid id])))
-        (:source-page highlighted-item))))
+    (or (:block/uuid (:source-block highlighted-item))
+        (or (:block/uuid (db/get-case-page (:source-page highlighted-item)))
+            ;; fallback for file graphs
+            (:source-page highlighted-item)))))
 
 (defmethod handle-action :open-page [_ state _event]
-  (when-let [page-name (get-highlighted-page-name state)]
-    (let [redirect-page-name (model/get-redirect-page-name page-name)
-          page (db/entity [:block/name (util/page-name-sanity-lc redirect-page-name)])
-          original-name (:block/original-name page)]
-      (if (= (:block/type page) "whiteboard")
-        (route-handler/redirect-to-whiteboard! original-name)
-        (route-handler/redirect-to-page! original-name)))
+  (when-let [page-name (get-highlighted-page-uuid-or-name state)]
+    (let [page (db/get-page page-name)]
+      (route-handler/redirect-to-page! (:block/uuid page)))
     (state/close-modal!)))
 
 (defmethod handle-action :open-block [_ state _event]
   (when-let [block-id (some-> state state->highlighted-item :source-block :block/uuid)]
     (p/let [repo (state/get-current-repo)
-            _ (db-async/<get-block repo block-id :children? false)]
-      (let [get-block-page (partial model/get-block-page repo)
-           block (db/entity [:block/uuid block-id])]
-       (when block
-         (when-let [page (some-> block-id get-block-page)]
-           (let [page-name (:block/name page)]
-             (cond
-               (= (:block/type page) "whiteboard")
-               (route-handler/redirect-to-whiteboard! page-name {:block-id block-id})
-               (model/parents-collapsed? (state/get-current-repo) block-id)
-               (route-handler/redirect-to-page! block-id)
-               :else
-               (route-handler/redirect-to-page! page-name {:anchor (str "ls-block-" block-id)})))
-           (state/close-modal!)))))))
+            _ (db-async/<get-block repo block-id :children? false)
+            block (db/entity [:block/uuid block-id])
+            parents (db-async/<get-block-parents (state/get-current-repo) (:db/id block) 1000)
+            created-from-block (some (fn [block']
+                                       (let [block (db/entity (:db/id block'))]
+                                         (when (:logseq.property/created-from-property block)
+                                           (:block/parent block)))) parents)
+            [block-id block] (if created-from-block
+                               (let [block (db/entity (:db/id created-from-block))]
+                                 [(:block/uuid block) block])
+                               [block-id block])]
+      (let [get-block-page (partial model/get-block-page repo)]
+        (when block
+          (when-let [page (some-> block-id get-block-page)]
+            (cond
+              (db/whiteboard-page? page)
+              (route-handler/redirect-to-page! (:block/uuid page) {:block-id block-id})
+              (model/parents-collapsed? (state/get-current-repo) block-id)
+              (route-handler/redirect-to-page! block-id)
+              :else
+              (route-handler/redirect-to-page! (:block/uuid page) {:anchor (str "ls-block-" block-id)}))
+            (state/close-modal!)))))))
 
 (defmethod handle-action :open-page-right [_ state _event]
-  (when-let [page-name (get-highlighted-page-name state)]
-    (let [redirect-page-name (model/get-redirect-page-name page-name)
-          page (db/entity [:block/name (util/page-name-sanity-lc redirect-page-name)])]
+  (when-let [page-name (get-highlighted-page-uuid-or-name state)]
+    (let [page (db/get-page page-name)]
       (when page
         (editor-handler/open-block-in-sidebar! (:block/uuid page))))
     (state/close-modal!)))
@@ -512,7 +495,7 @@
        create-whiteboard? (whiteboard-handler/<create-new-whiteboard-and-redirect! @!input)
        create-page? (page-handler/<create! @!input {:redirect? true}))
      (if create-class?
-       (state/pub-event! [:class/configure (db/entity [:block/name (util/page-name-sanity-lc class)])])
+       (state/pub-event! [:class/configure (db/get-page class)])
        (state/close-modal!)))))
 
 (defn- get-filter-user-input

src/main/frontend/components/container.cljs

@@ -73,85 +73,80 @@
       (when child [:div.bd child])]]))
 
 (rum/defc page-name
-  [name icon recent?]
-  (let [original-name (db-model/get-page-original-name name)
-        whiteboard-page? (db-model/whiteboard-page? name)
-        untitled? (db-model/untitled-page? name)
-        name (util/safe-page-name-sanity-lc name)
+  [page icon recent?]
+  (let [repo (state/get-current-repo)
+        page (or (db/get-alias-source-page repo (:db/id page)) page)
+        original-name (:block/original-name page)
+        whiteboard-page? (db-model/whiteboard-page? page)
+        untitled? (db-model/untitled-page? original-name)
+        name (:block/name page)
         file-rpath (when (util/electron?) (page-util/get-page-file-rpath name))
-        source-page (db-model/get-alias-source-page (state/get-current-repo) name)
         ctx-icon #(shui/tabler-icon %1 {:class "scale-90 pr-1 opacity-80"})
-        open-in-sidebar #(when-let [page-entity (and (not whiteboard-page?)
-                                                  (if (empty? source-page)
-                                                    (db/entity [:block/name name]) source-page))]
-                           (state/sidebar-add-block!
-                             (state/get-current-repo)
-                             (:db/id page-entity)
-                             :page))
+        open-in-sidebar #(state/sidebar-add-block!
+                          (state/get-current-repo)
+                          (:db/id page)
+                          :page)
         x-menu-content (fn []
                          (let [x-menu-item shui/dropdown-menu-item
                                x-menu-shortcut shui/dropdown-menu-shortcut]
                            [:<>
                             (when-not recent?
                               (x-menu-item
-                                {:on-click #(page-handler/<unfavorite-page! original-name)}
-                                (ctx-icon "star-off")
-                                (t :page/unfavorite)
-                                (x-menu-shortcut (when-let [binding (shortcut-dh/shortcut-binding :command/toggle-favorite)]
-                                                   (some-> binding
-                                                     (first)
-                                                     (shortcut-utils/decorate-binding))))))
+                               {:on-click #(page-handler/<unfavorite-page! original-name)}
+                               (ctx-icon "star-off")
+                               (t :page/unfavorite)
+                               (x-menu-shortcut (when-let [binding (shortcut-dh/shortcut-binding :command/toggle-favorite)]
+                                                  (some-> binding
+                                                          (first)
+                                                          (shortcut-utils/decorate-binding))))))
                             (when-let [page-fpath (and (util/electron?) file-rpath
-                                                    (config/get-repo-fpath (state/get-current-repo) file-rpath))]
+                                                       (config/get-repo-fpath (state/get-current-repo) file-rpath))]
                               [:<>
                                (x-menu-item
-                                 {:on-click #(ipc/ipc :openFileInFolder page-fpath)}
-                                 (ctx-icon "folder")
-                                 (t :page/open-in-finder))
+                                {:on-click #(ipc/ipc :openFileInFolder page-fpath)}
+                                (ctx-icon "folder")
+                                (t :page/open-in-finder))
 
                                (x-menu-item
-                                 {:on-click #(js/window.apis.openPath page-fpath)}
-                                 (ctx-icon "file")
-                                 (t :page/open-with-default-app))])
+                                {:on-click #(js/window.apis.openPath page-fpath)}
+                                (ctx-icon "file")
+                                (t :page/open-with-default-app))])
                             (x-menu-item
-                              {:on-click open-in-sidebar}
-                              (ctx-icon "layout-sidebar-right")
-                              (t :content/open-in-sidebar)
-                              (x-menu-shortcut (shortcut-utils/decorate-binding "shift+click")))]))]
+                             {:on-click open-in-sidebar}
+                             (ctx-icon "layout-sidebar-right")
+                             (t :content/open-in-sidebar)
+                             (x-menu-shortcut (shortcut-utils/decorate-binding "shift+click")))]))]
 
     ;; TODO: move to standalone component
     [:a.flex.items-center.justify-between.relative.group
      {:on-click
       (fn [e]
-        (let [name (if (empty? source-page) name (:block/name source-page))]
-          (if (gobj/get e "shiftKey")
-            (open-in-sidebar)
-            (if whiteboard-page?
-              (route-handler/redirect-to-whiteboard! name {:click-from-recent? recent?})
-              (route-handler/redirect-to-page! name {:click-from-recent? recent?})))))
+        (if (gobj/get e "shiftKey")
+          (open-in-sidebar)
+          (route-handler/redirect-to-page! (:block/uuid page) {:click-from-recent? recent?})))
       :on-context-menu (fn [^js e]
                          (shui/popup-show! e (x-menu-content)
-                           {:as-dropdown? true
-                            :content-props {:on-click (fn [] (shui/popup-hide!))
-                                            :class "w-60"}})
+                                           {:as-dropdown? true
+                                            :content-props {:on-click (fn [] (shui/popup-hide!))
+                                                            :class "w-60"}})
                          (util/stop e))}
      [:span.page-icon.ml-3.justify-center (if whiteboard-page? (ui/icon "whiteboard" {:extension? true}) icon)]
      [:span.page-title {:class (when untitled? "opacity-50")}
       (if untitled? (t :untitled)
-                    (pdf-utils/fix-local-asset-pagename original-name))]
+          (pdf-utils/fix-local-asset-pagename original-name))]
 
      ;; dots trigger
      (shui/button
-       {:size :sm
-        :variant :ghost
-        :class "absolute right-2 top-0 px-1.5 scale-75 opacity-30 hidden group-hover:block hover:opacity-80 active:opacity-100"
-        :on-click #(do
-                     (shui/popup-show! (.-target %) (x-menu-content)
-                       {:as-dropdown? true
-                        :content-props {:on-click (fn [] (shui/popup-hide!))
-                                        :class "w-60"}})
-                     (util/stop %))}
-       [:i.relative {:style {:top "1px"}} (shui/tabler-icon "dots")])]))
+      {:size :sm
+       :variant :ghost
+       :class "absolute right-2 top-0 px-1.5 scale-75 opacity-30 hidden group-hover:block hover:opacity-80 active:opacity-100"
+       :on-click #(do
+                    (shui/popup-show! (.-target %) (x-menu-content)
+                                      {:as-dropdown? true
+                                       :content-props {:on-click (fn [] (shui/popup-hide!))
+                                                       :class "w-60"}})
+                    (util/stop %))}
+      [:i.relative {:style {:top "1px"}} (shui/tabler-icon "dots")])]))
 
 ;; Fall back to default if icon is undefined or empty
 
@@ -173,11 +168,10 @@
      (when (seq favorite-entities)
        (let [favorites (map
                         (fn [e]
-                          (let [name (:block/name e)
-                                icon (icon/get-page-icon e {})]
+                          (let [icon (icon/get-page-icon e {})]
                             {:id (str (:db/id e))
-                             :value name
-                             :content [:li.favorite-item (page-name name icon false)]}))
+                             :value (:block/uuid e)
+                             :content [:li.favorite-item (page-name e icon false)]}))
                         favorite-entities)]
          (dnd-component/items favorites
                               {:on-drag-end (fn [favorites]
@@ -197,15 +191,14 @@
       :count (count pages)}
 
      [:ul.text-sm
-      (for [name pages]
-        (when-let [entity (db/entity [:block/name (util/safe-page-name-sanity-lc name)])]
-          [:li.recent-item.select-none
-           {:key name
-            :title name
-            :draggable true
-            :on-drag-start (fn [event] (editor-handler/block->data-transfer! name event))
-            :data-ref name}
-           (page-name name (icon/get-page-icon entity {}) true)]))])))
+      (for [page pages]
+        [:li.recent-item.select-none
+         {:key (str "recent-" (:db/id page))
+          :title (:block/original-name page)
+          :draggable true
+          :on-drag-start (fn [event] (editor-handler/block->data-transfer! (:block/name page) event true))
+          :data-ref name}
+         (page-name page (icon/get-page-icon page {}) true)])])))
 
 (rum/defcs flashcards < db-mixins/query rum/reactive
   {:did-mount (fn [state]
@@ -232,7 +225,7 @@
     (let [page (:page default-home)
           page (when (and (string? page)
                           (not (string/blank? page)))
-                 (db/entity [:block/name (util/safe-page-name-sanity-lc page)]))]
+                 (db/get-page page))]
       (if page
         default-home
         (dissoc default-home :page)))))
@@ -538,7 +531,7 @@
                  (state/set-editor-last-pos! pos)))
     :onStop (fn [_event]
               (when-let [block (get @(get @state/state :editor/block) :block/uuid)]
-                (editor-handler/edit-block! block :max nil)
+                (editor-handler/edit-block! block :max)
                 (when-let [input (state/get-input)]
                   (when-let [saved-cursor (state/get-editor-last-pos)]
                     (cursor/move-cursor-to input saved-cursor)))))}
@@ -659,7 +652,7 @@
                    (let [page (util/safe-page-name-sanity-lc page)
                          [db-id block-type] (if (= page "contents")
                                               ["contents" :contents]
-                                              [(:db/id (db/pull [:block/name page])) :page])]
+                                              [(:db/id (db/get-page page)) :page])]
                      (state/sidebar-add-block! current-repo db-id block-type)))
                  (reset! sidebar-inited? true))))
            (when (state/mobile?)
@@ -838,7 +831,7 @@
                   block-el (.closest target ".bullet-container[blockid]")
                   block-id (some-> block-el (.getAttribute "blockid"))
                   {:keys [block block-ref]} (state/sub :block-ref/context)
-                  {:keys [page]} (state/sub :page-title/context)]
+                  {:keys [page page-entity]} (state/sub :page-title/context)]
 
               (let [show!
                     (fn [content]
@@ -852,7 +845,7 @@
                     (cond
                       page
                       (do
-                        (show! (cp-content/page-title-custom-context-menu-content page))
+                        (show! (cp-content/page-title-custom-context-menu-content page-entity))
                         (state/set-state! :page-title/context nil))
 
                       block-ref
@@ -893,12 +886,13 @@
                                   (and
                                    ;; FIXME: this does not work on CI tests
                                    util/node-test?
-                                   state/*editor-editing-ref)))
+                                   (state/editing?))))
                           (state/close-modal!)
                           (hide-context-menu-and-clear-selection e)))))))
   [state route-match main-content]
   (let [{:keys [open-fn]} state
         current-repo (state/sub :git/current-repo)
+        selection-mode? (state/sub :selection/mode)
         granted? (state/sub [:nfs/user-granted? (state/get-current-repo)])
         theme (state/sub :ui/theme)
         accent-color (some-> (state/sub :ui/radix-color) (name))
@@ -912,14 +906,15 @@
         onboarding-state (state/sub :file-sync/onboarding-state)
         right-sidebar-blocks (state/sub-right-sidebar-blocks)
         route-name (get-in route-match [:data :name])
-        margin-less-pages? (boolean (#{:graph :whiteboard} route-name))
+        margin-less-pages? (or (boolean (#{:graph} route-name))
+                               (db-model/whiteboard-page? (state/get-current-page)))
         db-restoring? (state/sub :db/restoring?)
         indexeddb-support? (state/sub :indexeddb/support?)
         page? (= :page route-name)
         home? (= :home route-name)
         native-titlebar? (state/sub [:electron/user-cfgs :window/native-titlebar?])
         window-controls? (and (util/electron?) (not util/mac?) (not native-titlebar?))
-        edit? (some? @state/*editor-editing-ref)
+        edit? (state/editing?)
         default-home (get-default-home-if-valid)
         logged? (user-handler/logged-in?)
         fold-button-on-right? (state/enable-fold-button-right?)
@@ -959,7 +954,8 @@
                      (when (= "Enter" (.-key e))
                        (ui/focus-element (ui/main-node))))}
        (t :accessibility/skip-to-main-content)]
-      [:div.#app-container
+      [:div.#app-container (cond-> {} selection-mode?
+                             (assoc :class "blocks-selection-mode"))
        [:div#left-container
         {:class (if (state/sub :ui/sidebar-open?) "overflow-hidden" "w-full")}
         (header/header {:open-fn        open-fn
@@ -1002,10 +998,10 @@
       (select/select-modal)
       (custom-context-menu)
       (plugins/custom-js-installer
-        {:t t
-         :current-repo current-repo
-         :nfs-granted? granted?
-         :db-restoring? db-restoring?})
+       {:t t
+        :current-repo current-repo
+        :nfs-granted? granted?
+        :db-restoring? db-restoring?})
       (app-context-menu-observer)
 
       [:a#download.hidden]

src/main/frontend/components/container.css

@@ -743,10 +743,6 @@
       }
     }
   }
-
-  .page-hierarchy {
-    @apply pl-[28px];
-  }
 }
 
 .cp__sidebar-main-content[data-is-full-width='true'] {
@@ -817,3 +813,7 @@ html[data-theme='dark'] {
     }
   }
 }
+
+.blocks-selection-mode .page-title, .blocks-selection-mode .block-content-inner, .blocks-selection-mode .block-body, .blocks-selection-mode .ls-properties-area {
+    user-select: none;
+}

src/main/frontend/components/content.cljs

@@ -41,9 +41,13 @@
   []
   (let [repo (state/get-current-repo)]
     [:<>
-     (ui/menu-background-color #(property-handler/batch-set-block-property! repo (state/get-selection-block-ids) :background-color %)
-                               #(property-handler/batch-remove-block-property! repo (state/get-selection-block-ids) :background-color))
-
+     (ui/menu-background-color #(property-handler/batch-set-block-property! repo
+                                                                            (state/get-selection-block-ids)
+                                                                            (pu/get-pid :logseq.property/background-color)
+                                                                            %)
+                               #(property-handler/batch-remove-block-property! repo
+                                                                               (state/get-selection-block-ids)
+                                                                               (pu/get-pid :logseq.property/background-color)))
      (ui/menu-heading #(editor-handler/batch-set-heading! (state/get-selection-block-ids) %)
                       #(editor-handler/batch-set-heading! (state/get-selection-block-ids) true)
                       #(editor-handler/batch-remove-heading! (state/get-selection-block-ids)))
@@ -170,9 +174,11 @@
                                           [:p (t :context-menu/template-exists-warning)]
                                           :error)
                                          (p/do!
-                                          (property-handler/set-block-property! repo block-id :template title)
+                                          (property-handler/set-block-property! repo block-id (pu/get-pid :logseq.property/template) title)
                                           (when (false? template-including-parent?)
-                                            (property-handler/set-block-property! repo block-id :template-including-parent false))
+                                            (property-handler/set-block-property! repo block-id
+                                                                                  (pu/get-pid :logseq.property/template-including-parent)
+                                                                                  false))
                                           (state/hide-custom-context-menu!))))))))]
          (shui/dropdown-menu-separator)])
       (shui/dropdown-menu-item
@@ -191,10 +197,12 @@
       (let [properties (:block/properties block)
             heading (or (pu/lookup properties :logseq.property/heading)
                         false)]
-
         [:<>
-         (ui/menu-background-color #(property-handler/set-block-property! repo block-id :background-color %)
-                                   #(property-handler/remove-block-property! repo block-id :background-color))
+         (ui/menu-background-color #(property-handler/set-block-property! repo block-id
+                                                                          (pu/get-pid :logseq.property/background-color)
+                                                                          %)
+                                   #(property-handler/remove-block-property! repo block-id
+                                                                             (pu/get-pid :logseq.property/background-color)))
 
          (ui/menu-heading heading
                           #(editor-handler/set-heading! block-id %)
@@ -384,7 +392,7 @@
 
 (rum/defc page-title-custom-context-menu-content
   [page]
-  (when-not (string/blank? page)
+  (when page
     (let [page-menu-options (page-menu/page-menu page)]
       [:<>
        (for [{:keys [title options]} page-menu-options]

src/main/frontend/components/db_based/page.cljs

@@ -5,101 +5,67 @@
             [frontend.components.class :as class-component]
             [frontend.components.property :as property-component]
             [frontend.components.property.value :as pv]
-            [frontend.components.icon :as icon-component]
             [frontend.config :as config]
             [frontend.db :as db]
             [frontend.handler.db-based.property :as db-property-handler]
-            [frontend.handler.property.util :as pu]
-            [frontend.handler.db-based.property.util :as db-pu]
             [frontend.ui :as ui]
             [frontend.state :as state]
             [rum.core :as rum]
             [logseq.shui.ui :as shui]
             [frontend.util :as util]
             [clojure.set :as set]
-            [clojure.string :as string]))
+            [clojure.string :as string]
+            [logseq.db.frontend.property :as db-property]))
 
-(rum/defc page-properties < rum/reactive
+(rum/defc page-properties
+  "This component is called by page-inner and within configure/info modal. This should not
+   be displaying properties from both components at the same time"
+  < rum/reactive
   [page {:keys [mode configure?]}]
-  (let [types (:block/type page)
-        class? (= mode :class)
-        property? (contains? types "property")
+  (let [class? (= mode :class)
         edit-input-id-prefix (str "edit-block-" (:block/uuid page))
         configure-opts {:selected? false
-                        :page-configure? true}
+                        :page-configure? configure?}
         has-viewable-properties? (db-property-handler/block-has-viewable-properties? page)
-        has-class-properties? (seq (:properties (:block/schema page)))
-        has-tags? (seq (:block/tags page))
-        hide-properties? (get-in page [:block/properties (:block/uuid (db/entity :logseq.property/hide-properties?))])]
+        has-class-properties? (seq (:class/schema.properties page))
+        hide-properties? (:logseq.property/hide-properties? page)]
     (when (or configure?
               (and
                (not hide-properties?)
                (or has-viewable-properties?
-                   has-class-properties?
-                   property?
-                   has-tags?)))
+                   has-class-properties?)))
       [:div.ls-page-properties
-       {:class (util/classnames [{:no-mode (nil? mode)
-                                  :no-properties (if class?
+       {:class (util/classnames [{:no-properties (if class?
                                                    (not has-class-properties?)
                                                    (not has-viewable-properties?))}])}
-       (cond
-         (= mode :class)
-         (if (and config/publishing? (not configure?))
-           (component-block/db-properties-cp {:editor-box editor/box}
-                                             page
-                                             (str edit-input-id-prefix "-page")
-                                             (assoc configure-opts :class-schema? false))
+       (if configure?
+         (cond
+           (= mode :class)
            (component-block/db-properties-cp {:editor-box editor/box}
                                              page
                                              (str edit-input-id-prefix "-schema")
-                                             (assoc configure-opts :class-schema? true)))
+                                             (assoc configure-opts :class-schema? true))
 
-         (= mode :page)
+           (= mode :page)
+           (component-block/db-properties-cp {:editor-box editor/box}
+                                             page
+                                             (str edit-input-id-prefix "-page")
+                                             (assoc configure-opts :class-schema? false :page? true)))
+         ;; default view for page-inner
          (component-block/db-properties-cp {:editor-box editor/box}
                                            page
                                            (str edit-input-id-prefix "-page")
                                            (assoc configure-opts :class-schema? false :page? true)))])))
 
-(rum/defc icon-row < rum/reactive
-  [page]
-  [:div.grid.grid-cols-5.gap-1.items-center
-   [:label.col-span-2 "Icon:"]
-   (let [icon-value (pu/get-block-property-value page :logseq.property/icon)]
-     [:div.col-span-3.flex.flex-row.items-center.gap-2
-      (icon-component/icon-picker icon-value
-                                  {:disabled? config/publishing?
-                                   :on-chosen (fn [_e icon]
-                                                (let [icon-property-id (db-pu/get-built-in-property-uuid :logseq.property/icon)]
-                                                  (db-property-handler/<update-property!
-                                                   (state/get-current-repo)
-                                                   (:block/uuid page)
-                                                   {:properties {icon-property-id icon}})))})
-      (when (and icon-value (not config/publishing?))
-        [:a.fade-link.flex {:on-click (fn [_e]
-                                        (db-property-handler/remove-block-property!
-                                         (state/get-current-repo)
-                                         (:block/uuid page)
-                                         (db-pu/get-built-in-property-uuid :logseq.property/icon)))
-                            :title "Delete this icon"}
-         (ui/icon "X")])])])
-
 (rum/defc tags
   [page]
-  (let [tags-property (db/entity :logseq.property/tags)]
+  (let [tags-property (db/entity :block/tags)]
     (pv/property-value page tags-property
-                       (map :block/uuid (:block/tags page))
+                       (:block/tags page)
                        {:page-cp (fn [config page]
                                    (component-block/page-cp (assoc config :tag? true) page))
                         :inline-text component-block/inline-text})))
 
-(rum/defc tags-row < rum/reactive
-  [page]
-  [:div.grid.grid-cols-5.gap-1.items-center
-   [:label.col-span-2 "Tags:"]
-   [:div.col-span-3.flex.flex-row.items-center.gap-2
-    (tags page)]])
-
 (rum/defcs page-configure < rum/reactive
   [state page *mode]
   (let [*mode *mode
@@ -110,19 +76,20 @@
         page-opts {:configure? true}]
     (when (nil? mode)
       (reset! *mode (cond
-                      property? :property
                       class? :class
+                      property? :property
                       :else :page)))
-    [:div.flex.flex-col.gap-1.page-configure
-     (if (= mode :property)
+    [:div.flex.flex-col.gap-1.pb-4
+     (case mode
+       :property
        (property-component/property-config page {:inline-text component-block/inline-text})
-       [:<>
-        (when (= mode :class)
-          (class-component/configure page {:show-title? false}))
-        (when-not (or config/publishing? class?)
-          (tags-row page))
-        (when-not config/publishing? (icon-row page))
-        (page-properties page (assoc page-opts :mode mode))])]))
+
+       :class
+       [:div.mt-2.flex.flex-col.gap-2
+        (class-component/configure page {:show-title? false})
+        (page-properties page (assoc page-opts :mode mode))]
+
+       (page-properties page (assoc page-opts :mode mode)))]))
 
 (rum/defc page-properties-react < rum/reactive
   [page* page-opts]
@@ -135,12 +102,10 @@
 (rum/defc mode-switch < rum/reactive
   [types *mode]
   (let [current-mode (rum/react *mode)
-        property? (contains? types "property")
         class? (contains? types "class")
+        property? (contains? types "property")
         modes (->
                (cond
-                 (and property? class?)
-                 ["Property" "Class"]
                  property?
                  ["Property"]
                  class?
@@ -163,71 +128,50 @@
 (rum/defcs page-info < rum/reactive
   (rum/local false ::hover?)
   (rum/local nil ::mode)
-  {:init (fn [state]
-           (assoc state ::collapsed? (atom true)))}
-  [state page *hover-title?]
+  [state page *show-info?]
   (let [page (db/sub-block (:db/id page))
-        *collapsed? (::collapsed? state)
         *hover? (::hover? state)
         *mode (::mode state)
         types (:block/type page)
         class? (contains? types "class")
-        hover-title? (rum/react *hover-title?)
-        collapsed? (rum/react *collapsed?)
-        has-tags? (seq (:block/tags page))
-        has-properties? (seq (:block/properties page))
-        hover-or-expanded? (or @*hover? hover-title? (not collapsed?))
-        show-info? (or hover-or-expanded? has-tags? has-properties? class?)]
+        collapsed? (not @*show-info?)
+        has-properties? (seq (remove (set (keys db-property/built-in-properties))
+                                     (keys (:block/properties page))))
+        show-info? (or @*show-info? has-properties?)]
     (when (if config/publishing?
             ;; Since publishing is read-only, hide this component if it has no info to show
             ;; as it creates a fair amount of empty vertical space
-            (or has-tags? (some? types))
-            true)
+            (some? types)
+            show-info?)
       [:div.page-info
        {:class (util/classnames [{:is-collapsed collapsed?}])}
-       [:div.page-info-inner
-        [:div.info-title.cursor
-         {:on-mouse-over #(reset! *hover? true)
-          :on-mouse-leave #(when-not (state/dropdown-opened?)
-                             (reset! *hover? false))
-          :on-pointer-up #(reset! *hover? false)
-          :on-click (if config/publishing?
-                      (fn [_]
-                        (when (seq (set/intersection #{"class" "property"} types))
-                          (swap! *collapsed? not)))
-                      #(swap! *collapsed? not))}
-         (when show-info?
+       [:div {:class (if (or @*hover? (not collapsed?))
+                       "border rounded"
+                       "border rounded border-transparent")}
+        (when-not collapsed?
+          [:div.info-title.cursor.py-1
+           {:on-mouse-over #(reset! *hover? true)
+            :on-mouse-leave #(when-not (state/dropdown-opened?)
+                               (reset! *hover? false))
+            :on-click (if config/publishing?
+                        (fn [_]
+                          (when (seq (set/intersection #{"class" "property"} types))
+                            (swap! *show-info? not)))
+                        #(do
+                           (swap! *show-info? not)
+                           (swap! *hover? not)))}
            [:<>
-            [:div.flex.flex-row.items-center.gap-2
-             (if collapsed?
-               (if (or has-tags? @*hover? config/publishing?)
-                 [:<>
-                  (if has-tags?
-                    [:div.px-1 {:style {:min-height 28}}]
-                    [:a.flex.fade-link.ml-2 (ui/icon "tags")])
-                  (if (and config/publishing? (seq (set/intersection #{"class" "property"} types)))
-                    [:div
-                     [:div.opacity-50.pointer.text-sm "Expand for more info"]]
-                    [:div {:on-click util/stop-propagation}
-                     (tags page)])]
-                 [:div.page-info-title-placeholder])
-               [:div.flex.flex-row.items-center
-                [:a.flex.fade-link.ml-3 (ui/icon "info-circle")]
-                (mode-switch types *mode)])]
-            (when (or @*hover? (not collapsed?))
-              [:div.px-1.absolute.right-1.top-0
-               (shui/button
-                 {:variant :ghost :size :sm :class "!px-1"}
-                 (if collapsed?
-                   [:span.opacity-80.flex.items-center
-                    (ui/icon "adjustments-horizontal" {:size 16})]
-                   (ui/icon "x")))])])]
-
-        (when show-info?
-          (if collapsed?
-            (when (or (seq (:block/properties page))
-                      (and class? (seq (:properties (:block/schema page)))))
-              [:div.properties-wrap
-               (page-properties page {:mode (if class? :class :page)})])
-            [:div.configure-wrap
-             (page-configure page *mode)]))]])))
+            [:div.flex.flex-row.items-center.gap-1
+             [:a.flex.fade-link.ml-3 (ui/icon "info-circle")]
+             (mode-switch types *mode)]
+            [:div.px-1.absolute.right-0.top-0
+             (shui/button
+              {:variant :ghost :size :sm}
+              (ui/icon "x"))]]])
+        (if collapsed?
+          (when (or (seq (:block/properties page))
+                    (and class? (seq (:class/schema.properties page))))
+            [:div.px-2 {:style {:margin-left 2}}
+             (page-properties page {:mode @*mode})])
+          [:div.px-3
+           (page-configure page *mode)])]])))

src/main/frontend/components/dnd.cljs

@@ -43,6 +43,7 @@
                   :onDragEnd (fn [event]
                                (let [active-id (.-id (.-active event))
                                      over-id (.-id (.-over event))]
+                                 (js/console.dir event)
                                  (when-not (= active-id over-id)
                                    (let [old-index (.indexOf ids active-id)
                                          new-index (.indexOf ids over-id)
@@ -61,7 +62,11 @@
                                               :new-items new-items})
                                            (do
                                              (set-items new-items)
-                                             (on-drag-end new-values)))))))
+                                             (on-drag-end new-values {:active-id active-id
+                                                                      :over-id over-id
+                                                                      :direction (if (> new-index old-index)
+                                                                                   :down
+                                                                                   :up)})))))))
                                  (set-active-id nil)))}
         sortable-opts {:items items
                        :strategy verticalListSortingStrategy}

src/main/frontend/components/editor.cljs

@@ -114,16 +114,16 @@
                         (common-util/safe-subs value (+ (count q) 4 pos)))]
         (state/set-edit-content! (.-id input) value')
         (state/clear-editor-action!)
-        (p/let [page-name (util/page-name-sanity-lc chosen-item)
-                page (db/entity [:block/name page-name])
+        (p/let [page (db/get-page chosen-item)
                 _ (when-not page (page-handler/<create! chosen-item {:redirect? false
                                                                      :create-first-block? false}))
+                page' (db/get-page chosen-item)
                 current-block (state/get-edit-block)]
           (editor-handler/api-insert-new-block! chosen-item
                                                 {:block-uuid (:block/uuid current-block)
                                                  :sibling? true
                                                  :replace-empty-target? true
-                                                 :other-attrs {:block/link (:db/id (db/entity [:block/name page-name]))}}))))
+                                                 :other-attrs {:block/link (:db/id page')}}))))
     (page-handler/on-chosen-handler input id q pos format)))
 
 (rum/defc page-search-aux
@@ -151,7 +151,7 @@
                                       matched-pages)
                               (cons q matched-pages)))
 
-                                ;; reorder, shortest and starts-with first.
+                          ;; reorder, shortest and starts-with first.
                           :else
                           (let [matched-pages (remove nil? matched-pages)
                                 matched-pages (sort-by
@@ -253,10 +253,9 @@
       :on-enter    non-exist-block-handler
       :empty-placeholder   [:div.text-gray-500.text-sm.px-4.py-2 (t :editor/block-search)]
       :item-render (fn [{:block/keys [page uuid]}]  ;; content returned from search engine is normalized
-                     (let [page (or (:block/original-name page)
-                                    (:block/name page))
+                     (let [page-entity (db/entity [:block/uuid page])
                            repo (state/sub :git/current-repo)
-                           format (db/get-page-format page)
+                           format (get page-entity :block/format :markdown)
                            block (db-model/query-block-by-uuid uuid)
                            content (:block/content block)]
                        (when-not (string/blank? content)
@@ -607,8 +606,7 @@
   "Get textarea css class according to it's content"
   [block content format]
   (let [content (if content (str content) "")
-        properties (:block/properties block)
-        heading (pu/lookup properties :logseq.property/heading)
+        heading (pu/get-block-property-value block :logseq.property/heading)
         heading (if (true? heading)
                   (min (inc (:block/level block)) 6)
                   heading)]

src/main/frontend/components/export.cljs

@@ -95,14 +95,15 @@
 
 (defn- get-image-blob
   [block-uuids-or-page-name {:keys [transparent-bg? x y width height zoom]} callback]
-  (let [style (js/window.getComputedStyle js/document.body)
+  (let [top-block-id (if (coll? block-uuids-or-page-name) (first block-uuids-or-page-name) block-uuids-or-page-name)
+        style (js/window.getComputedStyle js/document.body)
         background (when-not transparent-bg? (.getPropertyValue style "--ls-primary-background-color"))
-        page? (string? block-uuids-or-page-name)
+        page? (and (uuid? top-block-id) (db/page? (db/entity [:block/uuid top-block-id])))
         selector (if page?
                    "#main-content-container"
-                   (str "[blockid='" (str (first block-uuids-or-page-name)) "']"))
+                   (str "[blockid='" top-block-id "']"))
         container  (js/document.querySelector selector)
-        scale (if page? (/ 1 (or zoom (get-zoom-level block-uuids-or-page-name))) 1)
+        scale (if page? (/ 1 (or zoom (get-zoom-level top-block-id))) 1)
         options #js {:allowTaint true
                      :useCORS true
                      :backgroundColor (or background "transparent")
@@ -113,7 +114,7 @@
                      :scrollX 0
                      :scrollY 0
                      :scale scale
-                     :windowHeight (when (string? block-uuids-or-page-name)
+                     :windowHeight (when page?
                                      (.-scrollHeight container))}]
     (-> (js/html2canvas container options)
         (.then (fn [canvas] (.toBlob canvas (fn [blob]
@@ -142,7 +143,7 @@
                  (reset! (::text-indent-style state) (state/get-export-block-text-indent-style))
                  (reset! (::text-other-options state) (state/get-export-block-text-other-options))
                  state)}
-  [state root-block-uuids-or-page-name {:keys [whiteboard?] :as options}]
+  [state root-block-uuids-or-page-uuid {:keys [whiteboard?] :as options}]
   (let [tp @*export-block-type
         *text-other-options (::text-other-options state)
         *text-remove-options (::text-remove-options state)
@@ -157,21 +158,21 @@
          (ui/button "Text"
            :class "mr-4 w-20"
            :on-click #(do (reset! *export-block-type :text)
-                          (reset! *content (export-helper root-block-uuids-or-page-name))))
+                          (reset! *content (export-helper root-block-uuids-or-page-uuid))))
          (ui/button "OPML"
            :class "mr-4 w-20"
            :on-click #(do (reset! *export-block-type :opml)
-                          (reset! *content (export-helper root-block-uuids-or-page-name))))
+                          (reset! *content (export-helper root-block-uuids-or-page-uuid))))
          (ui/button "HTML"
            :class "mr-4 w-20"
            :on-click #(do (reset! *export-block-type :html)
-                          (reset! *content (export-helper root-block-uuids-or-page-name))))
-         (when-not (seq? root-block-uuids-or-page-name)
+                          (reset! *content (export-helper root-block-uuids-or-page-uuid))))
+         (when-not (seq? root-block-uuids-or-page-uuid)
            (ui/button "PNG"
              :class "w-20"
              :on-click #(do (reset! *export-block-type :png)
                             (reset! *content nil)
-                            (get-image-blob root-block-uuids-or-page-name (merge options {:transparent-bg? false}) (fn [blob] (reset! *content blob))))))])
+                            (get-image-blob root-block-uuids-or-page-uuid (merge options {:transparent-bg? false}) (fn [blob] (reset! *content blob))))))])
 
       (if (= :png tp)
         [:div.flex.items-center.justify-center.relative
@@ -186,7 +187,7 @@
          (ui/checkbox {:class "mr-2 ml-4"
                        :on-change (fn [e]
                                     (reset! *content nil)
-                                    (get-image-blob root-block-uuids-or-page-name (merge options {:transparent-bg? e.currentTarget.checked}) (fn [blob] (reset! *content blob))))})]
+                                    (get-image-blob root-block-uuids-or-page-uuid (merge options {:transparent-bg? e.currentTarget.checked}) (fn [blob] (reset! *content blob))))})]
         (let [options (->> text-indent-style-options
                         (mapv (fn [opt]
                                 (if (= @*text-indent-style (:label opt))
@@ -202,7 +203,7 @@
                                 (let [value (util/evalue e)]
                                   (state/set-export-block-text-indent-style! value)
                                   (reset! *text-indent-style value)
-                                  (reset! *content (export-helper root-block-uuids-or-page-name))))}
+                                  (reset! *content (export-helper root-block-uuids-or-page-uuid))))}
                   (for [{:keys [label value selected]} options]
                     [:option (cond->
                                {:key label
@@ -217,7 +218,7 @@
                           :on-change (fn [e]
                                        (state/update-export-block-text-remove-options! e :page-ref)
                                        (reset! *text-remove-options (state/get-export-block-text-remove-options))
-                                       (reset! *content (export-helper root-block-uuids-or-page-name)))})
+                                       (reset! *content (export-helper root-block-uuids-or-page-uuid)))})
             [:div {:style {:visibility (if (#{:text :html :opml} tp) "visible" "hidden")}}
              "[[text]] -> text"]
 
@@ -227,7 +228,7 @@
                           :on-change (fn [e]
                                        (state/update-export-block-text-remove-options! e :emphasis)
                                        (reset! *text-remove-options (state/get-export-block-text-remove-options))
-                                       (reset! *content (export-helper root-block-uuids-or-page-name)))})
+                                       (reset! *content (export-helper root-block-uuids-or-page-uuid)))})
 
             [:div {:style {:visibility (if (#{:text :html :opml} tp) "visible" "hidden")}}
              "remove emphasis"]
@@ -238,7 +239,7 @@
                           :on-change (fn [e]
                                        (state/update-export-block-text-remove-options! e :tag)
                                        (reset! *text-remove-options (state/get-export-block-text-remove-options))
-                                       (reset! *content (export-helper root-block-uuids-or-page-name)))})
+                                       (reset! *content (export-helper root-block-uuids-or-page-uuid)))})
 
             [:div {:style {:visibility (if (#{:text :html :opml} tp) "visible" "hidden")}}
              "remove #tags"]]
@@ -251,7 +252,7 @@
                                        (state/update-export-block-text-other-options!
                                          :newline-after-block (boolean (util/echecked? e)))
                                        (reset! *text-other-options (state/get-export-block-text-other-options))
-                                       (reset! *content (export-helper root-block-uuids-or-page-name)))})
+                                       (reset! *content (export-helper root-block-uuids-or-page-uuid)))})
             [:div {:style {:visibility (if (#{:text} tp) "visible" "hidden")}}
              "newline after block"]
 
@@ -261,7 +262,7 @@
                           :on-change (fn [e]
                                        (state/update-export-block-text-remove-options! e :property)
                                        (reset! *text-remove-options (state/get-export-block-text-remove-options))
-                                       (reset! *content (export-helper root-block-uuids-or-page-name)))})
+                                       (reset! *content (export-helper root-block-uuids-or-page-uuid)))})
             [:div {:style {:visibility (if (#{:text} tp) "visible" "hidden")}}
              "remove properties"]]
 
@@ -276,7 +277,7 @@
                                  level (if (= "all" value) :all (util/safe-parse-int value))]
                              (state/update-export-block-text-other-options! :keep-only-level<=N level)
                              (reset! *text-other-options (state/get-export-block-text-other-options))
-                             (reset! *content (export-helper root-block-uuids-or-page-name))))}
+                             (reset! *content (export-helper root-block-uuids-or-page-uuid))))}
              (for [n (cons "all" (range 1 10))]
                [:option {:key n :value n} n])]]]))
 
@@ -290,8 +291,8 @@
                          (util/copy-to-clipboard! @*content :html (when (= tp :html) @*content)))
                        (reset! *copied? true)))
          (ui/button (t :export-save-to-file)
-           :on-click #(let [file-name (if (string? root-block-uuids-or-page-name)
-                                        (-> (db/get-page root-block-uuids-or-page-name)
+           :on-click #(let [file-name (if (uuid? root-block-uuids-or-page-uuid)
+                                        (-> (db/get-page root-block-uuids-or-page-uuid)
                                           (util/get-page-original-name))
                                         (t/now))]
                         (utils/saveToFile (js/Blob. [@*content]) (str "logseq_" file-name) (if (= tp :text) "txt" (name tp)))))])]]))

src/main/frontend/components/file.cljs

@@ -127,7 +127,7 @@
                                 :href (rfe/href :page {:name original-name})
                                 :on-click (fn [e]
                                             (when (gobj/get e "shiftKey")
-                                              (when-let [page (db/entity [:block/name (util/page-name-sanity-lc original-name)])]
+                                              (when-let [page (db/get-page original-name)]
                                                 (state/sidebar-add-block!
                                                  (state/get-current-repo)
                                                  (:db/id page)