Merge branch 'feat/db' into enhance/reference-view

.clj-kondo/config.edn

@@ -65,6 +65,7 @@
              electron.ipc ipc
              electron.utils utils
              frontend.commands commands
+             frontend.common.file-based.db common-file-db
              frontend.common.date common-date
              frontend.common.file.core common-file
              frontend.common.file.util wfu

.github/workflows/deploy-db-test-pages.yml

@@ -37,11 +37,12 @@ jobs:
 
       - name: Set Build Environment Variables
         run: |
-          echo "ENABLE_FILE_SYNC_PRODUCTION=false" >> $GITHUB_ENV
+          echo "ENABLE_FILE_SYNC_PRODUCTION=true" >> $GITHUB_ENV
+          echo "ENABLE_RTC_SYNC_PRODUCTION=true" >> $GITHUB_ENV
 
       - name: Build Released-Web
         run: |
-          yarn gulp:build && clojure -M:cljs release app  --config-merge '{:compiler-options {:source-map true :source-map-include-sources-content false :source-map-detail-level :symbols}}'
+          yarn gulp:build && clojure -M:cljs release app  --config-merge '{:compiler-options {:source-map true}}'
           rsync -avz --exclude node_modules --exclude android --exclude ios ./static/ ./public/static/
           ls -lR ./public
         env:

deps/db/.carve/ignore

@@ -1,4 +1,6 @@
 ;; API
+logseq.db.file-based.rules/rules
+;; API
 logseq.db.file-based.schema/retract-attributes
 ;; API
 logseq.db.frontend.rules/db-query-dsl-rules

deps/db/src/logseq/db.cljs

@@ -13,7 +13,6 @@
             [logseq.db.common.delete-blocks :as delete-blocks] ;; Load entity extensions
             [logseq.db.common.entity-util :as common-entity-util]
             [logseq.db.common.sqlite :as sqlite-common-db]
-            [logseq.db.file-based.rules :as file-rules]
             [logseq.db.frontend.class :as db-class]
             [logseq.db.frontend.entity-plus :as entity-plus]
             [logseq.db.frontend.entity-util :as entity-util]
@@ -533,39 +532,6 @@
   [db]
   (when db (get-key-value db :logseq.kv/remote-schema-version)))
 
-;; File based fns
-(defn get-namespace-pages
-  "Accepts both sanitized and unsanitized namespaces"
-  [db namespace' {:keys [db-graph?]}]
-  (assert (string? namespace'))
-  (let [namespace'' (common-util/page-name-sanity-lc namespace')
-        pull-attrs  (cond-> [:db/id :block/name :block/title :block/namespace]
-                      (not db-graph?)
-                      (conj {:block/file [:db/id :file/path]}))]
-    (d/q
-     [:find [(list 'pull '?c pull-attrs) '...]
-      :in '$ '% '?namespace
-      :where
-      ['?p :block/name '?namespace]
-      (list 'namespace '?p '?c)]
-     db
-     (:namespace file-rules/rules)
-     namespace'')))
-
-(defn get-pages-by-name-partition
-  [db partition']
-  (when-not (string/blank? partition')
-    (let [partition'' (common-util/page-name-sanity-lc (string/trim partition'))
-          ids (->> (d/datoms db :aevt :block/name)
-                   (filter (fn [datom]
-                             (let [page (:v datom)]
-                               (string/includes? page partition''))))
-                   (map :e))]
-      (when (seq ids)
-        (d/pull-many db
-                     '[:db/id :block/name :block/title]
-                     ids)))))
-
 (defn get-all-properties
   [db]
   (->> (d/datoms db :avet :block/tags :logseq.class/Property)

deps/db/src/logseq/db/frontend/entity_util.cljs

@@ -7,10 +7,11 @@
 
 (defn- has-tag?
   [entity tag-ident]
-  (some (fn [t]
-          (or (keyword-identical? (:db/ident t) tag-ident)
-              (keyword-identical? t tag-ident)))
-        (:block/tags entity)))
+  (when (or (map? entity) (de/entity? entity))
+    (some (fn [t]
+            (or (keyword-identical? (:db/ident t) tag-ident)
+                (keyword-identical? t tag-ident)))
+          (:block/tags entity))))
 
 (comment
   (require '[logseq.common.profile :as c.p])

deps/db/src/logseq/db/sqlite/build.cljs

@@ -469,7 +469,7 @@
              (map #(hash-map :page {:block/title %})))]
     (when (seq new-pages-from-refs)
       (println "Building additional pages from content refs:" (pr-str (mapv #(get-in % [:page :block/title]) new-pages-from-refs))))
-    (concat pages-and-blocks new-pages-from-refs)))
+    (concat new-pages-from-refs pages-and-blocks)))
 
 (defn- add-new-pages-from-properties
   [properties pages-and-blocks]
@@ -485,7 +485,8 @@
     (when (seq new-pages)
       (println "Building additional pages from property values:"
                (pr-str (mapv #(or (get-in % [:page :block/title]) (get-in % [:page :build/journal])) new-pages))))
-    (concat pages-and-blocks new-pages)))
+    ;; new-pages must come first because they are referenced by pages-and-blocks
+    (concat new-pages pages-and-blocks)))
 
 (defn- expand-build-children
   "Expands any blocks with :build/children to return a flattened vec with

deps/db/src/logseq/db/sqlite/export.cljs

@@ -2,6 +2,7 @@
   "Builds sqlite.build EDN to represent nodes in a graph-agnostic way.
    Useful for exporting and importing across DB graphs"
   (:require [clojure.set :as set]
+            [clojure.walk :as walk]
             [datascript.core :as d]
             [datascript.impl.entity :as de]
             [logseq.db :as ldb]
@@ -49,7 +50,10 @@
         (if (= :node (:logseq.property/type property-ent))
           ;; Have to distinguish from block references that don't exist like closed values
           ^::existing-property-value? [:block/uuid (:block/uuid pvalue)]
-          (or (:db/ident pvalue) (db-property/property-value-content pvalue)))))
+          (or (:db/ident pvalue)
+              ;; nbb-compatible version of db-property/property-value-content
+              (or (block-title pvalue)
+                  (:logseq.property/value pvalue))))))
 
 (defn- buildable-properties
   "Originally copied from db-test/readable-properties. Modified so that property values are
@@ -208,7 +212,16 @@
        set))
 
 (defn- merge-export-maps [& export-maps]
-  (let [pages-and-blocks (reduce into [] (keep :pages-and-blocks export-maps))
+  (let [pages-and-blocks
+        (->> (mapcat :pages-and-blocks export-maps)
+             ;; TODO: Group by more correct identity for title, same as check-for-existing-entities
+             (group-by #(select-keys (:page %) [:block/title :build/journal]))
+             (mapv #(apply merge-with (fn [e1 e2]
+                                        ;; merge :page and add :blocks
+                                        (if (and (map? e1) (map e2))
+                                          (merge e1 e2)
+                                          (into e1 e2)))
+                           (second %))))
         ;; Use merge-with to preserve new-property? and to allow full copies to overwrite shallow ones
         properties (apply merge-with merge (keep :properties export-maps))
         classes (apply merge-with merge (keep :classes export-maps))]
@@ -221,8 +234,10 @@
 (defn- build-content-ref-export
   "Builds an export config (and additional info) for refs in the given blocks. All the exported
    entities found in block refs include their uuid in order to preserve the relationship to the blocks"
-  [db page-blocks]
-  (let [content-ref-uuids (set (mapcat (comp db-content/get-matched-ids block-title) page-blocks))
+  [db blocks*]
+  (let [;; Remove property value blocks that can't have content refs
+        blocks (remove :logseq.property/value blocks*)
+        content-ref-uuids (set (mapcat (comp db-content/get-matched-ids block-title) blocks))
         content-ref-ents (map #(d/entity db [:block/uuid %]) content-ref-uuids)
         content-ref-pages (filter #(or (ldb/internal-page? %) (entity-util/journal? %)) content-ref-ents)
         content-ref-properties (when-let [prop-ids (seq (map :db/ident (filter ldb/property? content-ref-ents)))]
@@ -322,7 +337,11 @@
   "Exports block for given block eid"
   [db eid]
   (let [block-entity (d/entity db eid)
-        {:keys [content-ref-uuids content-ref-ents] :as content-ref-export} (build-content-ref-export db [block-entity])
+        property-value-ents (->> (dissoc (db-property/properties block-entity) :block/tags)
+                                 vals
+                                 (filter de/entity?))
+        {:keys [content-ref-uuids content-ref-ents] :as content-ref-export}
+        (build-content-ref-export db (into [block-entity] property-value-ents))
         node-export (build-node-export db block-entity {:include-uuid-fn content-ref-uuids})
         pvalue-uuids (get-pvalue-uuids (:node node-export))
         uuid-block-export (build-uuid-block-export db pvalue-uuids content-ref-ents {})
@@ -345,12 +364,12 @@
   (let [page-entity (d/entity db eid)
         datoms (d/datoms db :avet :block/page eid)
         block-eids (mapv :e datoms)
-        page-blocks (->> block-eids
-                         (map #(d/entity db %))
+        page-blocks* (map #(d/entity db %) block-eids)
+        {:keys [content-ref-uuids content-ref-ents] :as content-ref-export} (build-content-ref-export db page-blocks*)
+        page-blocks (->> page-blocks*
                          (sort-by :block/order)
-                         ;; Remove property value blocks as they are included in the block they belong to
+                         ;; Remove property value blocks as they are exported in a block's :build/properties
                          (remove #(:logseq.property/created-from-property %)))
-        {:keys [content-ref-uuids content-ref-ents] :as content-ref-export} (build-content-ref-export db page-blocks)
         {:keys [pvalue-uuids] :as blocks-export}
         (build-blocks-export db page-blocks {:include-uuid-fn content-ref-uuids})
         uuid-block-export (build-uuid-block-export db pvalue-uuids content-ref-ents {:page-entity page-entity})
@@ -455,48 +474,60 @@
 
 ;; Import fns
 ;; ==========
+(defn- add-uuid-to-page-if-exists
+  [db m]
+  (if-let [ent (some->> (:build/journal m)
+                        (d/datoms db :avet :block/journal-day)
+                        first
+                        :e
+                        (d/entity db))]
+    (assoc m :block/uuid (:block/uuid ent))
+    ;; TODO: For now only check page uniqueness by title. Could handle more uniqueness checks later
+    (if-let [ent (some->> (:block/title m) (ldb/get-case-page db))]
+      (assoc m :block/uuid (:block/uuid ent))
+      m)))
+
 (defn- check-for-existing-entities
   "Checks export map for existing entities and adds :block/uuid to them if they exist in graph to import.
    Also checks for property conflicts between existing properties and properties to be imported"
   [db {:keys [pages-and-blocks classes properties]} property-conflicts]
-  (cond-> {:build-existing-tx? true}
-    (seq pages-and-blocks)
-    (assoc :pages-and-blocks
-           (mapv (fn [m]
-                   (if-let [ent (some->> (get-in m [:page :build/journal])
-                                         (d/datoms db :avet :block/journal-day)
-                                         first
-                                         :e
-                                         (d/entity db))]
-                     (assoc-in m [:page :block/uuid] (:block/uuid ent))
-                     ;; For now only check page uniqueness by title. Could handle more uniqueness checks later
-                     (if-let [ent (some->> (get-in m [:page :block/title]) (ldb/get-case-page db))]
-                       (assoc-in m [:page :block/uuid] (:block/uuid ent))
-                       m)))
-                 pages-and-blocks))
-    (seq classes)
-    (assoc :classes
-           (->> classes
-                (map (fn [[k v]]
-                       (if-let [ent (d/entity db k)]
-                         [k (assoc v :block/uuid (:block/uuid ent))]
-                         [k v])))
-                (into {})))
-    (seq properties)
-    (assoc :properties
-           (->> properties
-                (map (fn [[k v]]
-                       (if-let [ent (d/entity db k)]
-                         (do
-                           (when (not= (select-keys ent [:logseq.property/type :db/cardinality])
-                                       (select-keys v [:logseq.property/type :db/cardinality]))
-                             (swap! property-conflicts conj
-                                    {:property-id k
-                                     :actual (select-keys v [:logseq.property/type :db/cardinality])
-                                     :expected (select-keys ent [:logseq.property/type :db/cardinality])}))
-                           [k (assoc v :block/uuid (:block/uuid ent))])
-                         [k v])))
-                (into {})))))
+  (let [export-map
+        (cond-> {:build-existing-tx? true}
+          (seq pages-and-blocks)
+          (assoc :pages-and-blocks
+                 (mapv (fn [m]
+                         (update m :page (partial add-uuid-to-page-if-exists db)))
+                       pages-and-blocks))
+          (seq classes)
+          (assoc :classes
+                 (->> classes
+                      (map (fn [[k v]]
+                             (if-let [ent (d/entity db k)]
+                               [k (assoc v :block/uuid (:block/uuid ent))]
+                               [k v])))
+                      (into {})))
+          (seq properties)
+          (assoc :properties
+                 (->> properties
+                      (map (fn [[k v]]
+                             (if-let [ent (d/entity db k)]
+                               (do
+                                 (when (not= (select-keys ent [:logseq.property/type :db/cardinality])
+                                             (select-keys v [:logseq.property/type :db/cardinality]))
+                                   (swap! property-conflicts conj
+                                          {:property-id k
+                                           :actual (select-keys v [:logseq.property/type :db/cardinality])
+                                           :expected (select-keys ent [:logseq.property/type :db/cardinality])}))
+                                 [k (assoc v :block/uuid (:block/uuid ent))])
+                               [k v])))
+                      (into {}))))
+        export-map'
+        (walk/postwalk (fn [f]
+                         (if (and (vector? f) (= :build/page (first f)))
+                           [:build/page (add-uuid-to-page-if-exists db (second f))]
+                           f))
+                       export-map)]
+    export-map'))
 
 (defn- build-block-import-options
   "Builds options for sqlite-build to import into current-block"

deps/db/test/logseq/db/sqlite/export_test.cljs

@@ -9,6 +9,8 @@
             [logseq.db.test.helper :as db-test]
             [medley.core :as medley]))
 
+;; Test helpers
+;; ============
 (defn- export-block-and-import-to-another-block
   "Exports given block from one graph/conn, imports it to a 2nd block and then
    exports the 2nd block. The two blocks do not have to be in the same graph"
@@ -54,6 +56,38 @@
                  (assoc :block/title (name k)))]))
        (into {})))
 
+;; Tests
+;; =====
+
+(deftest merge-export-maps
+  (is (= {:pages-and-blocks
+          [{:page {:block/title "page1"}
+            :blocks [{:block/title "b1"}
+                     {:block/title "b2"}]}
+           {:page {:block/title "page2"}}]}
+         (#'sqlite-export/merge-export-maps
+          {:pages-and-blocks
+           [{:page {:block/title "page1"}
+             :blocks [{:block/title "b1"}]}]}
+          {:pages-and-blocks
+           [{:page {:block/title "page1"}
+             :blocks [{:block/title "b2"}]}
+            {:page {:block/title "page2"}}]}))
+      "In :pages-and-blocks, identical pages and their :blocks are merged")
+
+  (is (= {:pages-and-blocks
+          [{:page {:build/journal 20250220}
+            :blocks [{:block/title "b1"}]}
+           {:page {:build/journal 20250221}}]}
+       (#'sqlite-export/merge-export-maps
+          {:pages-and-blocks
+           [{:page {:build/journal 20250220}
+             :blocks [{:block/title "b1"}]}]}
+          {:pages-and-blocks
+           [{:page {:build/journal 20250220}}
+            {:page {:build/journal 20250221}}]}))
+      "In :pages-and-blocks, identical journals and their :blocks are merged"))
+
 (deftest import-block-in-same-graph
   (let [original-data
         {:properties {:user.property/default-many {:logseq.property/type :default :db/cardinality :db.cardinality/many}}
@@ -110,13 +144,17 @@
         (is (= (expand-properties (:properties original-data)) (:properties imported-block)))
         (is (= (expand-classes (:classes original-data)) (:classes imported-block)))))))
 
-(deftest import-block-with-block-ref
+(deftest import-block-with-different-ref-types
   (let [page-uuid (random-uuid)
+        block-uuid (random-uuid)
         original-data
-        {:pages-and-blocks
+        {:properties {:user.property/p1 {:logseq.property/type :default}}
+         :pages-and-blocks
          [{:page {:block/title "page1"}
-           :blocks [{:block/title (str "page ref to " (page-ref/->page-ref page-uuid))}]}
-          {:page {:block/title "another page" :block/uuid page-uuid :build/keep-uuid? true}}]}
+           :blocks [{:block/title (str "page ref to " (page-ref/->page-ref page-uuid))
+                     :build/properties {:user.property/p1 (str "block ref to " (page-ref/->page-ref block-uuid))}}]}
+          {:page {:block/title "another page" :block/uuid page-uuid :build/keep-uuid? true}
+           :blocks [{:block/title "b1" :block/uuid block-uuid :build/keep-uuid? true}]}]}
         conn (db-test/create-conn-with-blocks original-data)
         conn2 (db-test/create-conn-with-blocks
                {:pages-and-blocks [{:page {:block/title "page2"}
@@ -207,6 +245,7 @@
         internal-block-uuid (random-uuid)
         class-uuid (random-uuid)
         page-uuid (random-uuid)
+        pvalue-page-uuid (random-uuid)
         property-uuid (random-uuid)
         journal-uuid (random-uuid)
         block-object-uuid (random-uuid)
@@ -217,11 +256,15 @@
                       {:logseq.property/type :node
                        :block/uuid property-uuid
                        :build/keep-uuid? true
-                       :build/property-classes [:user.class/NodeClass]}}
+                       :build/property-classes [:user.class/NodeClass]}
+                      :user.property/p2
+                      {:logseq.property/type :default}}
          :pages-and-blocks
          [{:page {:block/title "page1"}
            :blocks [{:block/title (str "page ref to " (page-ref/->page-ref page-uuid))}
                     {:block/title (str "block ref to " (page-ref/->page-ref block-uuid))}
+                    {:block/title "ref in properties"
+                     :build/properties {:user.property/p2 (str "pvalue ref to " (page-ref/->page-ref pvalue-page-uuid))}}
                     {:block/title "hola" :block/uuid internal-block-uuid :build/keep-uuid? true}
                     {:block/title (str "internal block ref to " (page-ref/->page-ref internal-block-uuid))}
                     {:block/title (str "class ref to " (page-ref/->page-ref class-uuid))}
@@ -231,7 +274,8 @@
           {:page {:block/title "page with block ref"}
            :blocks [{:block/title "hi" :block/uuid block-uuid :build/keep-uuid? true
                      :build/properties {:user.property/p1 [:block/uuid block-object-uuid]}}]}
-          {:page {:block/title "another page" :block/uuid page-uuid :build/keep-uuid? true}}
+          {:page {:block/title "page ref page" :block/uuid page-uuid :build/keep-uuid? true}}
+          {:page {:block/title "pvalue ref page" :block/uuid pvalue-page-uuid :build/keep-uuid? true}}
           {:page {:build/journal 20250207 :block/uuid journal-uuid :build/keep-uuid? true}}
           {:page {:block/title "Blocks"}
            :blocks [{:block/title "myclass object"
@@ -368,7 +412,13 @@
                ;; adjust shallow block
                (medley/dissoc-in [1 :blocks 0 :build/tags]))
            (:pages-and-blocks imported-page))
-        "Page's blocks are imported")))
+        "Page's blocks are imported")
+
+    (import-second-time-assertions conn conn2 "page1" original-data)
+    (is (= 1 (count (d/datoms @conn2 :avet :block/title "page object")))
+        "Page property value is only created first time")
+    (is (= 1 (count (d/datoms @conn2 :avet :block/journal-day 20250203)))
+        "Journal property value is only created first time")))
 
 (deftest import-graph-ontology
   (let [original-data

deps/graph-parser/src/logseq/graph_parser/block.cljs

@@ -376,8 +376,10 @@
    & {:keys [page-uuid class? created-by] :as options}]
   (when-not (and db (common-util/uuid-string? original-page-name)
                  (not (ldb/page? (d/entity db [:block/uuid (uuid original-page-name)]))))
-    (let [original-page-name (-> (string/trim original-page-name)
-                                 sanitize-hashtag-name)
+    (let [db-based? (ldb/db-based-graph? db)
+          original-page-name (cond-> (string/trim original-page-name)
+                               db-based?
+                               sanitize-hashtag-name)
           [page _page-entity] (cond
                                 (and original-page-name (string? original-page-name))
                                 (page-name-string->map original-page-name db date-formatter
@@ -393,7 +395,7 @@
                                                  nil)]
                                   [page nil]))]
       (when page
-        (if (ldb/db-based-graph? db)
+        (if db-based?
           (let [tags (if class? [:logseq.class/Tag]
                          (or (:block/tags page)
                              [:logseq.class/Page]))]

deps/outliner/src/logseq/outliner/core.cljs

@@ -424,7 +424,9 @@
                  (do
                    (assert (or (:db/id block) (:block/uuid block)) "save-block db/id not exists")
                    (when-let [eid (or (:db/id block) (when-let [id (:block/uuid block)] [:block/uuid id]))]
-                     (merge (d/entity @conn eid) block))))]
+                     (let [ent (d/entity @conn eid)]
+                       (assert (some? ent) "save-block entity not exists")
+                       (merge ent block)))))]
     (otree/-save block' txs-state conn repo date-formatter opts)
     {:tx-data @txs-state}))
 

scripts/src/logseq/tasks/dev/db_and_file_graphs.clj

@@ -26,6 +26,7 @@
         ["frontend.handler.file-based" "frontend.handler.file-sync"
          "frontend.db.file-based"
          "frontend.util.file-based"
+         "frontend.common.file-based"
          "frontend.worker.handler.page.file-based"
          ;; Want to only specify this ns and not the ones under it but don't have a way yet
          "frontend.worker.file"
@@ -56,6 +57,7 @@
   "Paths _only_ for file graphs"
   ["src/main/frontend/handler/file_based" "src/main/frontend/handler/file_sync.cljs" "src/main/frontend/db/file_based"
    "src/main/frontend/util/file_based" "src/main/frontend/worker/handler/page/file_based" "src/main/frontend/worker/file.cljs"
+   "src/main/frontend/common/file_based"
    "src/main/frontend/fs"
    "src/main/frontend/components/file_sync.cljs"
    "src/main/frontend/components/file_based"

shadow-cljs.edn

@@ -53,6 +53,7 @@
                           frontend.config/ENABLE-PLUGINS #shadow/env ["ENABLE_PLUGINS" :as :bool :default true]
                           ;; Set to switch file sync server to dev, set this to false in `yarn watch`
                           frontend.config/ENABLE-FILE-SYNC-PRODUCTION #shadow/env ["ENABLE_FILE_SYNC_PRODUCTION" :as :bool :default true]
+                          frontend.config/ENABLE-RTC-SYNC-PRODUCTION #shadow/env ["ENABLE_RTC_SYNC_PRODUCTION" :as :bool :default true]
                           frontend.config/REVISION #shadow/env ["LOGSEQ_REVISION" :default "dev"]} ;; set by git-revision-hook
 
         ;; NOTE: electron, browser/mobile-app use different asset-paths.

src/main/frontend/common/file_based/db.cljs

@@ -0,0 +1,37 @@
+(ns frontend.common.file-based.db
+  "Database fns for file graphs that are used by worker and frontend"
+  (:require [clojure.string :as string]
+            [datascript.core :as d]
+            [logseq.common.util :as common-util]
+            [logseq.db.file-based.rules :as file-rules]))
+
+(defn get-namespace-pages
+  "Accepts both sanitized and unsanitized namespaces"
+  [db namespace']
+  (assert (string? namespace'))
+  (let [namespace'' (common-util/page-name-sanity-lc namespace')
+        pull-attrs [:db/id :block/name :block/original-name :block/namespace
+                    {:block/file [:db/id :file/path]}]]
+    (d/q
+     [:find [(list 'pull '?c pull-attrs) '...]
+      :in '$ '% '?namespace
+      :where
+      ['?p :block/name '?namespace]
+      (list 'namespace '?p '?c)]
+     db
+     (:namespace file-rules/rules)
+     namespace'')))
+
+(defn get-pages-by-name-partition
+  [db partition']
+  (when-not (string/blank? partition')
+    (let [partition'' (common-util/page-name-sanity-lc (string/trim partition'))
+          ids (->> (d/datoms db :aevt :block/name)
+                   (filter (fn [datom]
+                             (let [page (:v datom)]
+                               (string/includes? page partition''))))
+                   (map :e))]
+      (when (seq ids)
+        (d/pull-many db
+                     '[:db/id :block/name :block/title]
+                     ids)))))

src/main/frontend/common/missionary.cljs

@@ -5,7 +5,13 @@
             [clojure.core.async :as a]
             [lambdaisland.glogi :as log]
             [missionary.core :as m]
-            [promesa.protocols :as pt]))
+            [promesa.protocols :as pt])
+  (:import [missionary Cancelled]))
+
+(extend-type Cancelled
+  IPrintWithWriter
+  (-pr-writer [o w _opts]
+    (write-all w "#missionary.Cancelled \"" (.-message o) "\"")))
 
 (defn continue-flow
   "ensure f is a continuous flow"

src/main/frontend/components/all_pages.cljs

@@ -39,15 +39,8 @@
 
 (defn- get-all-pages
   []
-  (let [pages (->> (page-handler/get-all-pages (state/get-current-repo))
-                   (map (fn [p] (assoc p :id (:db/id p)))))]
-    (if (config/db-based-graph? (state/get-current-repo))
-      pages
-      ;; FIXME: Remove when bug with page named 'page with #tag' is fixed
-      (let [buggy-pages (remove :block/type pages)]
-        (when (seq buggy-pages)
-          (js/console.error "The following pages aren't displayed because they don't have a :block/type" buggy-pages))
-        (filter :block/type pages)))))
+  (->> (page-handler/get-all-pages (state/get-current-repo))
+       (map (fn [p] (assoc p :id (:db/id p))))))
 
 (rum/defc all-pages < rum/static
   []

src/main/frontend/components/block.cljs

@@ -1056,6 +1056,7 @@
         (contains? config/audio-formats asset-type)
         (contains? config/video-formats asset-type))))
 
+(declare block-positioned-properties)
 (rum/defc page-reference < rum/reactive
   "Component for page reference"
   [html-export? uuid-or-title* {:keys [nested-link? show-brackets? id] :as config} label]
@@ -1082,6 +1083,10 @@
                     (not html-export?)
                     (not contents-page?))
            [:span.text-gray-500.bracket page-ref/left-brackets])
+         (when (and (config/db-based-graph?) (ldb/class-instance? (db/entity :logseq.class/Task) block))
+           [:div.inline-block {:style {:margin-right 1}
+                               :on-pointer-down (fn [e]
+                                                  (util/stop e))} (block-positioned-properties config block :block-left)])
          (page-cp config' (if (uuid? uuid-or-title)
                             {:block/uuid uuid-or-title}
                             {:block/name uuid-or-title}))
@@ -1573,7 +1578,7 @@
                          {:builder (query-builder-component/builder build-option {})
                           :query query}))])
 
-(defn- macro-function-cp
+(rum/defc macro-function-cp < rum/reactive
   [config arguments]
   (or
    (some-> (:query-result config) rum/react (block-macros/function-macro arguments))
@@ -3425,7 +3430,7 @@
         own-number-list? (:own-order-number-list? config)
         order-list? (boolean own-number-list?)
         children (ldb/get-children block)
-        selected? (contains? (set (state/get-selection-block-ids)) (:block/uuid block))
+        selected? (when (uuid? (:block/uuid block)) (contains? (set (state/get-selection-block-ids)) (:block/uuid block)))
         db-based? (config/db-based-graph? repo)
         page-icon (when (:page-title? config)
                     (let [icon' (get block (pu/get-pid :logseq.property/icon))]

src/main/frontend/components/block.css

@@ -379,7 +379,7 @@
 }
 
 .page-reference {
-  @apply inline rounded transition-[background];
+  @apply inline-flex flex-row items-center rounded transition-[background];
 
   .bracket {
     @apply opacity-30;

src/main/frontend/components/editor.cljs

@@ -30,6 +30,7 @@
             [goog.dom :as gdom]
             [goog.string :as gstring]
             [logseq.common.util :as common-util]
+            [logseq.common.util.page-ref :as page-ref]
             [logseq.db :as ldb]
             [logseq.db.frontend.class :as db-class]
             [logseq.graph-parser.property :as gp-property]
@@ -338,7 +339,17 @@
            (when (>= (count edit-content) current-pos)
              (subs edit-content pos current-pos)))]
     (when input
-      (block-search-auto-complete edit-block input id q format selected-text))))
+      (let [db? (config/db-based-graph? (state/get-current-repo))
+            embed? (and db? (= @commands/*current-command "Block embed"))
+            page (when embed? (page-ref/get-page-name edit-content))
+            embed-block-id (when (and embed? page (common-util/uuid-string? page))
+                             (uuid page))]
+        (if embed-block-id
+          (let [f (block-on-chosen-handler true input id q format nil)
+                block (db/entity embed-block-id)]
+            (when block (f block))
+            nil)
+          (block-search-auto-complete edit-block input id q format selected-text))))))
 
 (rum/defc template-search-aux
   [id q]
@@ -372,26 +383,30 @@
 (rum/defc property-search
   [id]
   (let [input (gdom/getElement id)
-        [matched-properties set-matched-properties!] (rum/use-state nil)]
+        [matched-properties set-matched-properties!] (rum/use-state nil)
+        [q set-q!] (rum/use-state "")]
     (when input
-      (let [q (or (:searching-property (editor-handler/get-searching-property input))
-                  "")]
-        (hooks/use-effect!
-         (fn []
-           (p/let [matched-properties (editor-handler/<get-matched-properties q)]
-             (set-matched-properties! matched-properties)))
-         [q])
-        (let [q-property (string/replace (string/lower-case q) #"\s+" "-")
-              non-exist-handler (fn [_state]
-                                  ((editor-handler/property-on-chosen-handler id q-property) nil))]
-          (ui/auto-complete
-           matched-properties
-           {:on-chosen (editor-handler/property-on-chosen-handler id q-property)
-            :on-enter non-exist-handler
-            :empty-placeholder [:div.px-4.py-2.text-sm (str "Create a new property: " q-property)]
-            :header [:div.px-4.py-2.text-sm.font-medium "Matched properties: "]
-            :item-render (fn [property] property)
-            :class       "black"}))))))
+      (hooks/use-effect!
+       (fn []
+         (.addEventListener input "input" (fn [_e]
+                                            (set-q! (or (:searching-property (editor-handler/get-searching-property input)) "")))))
+       [])
+      (hooks/use-effect!
+       (fn []
+         (p/let [matched-properties (editor-handler/<get-matched-properties q)]
+           (set-matched-properties! matched-properties)))
+       [q])
+      (let [q-property (string/replace (string/lower-case q) #"\s+" "-")
+            non-exist-handler (fn [_state]
+                                ((editor-handler/property-on-chosen-handler id q-property) nil))]
+        (ui/auto-complete
+         matched-properties
+         {:on-chosen (editor-handler/property-on-chosen-handler id q-property)
+          :on-enter non-exist-handler
+          :empty-placeholder [:div.px-4.py-2.text-sm (str "Create a new property: " q-property)]
+          :header [:div.px-4.py-2.text-sm.font-medium "Matched properties: "]
+          :item-render (fn [property] property)
+          :class       "black"})))))
 
 (rum/defc property-value-search-aux
   [id property q]

src/main/frontend/components/file_based/query_table.cljs

@@ -189,7 +189,7 @@
         property-separated-by-commas? (partial text/separated-by-commas? (state/get-config))]
     [:div.overflow-x-auto {:on-pointer-down (fn [e] (.stopPropagation e))
                            :style {:width "100%"}
-                           :class (when-not page? "query-table")}
+                           :class "query-table"}
      [:table.table-auto
       [:thead
        [:tr.cursor

src/main/frontend/components/page.cljs

@@ -241,8 +241,9 @@
               blocks (if block? [block] (db/sort-by-order children block))]
           (let [add-button? (not (or config/publishing?
                                      (let [last-child-id (model/get-block-deep-last-open-child-id (db/get-db) (:db/id (last blocks)))
-                                           block' (if last-child-id (db/entity last-child-id) (last blocks))]
-                                       (string/blank? (:block/title block')))))]
+                                           block' (if last-child-id (db/entity last-child-id) (last blocks))
+                                           link (:block/link block')]
+                                       (string/blank? (:block/title (or link block'))))))]
             [:div
              {:class (when add-button? "show-add-button")}
              (page-blocks-inner page-e blocks config sidebar? whiteboard? block-id)

src/main/frontend/components/plugins.cljs

@@ -213,6 +213,11 @@
    [:p.text-sm
     (t :plugin/security-warning)]))
 
+(defn format-number [num & {:keys [precision] :or {precision 2}}]
+  (cond
+    (< num 1000) (str num)
+    (>= num 1000) (str (.toFixed (/ num 1000) precision) "k")))
+
 (rum/defc card-ctls-of-market < rum/static
   [item stat installed? installing-or-updating?]
   [:div.ctl
@@ -225,7 +230,7 @@
     (when-let [downloads (and stat (:total_downloads stat))]
       (when (and downloads (> downloads 0))
         [:li.flex.text-sm.items-center.pr-3
-         (svg/cloud-down 16) [:span.pl-1 downloads]]))]
+         (svg/cloud-down 16) [:span.pl-1 (format-number downloads)]]))]
 
    [:div.r.flex.items-center
 
@@ -769,23 +774,25 @@
         pkgs (get grouped-pkgs false)
         ;; calculate weight
         [key pkgs] (if default?
-                     (let [decay-factor 0.01
+                     (let [decay-factor 0.001
                            download-weight 0.8
                            star-weight 0.2]
                        (letfn [(normalize [vals val]
                                  (let [min-val (apply min vals)
                                        max-val (apply max vals)]
-                                   (if (= max-val min-val) 1
-                                       (/ (- val min-val) (- max-val val)))))
+                                   (if (= max-val min-val) 0
+                                       (/ (- val min-val) (- max-val min-val)))))
                                (time-diff-in-days [ts]
-                                 (let [time-diff (- (js/Date.now) ts)]
+                                 (when-let [time-diff (and (number? ts) (- (js/Date.now) ts))]
                                    (/ time-diff (* 1000 60 60 24))))]
                          [:weight
-                          (let [all-downloads (map :downloads pkgs)
-                                all-stars (map :stars pkgs)]
+                          (let [all-downloads (->> (map :downloads pkgs) (remove #(not (number? %))))
+                                all-stars (->> (map :stars pkgs) (remove #(not (number? %))))]
                             (->> pkgs
                                  (map (fn [{:keys [downloads stars latestAt] :as pkg}]
-                                        (let [days-since-latest (time-diff-in-days latestAt)
+                                        (let [downloads (if (number? downloads) downloads 1)
+                                              stars (if (number? stars) stars 1)
+                                              days-since-latest (time-diff-in-days latestAt)
                                               decay (js/Math.exp (* -1 decay-factor days-since-latest))
                                               normalized-downloads (normalize all-downloads downloads)
                                               normalize-stars (normalize all-stars stars)

src/main/frontend/components/query.cljs

@@ -177,10 +177,9 @@
                  (custom-query-inner config q opts))
                {:default-collapsed? collapsed?
                 :title-trigger? true})]
-             (when-not (:table? config)
-               [:div.bd
-                (when-not collapsed?'
-                  (custom-query-inner config q opts))]))])))))
+             [:div.bd
+              (when-not collapsed?'
+                (custom-query-inner config q opts))])])))))
 
 (rum/defc trigger-custom-query
   [config q]

src/main/frontend/components/table.css

@@ -119,10 +119,6 @@ html.is-resizing-buf {
 
   th, td {
     @apply p-1.5 border border-collapse;
-
-    &[data-key=":block"] {
-      @apply whitespace-normal;
-    }
   }
 }
 

src/main/frontend/config.cljs

@@ -22,7 +22,7 @@
 (goog-define REVISION "unknown")
 (defonce revision REVISION)
 
-(def ENABLE-FILE-SYNC-PRODUCTION false)
+(goog-define ENABLE-FILE-SYNC-PRODUCTION false)
 
 ;; this is a feature flag to enable the account tab
 ;; when it launches (when pro plan launches) it should be removed

src/main/frontend/db/model.cljs

@@ -6,6 +6,7 @@
             [clojure.string :as string]
             [clojure.walk :as walk]
             [datascript.core :as d]
+            [frontend.common.file-based.db :as common-file-db]
             [frontend.config :as config]
             [frontend.date :as date]
             [frontend.db.conn :as conn]
@@ -912,7 +913,7 @@ independent of format as format specific heading characters are stripped"
 (defn get-namespace-pages
   "Accepts both sanitized and unsanitized namespaces"
   [repo namespace]
-  (ldb/get-namespace-pages (conn/get-db repo) namespace {:db-graph? (config/db-based-graph? repo)}))
+  (common-file-db/get-namespace-pages (conn/get-db repo) namespace))
 
 (defn- tree [flat-col root]
   (let [sort-fn #(sort-by :block/name %)

src/main/frontend/handler/editor.cljs

@@ -800,12 +800,14 @@
             (when-not (and has-children? left-has-children?)
               (when block-parent-id
                 (let [block-parent (gdom/getElement block-parent-id)
-                      sibling-block (if (:embed? config)
-                                      (util/get-prev-block-non-collapsed
-                                       block-parent
-                                       {:container (util/rec-get-blocks-container block-parent)})
-                                      (util/get-prev-block-non-collapsed-non-embed block-parent))
-                      {:keys [prev-block new-content edit-block-f]} (move-to-prev-block repo sibling-block format value)
+                      sibling-or-parent-block
+                      (if (:embed? config)
+                        (util/get-prev-block-non-collapsed
+                         block-parent
+                         {:container (util/rec-get-blocks-container block-parent)})
+                        (util/get-prev-block-non-collapsed-non-embed block-parent))
+                      {:keys [prev-block new-content edit-block-f]}
+                      (move-to-prev-block repo sibling-or-parent-block format value)
                       concat-prev-block? (boolean (and prev-block new-content))
                       transact-opts {:outliner-op :delete-blocks}]
                   (cond
@@ -817,7 +819,9 @@
                     concat-prev-block?
                     (let [children (:block/_parent (db/entity (:db/id block)))
                           db-based? (config/db-based-graph? repo)
+                          prev-block-is-not-parent? (not= (:block/uuid (:block/parent block)) (:block/uuid prev-block))
                           delete-prev-block? (and db-based?
+                                                  prev-block-is-not-parent?
                                                   (empty? (:block/tags block))
                                                   (not (:logseq.property.node/display-type block))
                                                   (seq (:block/properties block))
@@ -3235,10 +3239,14 @@
     (when-let [block-id (:block/uuid current-block)]
       (let [db? (config/db-based-graph? (state/get-current-repo))]
         (if (= format "embed")
-          (copy-block-ref! block-id
-                           (if db?
-                             block-ref/->block-ref
-                             #(str "{{embed ((" % "))}}")))
+          (if db?
+            (p/do!
+             (save-current-block!)
+             (util/copy-to-clipboard! (page-ref/->page-ref block-id)
+                                      {:graph (state/get-current-repo)
+                                       :blocks [{:block/uuid (:block/uuid current-block)}]
+                                       :embed-block? true}))
+            (copy-block-ref! block-id #(str "{{embed ((" % "))}}")))
           (copy-block-ref! block-id
                            (if db?
                              page-ref/->page-ref

src/main/frontend/handler/paste.cljs

@@ -190,7 +190,7 @@
   [input text e html]
   (util/stop e)
   (->
-   (p/let [{:keys [graph blocks]} (get-copied-blocks)]
+   (p/let [{:keys [graph blocks embed-block?]} (get-copied-blocks)]
      (if (and (seq blocks) (= graph (state/get-current-repo)))
        ;; Handle internal paste
        (let [revert-cut-txs (get-revert-cut-txs blocks)
@@ -198,8 +198,18 @@
              blocks (if (config/db-based-graph? (state/get-current-repo))
                       (map (fn [b] (dissoc b :block/properties)) blocks)
                       blocks)]
-         (editor-handler/paste-blocks blocks {:revert-cut-txs revert-cut-txs
-                                              :keep-uuid? keep-uuid?}))
+         (if embed-block?
+           (when-let [block-id (:block/uuid (first blocks))]
+             (when-let [current-block (state/get-edit-block)]
+               (p/do!
+                (editor-handler/api-insert-new-block! ""
+                                                      {:block-uuid (:block/uuid current-block)
+                                                       :sibling? true
+                                                       :replace-empty-target? true
+                                                       :other-attrs {:block/link (:db/id (db/entity [:block/uuid block-id]))}})
+                (state/clear-edit!))))
+           (editor-handler/paste-blocks blocks {:revert-cut-txs revert-cut-txs
+                                                :keep-uuid? keep-uuid?})))
        (paste-copied-text input text html)))
    (p/catch (fn [error]
               (log/error :msg "Paste failed" :exception error)

src/main/frontend/state.cljs

@@ -606,7 +606,7 @@ should be done through this fn in order to get global config and config defaults
 
 (defn custom-shortcuts []
   (merge (storage/get :ls-shortcuts)
-    (:shortcuts (get-config))))
+         (:shortcuts (get-config))))
 
 (defn get-commands
   []
@@ -1168,6 +1168,13 @@ Similar to re-frame subscriptions"
   []
   (get-selected-block-ids (get-selection-blocks)))
 
+(defn sub-block-selected?
+  [block-id]
+  (assert (uuid? block-id))
+  (rum/react
+   (r/cached-derived-atom (:selection/blocks @state) [(get-current-repo) ::ui-selected block-id]
+                          (fn [blocks] (some #{block-id} (get-selected-block-ids blocks))))))
+
 (defn dom-clear-selection!
   []
   (doseq [node (dom/by-class "ls-block selected")]

src/main/frontend/util.cljc

@@ -807,7 +807,7 @@
 
 #?(:cljs
    (defn copy-to-clipboard!
-     [text & {:keys [graph html blocks owner-window]}]
+     [text & {:keys [graph html blocks embed-block? owner-window]}]
      (let [blocks (map (fn [block] (if (de/entity? block)
                                      (-> (into {} block)
                                          ;; FIXME: why :db/id is not included?
@@ -820,6 +820,7 @@
                    :blocks (when (and graph (seq blocks))
                              (pr-str
                               {:graph graph
+                               :embed-block? embed-block?
                                :blocks (mapv #(dissoc % :block.temp/fully-loaded? %) blocks)}))}))]
        (if owner-window
          (utils/writeClipboard data owner-window)

src/main/frontend/worker/handler/page/file_based/rename.cljs

@@ -4,6 +4,7 @@
             [clojure.walk :as walk]
             [datascript.core :as d]
             [frontend.common.file.util :as wfu]
+            [frontend.common.file-based.db :as common-file-db]
             [frontend.worker.handler.page :as worker-page]
             [logseq.common.config :as common-config]
             [logseq.common.util :as common-util]
@@ -253,7 +254,7 @@
 (defn- rename-namespace-pages!
   "Original names (unsanitized only)"
   [repo conn config old-name new-name]
-  (let [pages (ldb/get-namespace-pages @conn old-name {})
+  (let [pages (common-file-db/get-namespace-pages @conn old-name)
         page (d/pull @conn '[*] [:block/name (common-util/page-name-sanity-lc old-name)])
         pages (cons page pages)]
     (doseq [{:block/keys [name title]} pages]
@@ -273,8 +274,8 @@
   (let [nested-page-str (page-ref/->page-ref (common-util/page-name-sanity-lc old-ns-name))
         ns-prefix-format-str (str page-ref/left-brackets "%s/")
         ns-prefix       (common-util/format ns-prefix-format-str (common-util/page-name-sanity-lc old-ns-name))
-        nested-pages    (ldb/get-pages-by-name-partition @conn nested-page-str)
-        nested-pages-ns (ldb/get-pages-by-name-partition @conn ns-prefix)]
+        nested-pages    (common-file-db/get-pages-by-name-partition @conn nested-page-str)
+        nested-pages-ns (common-file-db/get-pages-by-name-partition @conn ns-prefix)]
     (when nested-pages
       ;; rename page "[[obsidian]] is a tool" to "[[logseq]] is a tool"
       (doseq [{:block/keys [name title]} nested-pages]