Merge branch 'feat/db' into refactor/narrow-gap-between-page-and-block

deps/db/src/logseq/db/frontend/db_ident.cljs

@@ -39,6 +39,8 @@
               (string/replace-first #"^\d+" "")
               (string/replace " " "-")
               (string/replace "#" "")
+              ;; '/' cannot be in name - https://clojure.org/reference/reader
+              (string/replace "/" "-")
               (string/trim))]
     (assert (seq n) "name is not empty")
     (keyword user-namespace n)))

deps/db/src/logseq/db/frontend/malli_schema.cljs

@@ -29,7 +29,7 @@
 (def internal-property-ident
   [:or logseq-property-ident db-attribute-ident])
 
-(defn- user-property?
+(defn user-property?
   "Determines if keyword/ident is a user property"
   [kw]
   (db-property/user-property-namespace? (namespace kw)))

deps/db/src/logseq/db/frontend/property.cljs

@@ -203,7 +203,7 @@
 
 (def logseq-property-namespaces
   #{"logseq.property" "logseq.property.tldraw" "logseq.property.pdf" "logseq.task"
-    "logseq.property.linked-references"})
+    "logseq.property.linked-references" "logseq.property.asset"})
 
 (defn logseq-property?
   "Determines if keyword is a logseq property"

deps/db/test/logseq/db/frontend/db_ident_test.cljs

@@ -0,0 +1,9 @@
+(ns logseq.db.frontend.db-ident-test
+  (:require [cljs.test :refer [deftest is]]
+            [logseq.db.frontend.db-ident :as db-ident]))
+
+(deftest create-db-ident-from-name
+  (is (= "Whiteboard-Object"
+         ;; Example from docs graph
+         (name (db-ident/create-db-ident-from-name "user.class" "Whiteboard/Object")))
+      "ident names must not have '/' because it is a special symbol for the reader"))

deps/graph-parser/script/db_import.cljs

@@ -42,18 +42,19 @@
 
 (defn- notify-user [m]
   (println (:msg m))
-  (println "Ex-data:" (pr-str (dissoc (:ex-data m) :error)))
-  (println "Stacktrace:")
-  (if-let [stack (some-> (get-in m [:ex-data :error]) ex-data :sci.impl/callstack deref)]
-    (println (string/join
-              "\n"
-              (map
-               #(str (:file %)
-                     (when (:line %) (str ":" (:line %)))
-                     (when (:sci.impl/f-meta %)
-                       (str " calls #'" (get-in % [:sci.impl/f-meta :ns]) "/" (get-in % [:sci.impl/f-meta :name]))))
-               (reverse stack))))
-    (println (some-> (get-in m [:ex-data :error]) .-stack)))
+  (when (:ex-data m)
+    (println "Ex-data:" (pr-str (dissoc (:ex-data m) :error)))
+    (println "Stacktrace:")
+    (if-let [stack (some-> (get-in m [:ex-data :error]) ex-data :sci.impl/callstack deref)]
+      (println (string/join
+                "\n"
+                (map
+                 #(str (:file %)
+                       (when (:line %) (str ":" (:line %)))
+                       (when (:sci.impl/f-meta %)
+                         (str " calls #'" (get-in % [:sci.impl/f-meta :ns]) "/" (get-in % [:sci.impl/f-meta :name]))))
+                 (reverse stack))))
+      (println (some-> (get-in m [:ex-data :error]) .-stack))))
   (when (= :error (:level m))
     (js/process.exit 1)))
 

deps/graph-parser/src/logseq/graph_parser/exporter.cljs

@@ -202,7 +202,7 @@
            (assoc :logseq.task/deadline [:block/uuid (:block/uuid deadline-page)])
            (update :block/refs (fnil into []) [:logseq.task/deadline [:block/uuid (:block/uuid deadline-page)]])
            (update :block/path-refs (fnil into []) [:logseq.task/deadline [:block/uuid (:block/uuid deadline-page)]])
-           (dissoc :block/deadline :block/scheduled))
+           (dissoc :block/deadline :block/scheduled :block/repeated?))
        :properties-tx (when-not existing-journal-page [deadline-page])})
     {:block block :properties-tx []}))
 
@@ -285,13 +285,16 @@
 (def built-in-property-names
   "Set of all built-in property names as keywords. Using in-memory property
   names because these are legacy names already in a user's file graph"
-  (->> built-in-property-name-to-idents keys set))
+  (-> built-in-property-name-to-idents keys set
+      ;; :filters is not in built-in-properties because it maps to 2 new properties
+      (conj :filters)))
 
 (defn- update-built-in-property-values
-  [props {:keys [ignored-properties all-idents]} {:block/keys [title name]}]
+  [props {:keys [ignored-properties all-idents]} {:block/keys [title name]} options]
   (->> props
        (keep (fn [[prop val]]
-               (if (= :icon prop)
+               ;; FIXME: Migrate :filters to :logseq.property.linked-references/* properties
+               (if (#{:icon :filters} prop)
                  (do (swap! ignored-properties
                             conj
                             {:property prop :value val :location (if name {:page name} {:block title})})
@@ -299,19 +302,23 @@
                  [(built-in-property-name-to-idents prop)
                   (case prop
                     :query-properties
-                    (try
-                      (mapv #(if (#{:page :block :created-at :updated-at} %) % (get-ident @all-idents %))
-                            (edn/read-string val))
-                      (catch :default e
-                        (js/console.error "Translating query properties failed with:" e)
-                        []))
+                    (let [property-classes (set (map keyword (:property-classes options)))]
+                      (try
+                        (mapv #(cond (#{:page :block :created-at :updated-at} %)
+                                    %
+                                    (property-classes %)
+                                    :block/tags
+                                    (= :tags %)
+                                     ;; This could also be :logseq.property/page-tags
+                                    :block/tags
+                                    :else
+                                    (get-ident @all-idents %))
+                              (edn/read-string val))
+                        (catch :default e
+                          (js/console.error "Translating query properties failed with:" e)
+                          [])))
                     :query-sort-by
-                    (if (#{:page :block :created-at :updated-at} val) val (get-ident @all-idents (keyword val)))
-                    :filters
-                    (try (edn/read-string val)
-                         (catch :default e
-                           (js/console.error "Translating filters failed with:" e)
-                           {}))
+                    (if (#{:page :block :created-at :updated-at} (keyword val)) (keyword val) (get-ident @all-idents (keyword val)))
                     val)])))
        (into {})))
 
@@ -411,16 +418,8 @@
   updated properties in :block-properties and any property values tx in :pvalues-tx"
   [props _db page-names-to-uuids
    {:block/keys [properties-text-values] :as block}
-   {:keys [_whiteboard? import-state] :as options}]
-  (let [;; FIXME: Whiteboard
-        ;; prop-name->uuid (if whiteboard?
-        ;;                   (fn prop-name->uuid [k]
-        ;;                     (or (get-pid db k)
-        ;;                         (throw (ex-info (str "No uuid found for page " (pr-str k))
-        ;;                                         {:page k}))))
-        ;;                   (fn prop-name->uuid [k]
-        ;;                     (get-page-uuid page-names-to-uuids k)))
-        {:keys [all-idents property-schemas]} import-state
+   {:keys [import-state] :as options}]
+  (let [{:keys [all-idents property-schemas]} import-state
         get-ident' #(get-ident @all-idents %)
         user-properties (apply dissoc props built-in-property-names)]
     (when (seq user-properties)
@@ -435,7 +434,8 @@
       (let [props' (-> (update-built-in-property-values
                         (select-keys props built-in-property-names)
                         (select-keys import-state [:ignored-properties :all-idents])
-                        (select-keys block [:block/name :block/title]))
+                        (select-keys block [:block/name :block/title])
+                        (select-keys options [:property-classes]))
                        (merge (update-user-property-values user-properties page-names-to-uuids properties-text-values import-state options)))
             pvalue-tx-m (->property-value-tx-m block props' #(get @property-schemas %) @all-idents)
             block-properties (-> (merge props' (db-property-build/build-properties-with-ref-values pvalue-tx-m))
@@ -481,9 +481,9 @@
 
 (defn- handle-page-and-block-properties
   "Returns a map of :block with updated block and :properties-tx with any properties tx.
-   Handles modifying :block/properties, updating classes from property-classes
+   Handles modifying block properties, updating classes from property-classes
   and removing any deprecated property related attributes. Before updating most
-  :block/properties, their property schemas are inferred as that can affect how
+  block properties, their property schemas are inferred as that can affect how
   a property is updated. Only infers property schemas on user properties as
   built-in ones must not change"
   [{:block/keys [properties] :as block} db page-names-to-uuids refs
@@ -628,20 +628,28 @@
                    (update-block-marker options)
                    (update-block-priority options)
                    add-missing-timestamps
+                   ;; old whiteboards may have this
+                   (dissoc :block/left)
                    ;; ((fn [x] (prn :block-out x) x))
                    ;; TODO: org-mode content needs to be handled
                    (assoc :block/format :markdown))]
     ;; Order matters as properties are referenced in block
     (concat properties-tx deadline-properties-tx [block'])))
 
+(defn- update-page-alias
+  [m page-names-to-uuids]
+  (update m :block/alias (fn [aliases]
+                           (map #(vector :block/uuid (get-page-uuid page-names-to-uuids (:block/name %)))
+                                aliases))))
+
 (defn- build-new-page
   [m db tag-classes page-names-to-uuids]
-  (-> m
-      ;; Fix pages missing :block/title. Shouldn't happen
-      ((fn [m']
-         (if-not (:block/title m')
-           (assoc m' :block/title (:block/name m'))
-           m')))
+  (-> (cond-> m
+        ;; Fix pages missing :block/title. Shouldn't happen
+        (not (:block/title m))
+        (assoc :block/title (:block/name m))
+        (seq (:block/alias m))
+        (update-page-alias page-names-to-uuids))
       add-missing-timestamps
       ;; TODO: org-mode content needs to be handled
       (assoc :block/format :markdown)
@@ -672,7 +680,7 @@
                            (let [;; These attributes are not allowed to be transacted because they must not change across files
                                  disallowed-attributes [:block/name :block/uuid :block/format :block/title :block/journal-day
                                                         :block/created-at :block/updated-at]
-                                 allowed-attributes (into [:block/tags :block/alias :class/parent :block/type :block/namespace]
+                                 allowed-attributes (into [:block/tags :block/alias :class/parent :block/type]
                                                           (keep #(when (db-malli-schema/user-property? (key %)) (key %))
                                                                 m))
                                  block-changes (select-keys m allowed-attributes)]
@@ -681,6 +689,8 @@
                                                        ignored-attrs)}))
                              (when (seq block-changes)
                                (cond-> (merge block-changes {:block/uuid page-uuid})
+                                 (seq (:block/alias m))
+                                 (update-page-alias page-names-to-uuids)
                                  (:block/tags m)
                                  (update-page-tags @conn tag-classes page-names-to-uuids))))
                            (build-new-page m @conn tag-classes page-names-to-uuids)))
@@ -692,7 +702,7 @@
 
 (defn- build-upstream-properties-tx-for-default
   "Builds upstream-properties-tx for properties that change to :default type"
-  [db prop property-ident block-properties-text-values blocks-tx]
+  [db prop property-ident block-properties-text-values]
   (let [get-pvalue-content (fn get-pvalue-content [block-uuid prop']
                              (or (get-in block-properties-text-values [block-uuid prop'])
                                  (throw (ex-info (str "No :block/text-properties-values found when changing property values: " (pr-str block-uuid))
@@ -709,56 +719,41 @@
                   (rules/extract-rules rules/db-query-dsl-rules)))
         existing-blocks-tx
         (mapcat (fn [m]
-                  (let [prop-value-id (or (:db/id (get m property-ident))
-                                          (throw (ex-info (str "No property value found when changing property values: " (pr-str property-ident))
-                                                          {:property-ident property-ident
-                                                           :block-uuid (:block-uuid m)})))
-                        prop-value-content (get-pvalue-content (:block/uuid m) prop)]
-                    ;; Switch to :block/title since :default is stored differently
-                    [[:db/retract prop-value-id :property.value/content]
-                     [:db/add prop-value-id :block/title prop-value-content]]))
-                existing-blocks)
-        ;; Look up blocks about to be transacted for current file a.k.a. pending
-        ;; Map of property value uuids to their original block uuids
-        ;; original block uuid needed to look up property's text value
-        pending-pvalue-uuids (->> blocks-tx
-                                  (keep #(when-let [prop-value (get % property-ident)]
-                                           [(or (second prop-value)
-                                                (throw (ex-info (str "No property value found when changing property values: " (pr-str property-ident))
-                                                                {:property-ident property-ident
-                                                                 :block-uuid (:block-uuid %)})))
-                                            (:block/uuid %)]))
-                                  (into {}))
-        pending-blocks-tx
-        (mapcat (fn [m]
-                  (when-let [original-block-uuid (get pending-pvalue-uuids (:block/uuid m))]
-                    (let [prop-value-content (get-pvalue-content original-block-uuid prop)
-                          prop-value-id [:block/uuid (:block/uuid m)]]
-                      [[:db/retract prop-value-id :property.value/content]
-                       [:db/add prop-value-id :block/title prop-value-content]])))
-                blocks-tx)]
-    (concat existing-blocks-tx pending-blocks-tx)))
+                  (let [prop-value (get m property-ident)
+                        prop-value-content (get-pvalue-content (:block/uuid m) prop)
+                        new-value (db-property-build/build-property-value-block
+                                   m {:db/ident property-ident} prop-value-content)]
+                    (into (mapv #(vector :db/retractEntity (:db/id %))
+                                (if (sequential? prop-value) prop-value [prop-value]))
+                          [new-value
+                           {:block/uuid (:block/uuid m)
+                            property-ident [:block/uuid (:block/uuid new-value)]}])))
+                existing-blocks)]
+    existing-blocks-tx))
 
 (defn- build-upstream-properties-tx
   "Builds tx for upstream properties that have changed and any instances of its
   use in db or in given blocks-tx. Upstream properties can be properties that
   already exist in the DB from another file or from earlier uses of a property
   in the same file"
-  [db page-names-to-uuids upstream-properties import-state blocks-tx log-fn]
+  [db upstream-properties import-state log-fn]
   (if (seq upstream-properties)
     (let [block-properties-text-values @(:block-properties-text-values import-state)
-          all-idents @(:all-idents import-state)]
-      (log-fn :props-upstream-to-change upstream-properties)
-      (mapcat
-       (fn [[prop {:keys [schema]}]]
-         ;; property schema change
-         (let [prop-uuid (get-page-uuid page-names-to-uuids (name prop))]
-           (into [{:block/uuid prop-uuid :block/schema schema}]
-                 ;; handle changes to specific types
-                 (when (= :default (:type schema))
-                   (let [prop-ident (get-ident all-idents prop)]
-                     (build-upstream-properties-tx-for-default db prop prop-ident block-properties-text-values blocks-tx))))))
-       upstream-properties))
+          all-idents @(:all-idents import-state)
+          _ (log-fn :props-upstream-to-change upstream-properties)
+          txs
+          (mapcat
+           (fn [[prop {:keys [schema]}]]
+             (let [prop-ident (get-ident all-idents prop)
+                   upstream-tx
+                   (when (= :default (:type schema))
+                     (build-upstream-properties-tx-for-default db prop prop-ident block-properties-text-values))
+                   property-pages-tx [{:db/ident prop-ident :block/schema schema}]]
+               ;; If we handle cardinality changes we would need to return these separately
+               ;; as property-pages would need to be transacted separately
+               (concat property-pages-tx upstream-tx)))
+           upstream-properties)]
+      txs)
     []))
 
 (defn new-import-state
@@ -870,12 +865,12 @@
         {:keys [pages-tx page-properties-tx page-names-to-uuids existing-pages]} (build-pages-tx conn pages blocks tx-options)
         whiteboard-pages (->> pages-tx
                               ;; support old and new whiteboards
-                              (filter #(#{"whiteboard" ["whiteboard"]} (:block/type %)))
+                              (filter #(or (contains? (set (:block/type %)) "whiteboard")
+                                           (= "whiteboard" (:block/type %))))
                               (map (fn [page-block]
                                      (-> page-block
                                          (assoc :block/format :markdown
-                                                 ;; fixme: missing properties
-                                                :block/properties {(get-pid @conn :ls-type) :whiteboard-page})))))
+                                                :logseq.property/ls-type :whiteboard-page)))))
         pre-blocks (->> blocks (keep #(when (:block/pre-block? %) (:block/uuid %))) set)
         blocks-tx (->> blocks
                        (remove :block/pre-block?)
@@ -886,15 +881,8 @@
         {:keys [property-pages-tx property-page-properties-tx] pages-tx' :pages-tx}
         (split-pages-and-properties-tx pages-tx old-properties existing-pages (:import-state options))
         ;; Necessary to transact new property entities first so that block+page properties can be transacted next
-        _ (d/transact! conn property-pages-tx)
-
-        upstream-properties-tx (build-upstream-properties-tx
-                                @conn
-                                page-names-to-uuids
-                                @(:upstream-properties tx-options)
-                                (select-keys (:import-state tx-options) [:block-properties-text-values :all-idents])
-                                blocks-tx
-                                log-fn)
+        main-props-tx-report (d/transact! conn property-pages-tx)
+
         ;; Build indices
         pages-index (map #(select-keys % [:block/uuid]) pages-tx')
         block-ids (map (fn [block] {:block/uuid (:block/uuid block)}) blocks-tx)
@@ -907,12 +895,17 @@
         blocks-index (set/union (set block-ids) (set block-refs-ids))
         ;; Order matters. pages-index and blocks-index needs to come before their corresponding tx for
         ;; uuids to be valid. Also upstream-properties-tx comes after blocks-tx to possibly override blocks
-        tx (concat whiteboard-pages pages-index page-properties-tx property-page-properties-tx pages-tx'
-                   blocks-index blocks-tx upstream-properties-tx)
+        tx (concat whiteboard-pages pages-index page-properties-tx property-page-properties-tx pages-tx' blocks-index blocks-tx)
         tx' (common-util/fast-remove-nils tx)
         ;; _ (cljs.pprint/pprint {:tx tx'})
-        result (d/transact! conn tx')]
-    result))
+        main-tx-report (d/transact! conn tx')
+
+        upstream-properties-tx
+        (build-upstream-properties-tx @conn @(:upstream-properties tx-options) (:import-state options) log-fn)
+        upstream-tx-report (when (seq upstream-properties-tx) (d/transact! conn upstream-properties-tx))]
+
+    ;; Return all tx-reports that occurred in this fn as UI needs to know what changed
+    [main-props-tx-report main-tx-report upstream-tx-report]))
 
 ;; Higher level export fns
 ;; =======================
@@ -998,30 +991,29 @@
 
 (defn- export-class-properties
   [conn repo-or-conn]
-  (let [user-classes (->> (d/q '[:find (pull ?b [:db/id :block/name])
+  (let [user-classes (->> (d/q '[:find (pull ?b [:db/id :db/ident])
                                  :where [?b :block/type "class"]] @conn)
                           (map first)
-                          (remove #(db-class/built-in-classes (keyword (:block/name %)))))
+                          (remove #(db-class/built-in-classes (:db/ident %))))
         class-to-prop-uuids
-        (->> (d/q '[:find ?t ?prop-name ?prop-uuid #_?class
+        (->> (d/q '[:find ?t ?prop #_?class
                     :in $ ?user-classes
                     :where
                     [?b :block/tags ?t]
-                    [?t :block/name ?class]
+                    [?t :db/ident ?class]
                     [(contains? ?user-classes ?class)]
-                    [?b :block/properties ?bp]
-                    [?prop-b :block/name ?prop-name]
-                    [?prop-b :block/uuid ?prop-uuid]
-                    [(get ?bp ?prop-uuid) ?_v]]
+                    [?b ?prop _]
+                    [?prop-e :db/ident ?prop]
+                    [?prop-e :block/type "property"]]
                   @conn
-                  (set (map :block/name user-classes)))
-             (remove #(ldb/built-in? (ldb/get-page @conn (second %))))
-             (reduce (fn [acc [class-id _prop-name prop-uuid]]
-                       (update acc class-id (fnil conj #{}) prop-uuid))
+                  (set (map :db/ident user-classes)))
+             (remove #(ldb/built-in? (d/entity @conn (second %))))
+             (reduce (fn [acc [class-id prop-ident]]
+                       (update acc class-id (fnil conj #{}) prop-ident))
                      {}))
         tx (mapv (fn [[class-id prop-ids]]
                    {:db/id class-id
-                    :block/schema {:properties (vec prop-ids)}})
+                    :class/schema.properties (vec prop-ids)})
                  class-to-prop-uuids)]
     (ldb/transact! repo-or-conn tx)))
 

deps/graph-parser/src/logseq/graph_parser/extract.cljc

@@ -322,9 +322,10 @@
                           page-name)
         page-block (merge {:block/name page-name
                            :block/title title
-                           :block/type #{"whiteboard" "page"}
                            :block/file {:file/path (common-util/path-normalize file)}}
-                          serialized-page)
+                          serialized-page
+                          ;; Ensure old whiteboards have correct type
+                          {:block/type #{"whiteboard" "page"}})
         page-block (gp-whiteboard/migrate-page-block page-block)
         blocks (->> blocks
                     (map gp-whiteboard/migrate-shape-block)

deps/graph-parser/test/logseq/graph_parser/cli_test.cljs

@@ -32,7 +32,7 @@
     (docs-graph-helper/docs-graph-assertions @conn graph-dir files)
 
     (testing "Additional counts"
-      (is (= 48766 (count (d/datoms @conn :eavt))) "Correct datoms count"))
+      (is (= 48767 (count (d/datoms @conn :eavt))) "Correct datoms count"))
 
     (testing "Asts"
       (is (seq asts) "Asts returned are non-zero")

deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs

@@ -63,18 +63,21 @@
 
 (defn- notify-user [m]
   (println (:msg m))
-  (println "Ex-data:" (pr-str (dissoc (:ex-data m) :error)))
-  (println "Stacktrace:")
-  (if-let [stack (some-> (get-in m [:ex-data :error]) ex-data :sci.impl/callstack deref)]
-    (println (string/join
-              "\n"
-              (map
-               #(str (:file %)
-                     (when (:line %) (str ":" (:line %)))
-                     (when (:sci.impl/f-meta %)
-                       (str " calls #'" (get-in % [:sci.impl/f-meta :ns]) "/" (get-in % [:sci.impl/f-meta :name]))))
-               (reverse stack))))
-    (println (some-> (get-in m [:ex-data :error]) .-stack))))
+  (when (:ex-data m)
+    (println "Ex-data:" (pr-str (dissoc (:ex-data m) :error)))
+    (println "Stacktrace:")
+    (if-let [stack (some-> (get-in m [:ex-data :error]) ex-data :sci.impl/callstack deref)]
+      (println (string/join
+                "\n"
+                (map
+                 #(str (:file %)
+                       (when (:line %) (str ":" (:line %)))
+                       (when (:sci.impl/f-meta %)
+                         (str " calls #'" (get-in % [:sci.impl/f-meta :ns]) "/" (get-in % [:sci.impl/f-meta :name]))))
+                 (reverse stack))))
+      (println (some-> (get-in m [:ex-data :error]) .-stack))))
+  (when (= :error (:level m))
+    (js/process.exit 1)))
 
 (def default-export-options
   {;; common options
@@ -105,9 +108,10 @@
 (defn- import-files-to-db
   "Import specific doc files for dev purposes"
   [files conn options]
-  (let [doc-options (gp-exporter/build-doc-options {:macros {}} (merge options default-export-options))
-        files' (mapv #(hash-map :path %) files)]
-    (gp-exporter/export-doc-files conn files' <read-file doc-options)))
+  (p/let [doc-options (gp-exporter/build-doc-options {:macros {}} (merge options default-export-options))
+          files' (mapv #(hash-map :path %) files)
+          _ (gp-exporter/export-doc-files conn files' <read-file doc-options)]
+    {:import-state (:import-state doc-options)}))
 
 (defn- readable-properties
   [db query-ent]
@@ -136,12 +140,12 @@
 
     (testing "whole graph"
 
-      (is (nil? (:errors (db-validate/validate-db! @conn)))
+      (is (empty? (map :entity (:errors (db-validate/validate-db! @conn))))
           "Created graph has no validation errors")
 
       ;; Counts
       ;; Includes journals as property values e.g. :logseq.task/deadline
-      (is (= 14 (count (d/q '[:find ?b :where [?b :block/type "journal"]] @conn))))
+      (is (= 16 (count (d/q '[:find ?b :where [?b :block/type "journal"]] @conn))))
 
       ;; Don't count pages like url.md that have properties but no content
       (is (= 5
@@ -149,8 +153,8 @@
                                 :where [?b :block/title] [_ :block/page ?b]] @conn)
                          (filter #(= ["page"] (:block/type %))))))
           "Correct number of pages with block content")
-      (is (= 2 (count @(:ignored-properties import-state)))
-          "Only ignored properties should be related to :icon")
+      (is (= 4 (count (d/datoms @conn :avet :block/type "whiteboard"))))
+      (is (= 1 (count @(:ignored-properties import-state))) ":filters should be the only ignored property")
       (is (= 1 (count @assets))))
 
     (testing "logseq files"
@@ -169,7 +173,7 @@
               set))))
 
     (testing "user properties"
-      (is (= 14
+      (is (= 17
              (->> @conn
                   (d/q '[:find [(pull ?b [:db/ident]) ...]
                          :where [?b :block/type "property"]])
@@ -216,17 +220,17 @@
               :user.property/prop-num 5
               :user.property/prop-string "yeehaw"}
              (readable-properties @conn (find-page-by-name @conn "some page")))
-          "Existing page has correct properties"))
+          "Existing page has correct properties")
+
+      (is (= {:user.property/rating 5.5}
+             (readable-properties @conn (find-block-by-content @conn ":rating float")))
+          "Block with float property imports as a float"))
 
     (testing "built-in properties"
       (is (= [(:db/id (find-block-by-content @conn "original block"))]
              (mapv :db/id (:block/refs (find-block-by-content @conn #"ref to"))))
           "block with a block-ref has correct :block/refs")
 
-      (is (= 2
-             (count (filter #(= :icon (:property %)) @(:ignored-properties import-state))))
-          "icon properties are visibly ignored in order to not fail import")
-
       (let [b (find-block-by-content @conn #"MEETING TITLE")]
         (is (= {}
                (and b (readable-properties @conn b)))
@@ -256,6 +260,9 @@
              (readable-properties @conn (find-block-by-content @conn "list one")))
           "numered block has correct property")
 
+      (is (= #{"gpt"}
+             (:block/alias (readable-properties @conn (find-page-by-name @conn "chat-gpt")))))
+
       (is (= {:logseq.property/query-sort-by :user.property/prop-num
               :logseq.property/query-properties [:block :page :user.property/prop-string :user.property/prop-num]
               :logseq.property/query-table true}
@@ -274,16 +281,48 @@
       (is (= :page
              (get-in (d/entity @conn :user.property/participants) [:block/schema :type]))
           ":page property to :date value remains :page")
-      (is (= :default
-             (get-in (d/entity @conn :user.property/duration) [:block/schema :type]))
-          ":number property to :default value changes to :default")
 
       (is (= :default
              (get-in (d/entity @conn :user.property/description) [:block/schema :type]))
           ":default property to :page (or any non :default value) remains :default")
       (is (= "[[Jakob]]"
              (:user.property/description (readable-properties @conn (find-block-by-content @conn #":default to :page"))))
-          ":page property value correctly saved as :default with full text"))
+          ":default to :page property saves :default property value default with full text")
+
+      (testing "with changes to upstream/existing property value"
+        (is (= :default
+               (get-in (d/entity @conn :user.property/duration) [:block/schema :type]))
+            ":number property to :default value changes to :default")
+        (is (= "20"
+               (:user.property/duration (readable-properties @conn (find-block-by-content @conn "existing :number to :default"))))
+            "existing :number property value correctly saved as :default")
+
+        (is (= {:block/schema {:type :default} :db/cardinality :db.cardinality/many}
+               (select-keys (d/entity @conn :user.property/people) [:block/schema :db/cardinality]))
+            ":page property to :default value changes to :default and keeps existing cardinality")
+        (is (= #{"[[Jakob]] [[Gabriel]]"}
+               (:user.property/people (readable-properties @conn (find-block-by-content @conn ":page people"))))
+            "existing :page property value correctly saved as :default with full text")
+        (is (= #{"[[Gabriel]] [[Jakob]]"}
+               (:user.property/people (readable-properties @conn (find-block-by-content @conn #"pending block for :page"))))
+            "pending :page property value correctly saved as :default with full text")))
+
+    (testing "replacing refs in :block/title"
+      (is (= 2
+             (->> (find-block-by-content @conn #"replace with same start string")
+                  :block/title
+                  (re-seq #"\[\[~\^\S+\]\]")
+                  distinct
+                  count))
+          "A block with ref names that start with same string has 2 distinct refs")
+
+      (is (= 1
+             (->> (find-block-by-content @conn #"replace case insensitive")
+                  :block/title
+                  (re-seq #"\[\[~\^\S+\]\]")
+                  distinct
+                  count))
+          "A block with different case of same ref names has 1 distinct ref"))
 
     (testing "tags without tag options"
       (let [block (find-block-by-content @conn #"Inception")
@@ -298,7 +337,10 @@
 
         (is (= {:logseq.property/page-tags #{"Movie"}}
                (readable-properties @conn tagged-page))
-            "tagged page has tags imported as page-tags property by default")))))
+            "tagged page has existing page imported as a tag to page-tags")
+        (is (= #{"LargeLanguageModel" "fun" "ai"}
+               (:logseq.property/page-tags (readable-properties @conn (find-page-by-name @conn "chat-gpt"))))
+            "tagged page has new page and other pages marked with '#' and '[[]]` imported as tags to page-tags")))))
 
 (deftest-async export-file-with-tag-classes-option
   (p/let [file-graph-dir "test/resources/exporter-test-graph"
@@ -306,6 +348,10 @@
           conn (d/create-conn db-schema/schema-for-db-based-graph)
           _ (d/transact! conn (sqlite-create-graph/build-db-initial-data "{}"))
           _ (import-files-to-db files conn {:tag-classes ["movie"]})]
+
+    (is (empty? (map :entity (:errors (db-validate/validate-db! @conn))))
+        "Created graph has no validation errors")
+
     (let [block (find-block-by-content @conn #"Inception")
           tag-page (find-page-by-name @conn "Movie")
           another-tag-page (find-page-by-name @conn "p0")]
@@ -329,7 +375,26 @@
           files (mapv #(node-path/join file-graph-dir %) ["journals/2024_02_23.md" "pages/url.md"])
           conn (d/create-conn db-schema/schema-for-db-based-graph)
           _ (d/transact! conn (sqlite-create-graph/build-db-initial-data "{}"))
-          _ (import-files-to-db files conn {:property-classes ["type"]})]
+          _ (import-files-to-db files conn {:property-classes ["type"]})
+          _ (@#'gp-exporter/export-class-properties conn conn)]
+
+    (is (empty? (map :entity (:errors (db-validate/validate-db! @conn))))
+        "Created graph has no validation errors")
+
+    (is (= #{:user.class/Property :user.class/Movie}
+           (->> @conn
+                (d/q '[:find [?ident ...]
+                       :where [?b :block/type "class"] [?b :db/ident ?ident] (not [?b :logseq.property/built-in?])])
+                set))
+        "All classes are correctly defined by :type")
+
+    (is (= #{:user.property/url :user.property/sameas :user.property/rangeincludes}
+           (->> (d/entity @conn :user.class/Property)
+                :class/schema.properties
+                (map :db/ident)
+                set))
+        "Properties are correctly inferred for a class")
+
     (let [block (find-block-by-content @conn #"The Creator")
           tag-page (find-page-by-name @conn "Movie")]
       (is (= (:block/title block) "The Creator")
@@ -349,3 +414,13 @@
       (is (= [:user.class/Property]
              (:block/tags (readable-properties @conn (find-page-by-name @conn "url"))))
           "tagged page has configured tag imported as a class"))))
+
+(deftest-async export-file-with-ignored-properties
+  (p/let [file-graph-dir "test/resources/exporter-test-graph"
+          files (mapv #(node-path/join file-graph-dir %) ["ignored/icon-page.md"])
+          conn (d/create-conn db-schema/schema-for-db-based-graph)
+          _ (d/transact! conn (sqlite-create-graph/build-db-initial-data "{}"))
+          {:keys [import-state]} (import-files-to-db files conn {})]
+    (is (= 2
+           (count (filter #(= :icon (:property %)) @(:ignored-properties import-state))))
+        "icon properties are visibly ignored in order to not fail import")))

deps/graph-parser/test/resources/exporter-test-graph/journals/2024_02_15.md

@@ -1,4 +1,4 @@
-- b1
+- existing :number to :default
   duration:: 20
 - Review 15 candidates #Meeting
   participants:: [[Gabriel]] [[Jakob]]

deps/graph-parser/test/resources/exporter-test-graph/journals/2024_02_16.md

@@ -3,7 +3,7 @@
   finishedAt:: [[Feb 7th, 2024]]
 - test :date -> :page
   finishedAt:: [[Gabriel]]
-- MEETING TITLE #Meeting
+- MEETING TITLE
   template:: meeting
   participants:: TODO
 - pending block for :number to :default
@@ -13,4 +13,6 @@
 - test :default to :page
   description:: [[Jakob]]
 - test :page -> :date
-  participants:: [[Feb 7th, 2024]]
+  participants:: [[Feb 7th, 2024]]
+- :page people
+  people:: [[Jakob]] [[Gabriel]]

deps/graph-parser/test/resources/exporter-test-graph/journals/2024_02_28.md

@@ -1,3 +1,7 @@
 - collapsed block
   collapsed:: true
-	- child
+	- child
+- pending block for :page to :default
+  people:: [[Gabriel]] [[Jakob]]
+- test :page :many to :default
+  people:: some text

deps/graph-parser/test/resources/exporter-test-graph/journals/2024_02_29.md

@@ -0,0 +1,10 @@
+- b1
+  rating:: 5
+- :rating float
+  rating:: 5.5
+- query-table:: false
+  FIXME
+  #+BEGIN_QUERY
+  {:title "2nd level tasks with `#p1`"
+  :query (and (task todo doing) [[p1]])}
+  #+END_QUERY

deps/graph-parser/test/resources/exporter-test-graph/journals/2024_03_01.md

@@ -0,0 +1,5 @@
+- test ignore blank values so that property type doesn't change
+  sameAs::
+- replace tags edge cases
+	- replace with same start string #foo #foo-bar
+	- replace case insensitive #foo #Foo

deps/graph-parser/test/resources/exporter-test-graph/journals/2024_04_01.md

@@ -1,9 +1,10 @@
 - only deadline
+  id:: 669168ed-8734-4943-8a86-5e3a553a526d
   DEADLINE: <2022-11-26 Sat>
 - only scheduled
-  SCHEDULED: <2022-11-25 Fri>
+  SCHEDULED: <2022-11-25 Fri .+1d>
 - [#A] high priority
 - DOING [#B] status test
   :LOGBOOK:
   CLOCK: [2024-04-01 Mon 10:39:40]
-  :END:
+  :END:

deps/graph-parser/test/resources/exporter-test-graph/pages/Movie.md

@@ -0,0 +1 @@
+parent:: [[CreativeWork]]

deps/graph-parser/test/resources/exporter-test-graph/pages/chat-gpt.md

@@ -0,0 +1,6 @@
+filters:: {"contents" true}
+type:: [[LargeLanguageModel]]
+tags:: ai, #fun, [[LargeLanguageModel]] 
+alias:: gpt
+
+- some text

deps/graph-parser/test/resources/exporter-test-graph/whiteboards/page 9322.edn

@@ -0,0 +1,67 @@
+{:blocks ({:block/title "fix/published-embeds"
+           :block/format :markdown
+           :block/left
+           {:block/uuid #uuid "64551487-ffab-406c-adcf-a4e8f00db19c"}
+           :block/parent
+           {:block/uuid #uuid "64551487-ffab-406c-adcf-a4e8f00db19c"}
+           :block/uuid #uuid "645514a0-85c2-43db-ac08-12836ae3d147"}
+          {:block/created-at 1683639741650
+           :block/properties
+           {:ls-type :whiteboard-shape
+            :logseq.tldraw.shape
+            {:index 0
+             :scale [1 1]
+             :type "youtube"
+             :size [853 480]
+             :id "e694caf0-eb52-11ed-a877-89e9886a62fe"
+             :url "https://www.youtube.com/watch?v=C5m5dIiJMD0"
+             :isLocked false
+             :point [475.04503224455124 300.94676141866057]
+             :parentId "64551487-ffab-406c-adcf-a4e8f00db19c"
+             :nonce 1683297680671}}
+           :block/updated-at 1683639741650}
+          {:block/created-at 1683639741650
+           :block/properties
+           {:ls-type :whiteboard-shape
+            :logseq.tldraw.shape
+            {:isSizeLocked true
+             :stroke ""
+             :borderRadius 0
+             :index 1
+             :scale [1 1]
+             :scaleLevel "md"
+             :fill ""
+             :type "text"
+             :size [78 34]
+             :fontFamily "var(--ls-font-family)"
+             :strokeType "line"
+             :strokeWidth 2
+             :opacity 1
+             :id "51851490-ee6f-11ed-9c5e-21b9b3de5563"
+             :padding 4
+             :fontWeight 400
+             :noFill true
+             :point [719.5019628453911 862.2621018969135]
+             :lineHeight 1.2
+             :fontSize 20
+             :parentId "64551487-ffab-406c-adcf-a4e8f00db19c"
+             :nonce 1683639739496
+             :italic false
+             :text "fsdfsdf"}}
+           :block/updated-at 1683639741650})
+ :pages ({:block/uuid #uuid "64551487-ffab-406c-adcf-a4e8f00db19c"
+          :block/properties
+          {:ls-type :whiteboard-page
+           :logseq.tldraw.page
+           {:id "64551487-ffab-406c-adcf-a4e8f00db19c"
+            :name "page 9322"
+            :bindings
+            {}
+            :nonce 1
+            :assets []
+            :shapes-index ("e694caf0-eb52-11ed-a877-89e9886a62fe" "51851490-ee6f-11ed-9c5e-21b9b3de5563")}}
+          :block/updated-at 1683639741650
+          :block/created-at 1683297415382
+          :block/type "whiteboard"
+          :block/name "page 9322"
+          :block/title "page 9322"})}

deps/graph-parser/test/resources/exporter-test-graph/whiteboards/ref page.edn

@@ -0,0 +1,91 @@
+{:blocks (
+{:block/title "block with page ref [[some page]]"
+:block/created-at 1720805247589
+:block/format :markdown
+:block/parent
+{:block/uuid #uuid "6691676f-2eed-4619-b56a-69fd7d572c59"}
+:block/properties
+{}
+:block/updated-at 1720809014394
+:block/uuid #uuid "6691677f-c208-4c83-aa40-6efc4286100c"}
+{:block/created-at 1720808993087
+:block/properties
+{:ls-type :whiteboard-shape
+:logseq.tldraw.shape
+{:blockType "B"
+:stroke ""
+:collapsed false
+:borderRadius 8
+:scale [1 1]
+:pageId "6691677f-c208-4c83-aa40-6efc4286100c"
+:scaleLevel "md"
+:fill ""
+:compact true
+:isAutoResizing true
+:type "logseq-portal"
+:size [400 124.9781494140625]
+:strokeType "line"
+:strokeWidth 2
+:opacity 1
+:id "01b33340-4074-11ef-956e-7d9aebf284ae"
+:noFill false
+:point [430.41563108563423 233.00156784057617]
+:parentId "6691676f-2eed-4619-b56a-69fd7d572c59"
+:collapsedHeight 0
+:nonce 1720805246071
+:pageName nil}}
+:block/updated-at 1720808993087}
+{:block/title "block with block ref ((669168ed-8734-4943-8a86-5e3a553a526d))"
+:block/created-at 1720808993012
+:block/format :markdown
+:block/parent
+{:block/uuid #uuid "6691676f-2eed-4619-b56a-69fd7d572c59"}
+:block/properties
+{}
+:block/updated-at 1720809157098
+:block/uuid #uuid "66917621-93ae-475b-aa4c-6ae9e797cf68"}
+{:block/properties
+{:ls-type :whiteboard-shape
+:logseq.tldraw.shape
+{:blockType "B"
+:stroke ""
+:collapsed false
+:borderRadius 8
+:scale [1 1]
+:pageId "66917621-93ae-475b-aa4c-6ae9e797cf68"
+:scaleLevel "md"
+:fill ""
+:compact true
+:isAutoResizing true
+:type "logseq-portal"
+:size [400 79.9781265258789]
+:strokeType "line"
+:strokeWidth 2
+:opacity 1
+:id "b999ee10-407c-11ef-956e-7d9aebf284ae"
+:noFill false
+:point [478.6382088826831 337.7821947259741]
+:parentId "6691676f-2eed-4619-b56a-69fd7d572c59"
+:collapsedHeight 0
+:nonce 1720808990579
+:pageName nil}}
+:block/updated-at 1720809157310
+:block/created-at 1720809157310})
+:pages (
+{:block/tx-id 536871072
+:block/uuid #uuid "6691676f-2eed-4619-b56a-69fd7d572c59"
+:block/properties
+{:ls-type :whiteboard-page
+:logseq.tldraw.page
+{:id "6691676f-2eed-4619-b56a-69fd7d572c59"
+:name "ref page"
+:bindings
+{}
+:nonce 1
+:assets []}}
+:block/updated-at 1720809157310
+:block/created-at 1720805231835
+:block/format :markdown
+:block/type ["page" "whiteboard"]
+:block/name "ref page"
+:block/title "ref page"})}

deps/shui/src/logseq/shui/dialog/core.cljs

@@ -71,6 +71,10 @@
   (when-let [[index] (get-modal id)]
     (swap! *modals #(->> % (medley/remove-nth index) (vec)))))
 
+(defn has-modal?
+  []
+  (some-> @*modals (last) :open?))
+
 ;; apis
 (declare close!)
 

deps/shui/src/logseq/shui/popup/core.cljs

@@ -42,6 +42,7 @@
       (filter #(= id (:id (second %)))) (first))))
 
 (defn get-popups [] @*popups)
+(defn get-last-popup [] (last @*popups))
 
 (defn upsert-popup!
   [config]

resources/css/shui.css

@@ -220,11 +220,16 @@ html[data-theme=dark] {
 .ui__dialog-overlay, .ui__alert-dialog-overlay,
 .ui__dialog-content, .ui__alert-dialog-content,
 .ui__dropdown-menu-content, .ui__select-content {
+
   &.z-50 {
     @apply z-[999];
   }
 }
 
+div[data-radix-popper-content-wrapper] {
+  @apply !z-[999];
+}
+
 .ui__dialog-overlay {
   &[data-align=start],
   &[data-align=top] {

src/main/frontend/commands.cljs

@@ -150,10 +150,12 @@
                         (let [command (if db-based?
                                         [:div.flex.flex-row.items-center.gap-2 m [:div.text-xs.opacity-50 "Status"]]
                                         m)
-                              icon (case m
-                                     "Canceled" "Cancelled"
-                                     "Doing" "InProgress50"
-                                     m)]
+                              icon (if db-based?
+                                     (case m
+                                       "Canceled" "Cancelled"
+                                       "Doing" "InProgress50"
+                                       m)
+                                     "square-asterisk")]
                           [command (->marker m) (str "Set status to " m) icon]))))]
     (when (seq result)
       (update result 0 (fn [v] (conj v "TASK"))))))
@@ -170,6 +172,7 @@
 (defn get-priorities
   []
   (let [db-based? (config/db-based-graph? (state/get-current-repo))
+        with-no-priority #(if db-based? (cons ["No priority" (->priority nil) "" :icon/priorityLvlNone] %) %)
         result (->>
                 (if db-based?
                   (db-based-priorities)
@@ -178,8 +181,11 @@
                         (let [command (if db-based?
                                         [:div.flex.flex-row.items-center.gap-2 item [:div.text-xs.opacity-50 "Priority"]]
                                         item)]
-                          [command (->priority item) (str "Set priority to " item) (str "priorityLvl" item)])))
-                 (cons ["No priority" (->priority nil) "" :icon/priorityLvlNone])
+                          [command (->priority item) (str "Set priority to " item)
+                           (if db-based?
+                             (str "priorityLvl" item)
+                             (str "circle-letter-" (util/safe-lower-case item)))])))
+                 (with-no-priority)
                  (vec))]
     (when (seq result)
       (update result 0 (fn [v] (conj v "PRIORITY"))))))
@@ -308,7 +314,7 @@
                     [:editor/set-deadline]] "" :icon/calendar-stats]
        (when-not db?
          ["Scheduled" [[:editor/clear-current-slash]
-                       [:editor/set-scheduled]]])]
+                       [:editor/set-scheduled]] "" :icon/calendar-month])]
 
       ;; priority
       (get-priorities)
@@ -344,7 +350,7 @@
         :icon/query
         "ADVANCED"]
        (when-not db?
-         ["Zotero" (zotero-steps) "Import Zotero journal article"])
+         ["Zotero" (zotero-steps) "Import Zotero journal article" :icon/circle-letter-z])
        ["Query function" [[:editor/input "{{function }}" {:backward-pos 2}]] "Create a query function" :icon/queryCode]
        ["Calculator" [[:editor/input "```calc\n\n```" {:type "block"
                                                        :backward-pos 4}]

src/main/frontend/common/missionary_util.clj

@@ -0,0 +1,8 @@
+(ns frontend.common.missionary-util
+  "Macros for missionary"
+  (:require [missionary.core :as m]))
+
+(defmacro <?
+  "Like m/?, but async channel as arg"
+  [c]
+  `(m/? (<! ~c)))

src/main/frontend/common/missionary_util.cljs

@@ -1,5 +1,6 @@
 (ns frontend.common.missionary-util
   "Utils based on missionary."
+  (:require-macros [frontend.common.missionary-util])
   (:require [clojure.core.async :as a]
             [missionary.core :as m])
   ;; (:import [missionary Cancelled])
@@ -71,7 +72,6 @@
   completing with value when take is accepted, or nil if port was closed."
   [c] (doto (m/dfv) (->> (a/take! c))))
 
-
 (defn await-promise
   "Returns a task completing with the result of given promise"
   [p]

src/main/frontend/components/block.cljs

@@ -62,6 +62,8 @@
             [frontend.template :as template]
             [frontend.ui :as ui]
             [logseq.shui.ui :as shui]
+            [logseq.shui.dialog.core :as shui-dialog]
+            [logseq.shui.popup.core :as shui-popups]
             [frontend.util :as util]
             [frontend.extensions.pdf.utils :as pdf-utils]
             [frontend.util.drawer :as drawer]
@@ -602,7 +604,9 @@
                            (open-page-ref config page-entity e page-name contents-page?)))}
      (when-not hide-icon?
        (when-let [icon (get page-entity (pu/get-pid :logseq.property/icon))]
-         [:span.mr-1.inline-flex.items-center (icon/icon icon)]))
+         [:span.mr-1.inline-flex.items-center
+          {:style {:color (or (:color icon) "inherit")}}
+          (icon/icon icon)]))
      [:span
       (if (and (coll? children) (seq children))
         (for [child children]
@@ -657,58 +661,105 @@
                                                          (:db/id page-entity)))}
           (ui/icon "x" {:size 15})]))]))
 
+(rum/defc popup-preview-impl
+  [children {:keys [*timer *timer1 visible? set-visible! render *el-popup]}]
+  (let [*el-trigger (rum/use-ref nil)]
+    (rum/use-effect!
+      (fn []
+        (when (true? visible?)
+          (shui/popup-show!
+            (rum/deref *el-trigger) render
+            {:root-props {:onOpenChange (fn [v] (set-visible! v))
+                          :modal false}
+             :content-props {:class "ls-preview-popup"
+                             :onInteractOutside (fn [^js e] (.preventDefault e))
+                             :onEscapeKeyDown (fn [^js e]
+                                                (when (state/editing?)
+                                                  (.preventDefault e)
+                                                  (some-> (rum/deref *el-popup) (.focus))))}
+             :as-dropdown? false}))
+
+        (when (false? visible?)
+          (shui/popup-hide!)
+          (when (state/get-edit-block)
+            (state/clear-edit!)))
+        (rum/set-ref! *timer nil)
+        (rum/set-ref! *timer1 nil)
+        ;; teardown
+        (fn []
+          (when visible?
+            (shui/popup-hide!))))
+      [visible?])
+
+    [:span
+     {:ref *el-trigger
+      :on-mouse-enter (fn []
+                        (let [timer (rum/deref *timer)
+                              timer1 (rum/deref *timer1)]
+                          (when-not timer
+                            (rum/set-ref! *timer
+                              (js/setTimeout #(set-visible! true) 1000)))
+                          (when timer1
+                            (js/clearTimeout timer1)
+                            (rum/set-ref! *timer1 nil))))
+      :on-mouse-leave (fn []
+                        (let [timer (rum/deref *timer)
+                              timer1 (rum/deref *timer1)]
+                          (when timer
+                            (js/clearTimeout timer)
+                            (rum/set-ref! *timer nil))
+                          (when-not timer1
+                            (rum/set-ref! *timer1
+                              (js/setTimeout #(set-visible! false) 300)))))}
+     children]))
+
 (rum/defc page-preview-trigger
-  [{:keys [children sidebar? tippy-position tippy-distance fixed-position? open? manual?] :as config} page-name]
-  (let [*tippy-ref (rum/create-ref)
-        page-name (when page-name (util/page-name-sanity-lc page-name))
-        _  #_:clj-kondo/ignore (rum/defc html-template []
-                                 (let [*el-popup (rum/use-ref nil)]
-
-                                   (rum/use-effect!
-                                    (fn []
-                                      (let [el-popup (rum/deref *el-popup)
-                                            cb (fn [^js e]
-                                                 (when-not (state/editing?)
-                                                   ;; Esc
-                                                   (and (= e.which 27)
-                                                        (when-let [tp (rum/deref *tippy-ref)]
-                                                          (.hideTooltip tp)))))]
-
-                                        (js/setTimeout #(.focus el-popup))
-                                        (.addEventListener el-popup "keyup" cb)
-                                        #(.removeEventListener el-popup "keyup" cb)))
-                                    [])
-
-                                   (let [redirect-page-name (or (and page-name (model/get-redirect-page-name page-name (:block/alias? config)))
-                                                                page-name)]
-                                     (when redirect-page-name
-                                       [:div.tippy-wrapper.overflow-y-auto.p-4.outline-none.rounded-md
-                                        {:ref   *el-popup
-                                         :tab-index -1
-                                         :style {:width          600
-                                                 :text-align     "left"
-                                                 :font-weight    500
-                                                 :max-height     600
-                                                 :padding-bottom 64}}
-                                        (let [page-cp (state/get-page-blocks-cp)]
-                                          (page-cp {:repo (state/get-current-repo)
-                                                    :page-name redirect-page-name
-                                                    :sidebar? sidebar?
-                                                    :preview? true}))]))))]
-
-    (if (or (not manual?) open?)
-      (ui/tippy {:ref             *tippy-ref
-                 :in-editor?      true
-                 :html            html-template
-                 :interactive     true
-                 :delay           [1000, 100]
-                 :fixed-position? fixed-position?
-                 :position        (or tippy-position "top")
-                 :distance        (or tippy-distance 10)
-                 :popperOptions   {:modifiers {:preventOverflow
-                                               {:enabled           true
-                                                :boundariesElement "viewport"}}}}
-                children)
+  [{:keys [children sidebar? open? manual?] :as config} page-name]
+  (let [page-name (when page-name (util/page-name-sanity-lc page-name))
+        *timer (rum/use-ref nil)                            ;; show
+        *timer1 (rum/use-ref nil)                           ;; hide
+        *el-popup (rum/use-ref nil)
+        [visible? set-visible!] (rum/use-state nil)
+        _  #_:clj-kondo/ignore (rum/defc preview-render []
+                                 (rum/use-effect!
+                                   (fn []
+                                     (let [el-popup (rum/deref *el-popup)
+                                           focus! #(js/setTimeout (fn [] (.focus el-popup)))]
+                                       (focus!)
+                                       #(set-visible! false)))
+                                   [])
+
+                                 (let [redirect-page-name (or (and page-name (model/get-redirect-page-name page-name (:block/alias? config)))
+                                                            page-name)]
+                                   (when redirect-page-name
+                                     [:div.tippy-wrapper.as-page
+                                      {:ref *el-popup
+                                       :tab-index -1
+                                       :style {:width 600
+                                               :text-align "left"
+                                               :font-weight 500
+                                               :padding-bottom 64}
+                                       :on-mouse-enter (fn []
+                                                         (when-let [timer1 (rum/deref *timer1)]
+                                                           (js/clearTimeout timer1)))
+                                       :on-mouse-leave (fn []
+                                                         ;; check the top popup whether is the preview popup
+                                                         (when (= "ls-preview-popup"
+                                                                 (some-> (shui-popups/get-last-popup) :content-props :class))
+                                                           (rum/set-ref! *timer1
+                                                             (js/setTimeout #(set-visible! false) 500))))}
+                                      (let [page-cp (state/get-page-blocks-cp)]
+                                        (page-cp {:repo (state/get-current-repo)
+                                                  :page-name redirect-page-name
+                                                  :sidebar? sidebar?
+                                                  :preview? true}))])))]
+
+    (if (and (not (:preview? config))
+          (or (not manual?) open?))
+      (popup-preview-impl children
+        {:visible? visible? :set-visible! set-visible!
+         :*timer *timer :*timer1 *timer1
+         :render preview-render :*el-popup *el-popup})
       children)))
 
 (rum/defcs page-cp < db-mixins/query rum/reactive
@@ -728,7 +779,7 @@
       (let [page-name (some-> (:block/title page-entity) util/page-name-sanity-lc)
             whiteboard-page? (model/whiteboard-page? page-entity)
             inner (page-inner (assoc config :whiteboard-page? whiteboard-page?) page-entity children label)
-            modal? (:modal/show? @state/state)]
+            modal? (shui-dialog/has-modal?)]
         (if (and (not (util/mobile?))
                  (not= page-name (:id config))
                  (not (false? preview?))
@@ -905,6 +956,32 @@
 (declare block-content)
 (declare breadcrumb)
 
+(rum/defc block-reference-preview
+  [children {:keys [repo config id]}]
+  (let [*timer (rum/use-ref nil)                            ;; show
+        *timer1 (rum/use-ref nil)                           ;; hide
+        [visible? set-visible!] (rum/use-state nil)
+        _ #_:clj-kondo/ignore (rum/defc render []
+                                [:div.tippy-wrapper.as-block
+                                 {:style {:width 600
+                                          :font-weight 500
+                                          :text-align "left"}
+                                  :on-mouse-enter (fn []
+                                                    (when-let [timer1 (rum/deref *timer1)]
+                                                      (js/clearTimeout timer1)))
+
+                                  :on-mouse-leave (fn []
+                                                    (rum/set-ref! *timer1
+                                                      (js/setTimeout #(set-visible! false) 500)))}
+                                 [(breadcrumb config repo id {:indent? true})
+                                  (blocks-container
+                                    (assoc config :id (str id) :preview? true)
+                                    (db/get-block-and-children repo id))]])]
+    (popup-preview-impl children
+      {:visible? visible? :set-visible! set-visible!
+       :*timer *timer :*timer1 *timer1
+       :render render})))
+
 (rum/defc block-reference < rum/reactive db-mixins/query
   {:init (fn [state]
            (let [block-id (second (:rum/args state))]
@@ -973,18 +1050,8 @@
                       (not (:preview? config))
                       (not (:modal/show? @state/state))
                       (nil? block-type))
-               (ui/tippy {:html        (fn []
-                                         [:div.tippy-wrapper.overflow-y-auto.p-4
-                                          {:style {:width      735
-                                                   :text-align "left"
-                                                   :max-height 600}}
-                                          [(breadcrumb config repo block-id {:indent? true})
-                                           (blocks-container
-                                            (assoc config :id (str id) :preview? true)
-                                            (db/get-block-and-children repo block-id))]])
-                          :interactive true
-                          :in-editor?  true
-                          :delay       [1000, 100]} inner)
+               (block-reference-preview inner
+                 {:repo repo :config config :id block-id})
                inner)])
           [:span.warning.mr-1 {:title "Block ref invalid"}
            (block-ref/->block-ref id)])))
@@ -3612,7 +3679,7 @@
                 page (db/entity (:db/id page))
                 blocks (tree/non-consecutive-blocks->vec-tree blocks)
                 parent-blocks (group-by :block/parent blocks)]
-            [:div.custom-query-page-result.color-level {:key (str "page-" (:db/id page))}
+            [:div.custom-query-page-result {:key (str "page-" (:db/id page))}
              (ui/foldable
               [:div
                (page-cp config page)

src/main/frontend/components/block.css

@@ -245,19 +245,16 @@
   }
 }
 
-.block-control, .block-control:hover {
-  text-decoration: none;
-  cursor: default;
-  font-size: 14px;
-  min-width: 22px;
-  min-height: 22px;
-  padding: 2px;
-  color: var(--ls-secondary-text-color);
-  user-select: none;
-  opacity: .4;
+.block-control {
+  @apply decoration-0 text-sm cursor-default
+  min-w-[22px] min-h-[22px] p-0.5 select-none opacity-40;
 
   .control-hide {
-    display: none;
+    @apply hidden;
+  }
+
+  &:active {
+    @apply opacity-30;
   }
 }
 
@@ -825,6 +822,8 @@ html.is-mac {
 }
 
 .references-blocks {
+  @apply mb-3;
+
   &-wrap {
     .foldable-title {
       @apply ml-3;

src/main/frontend/components/editor.cljs

@@ -54,7 +54,8 @@
                command-doc (get item 2)
                plugin-id (get-in item [1 1 1 :pid])
                doc (when (state/show-command-doc?) command-doc)
-               icon-name (some-> item (get 3) (name))
+               options (some-> item (get 3))
+               icon-name (some-> (if (map? options) (:icon options) options) (name))
                command-name (if icon-name
                               [:span.flex.items-center.gap-1
                                (shui/tabler-icon icon-name)
@@ -837,9 +838,9 @@
       (= :input action)
       nil
 
+      ;; exit editing mode
       :else
-      (let [select? (and (= type :esc)
-                         (not (string/includes? value "```")))]
+      (let [select? (= type :esc)]
         (when-let [container (gdom/getElement "app-container")]
           (dom/remove-class! container "blocks-selection-mode"))
         (p/do!

src/main/frontend/components/editor.css

@@ -116,7 +116,9 @@ pre {
 
 .cp__commands-slash {
   .ui__icon {
-    @apply opacity-80;
+    opacity: .7;
+    margin-right: 1px;
+    color: inherit !important;
 
     &.ls-icon-queryCode {
       @apply relative left-[1px];

src/main/frontend/components/icon.cljs

@@ -34,7 +34,8 @@
         page-icon (get page-entity (pu/get-pid :logseq.property/icon))]
     (or
      (when-not (string/blank? page-icon)
-       (icon page-icon opts))
+       [:span {:style {:color (or (:color page-icon ) "inherit")}}
+        (icon page-icon opts)])
      default-icon)))
 
 (defn- search-emojis
@@ -252,9 +253,9 @@
                             (if c "" (shui/tabler-icon "minus" {:class "scale-75 opacity-70"}))))]))]
     (rum/use-effect!
       (fn []
-        (when-let [^js section (some-> (rum/deref *el) (.closest ".cp__emoji-icon-picker") (.querySelector ".pane-section"))]
+        (when-let [^js picker (some-> (rum/deref *el) (.closest ".cp__emoji-icon-picker"))]
           (let [color (if (string/blank? color) "inherit" color)]
-            (set! (. (.-style section) -color) color)
+            (.setProperty (.-style picker) "--ls-color-icon-preset" color)
             (storage/set :ls-icon-color-preset color)))
         (reset! *color color))
       [color])
@@ -272,8 +273,9 @@
   (rum/local nil ::result)
   (rum/local false ::select-mode?)
   (rum/local :all ::tab)
-  (rum/local (storage/get :ls-icon-color-preset) ::color)
   (rum/local nil ::hover)
+  {:init (fn [s]
+           (assoc s ::color (atom (storage/get :ls-icon-color-preset))))}
   [state {:keys [on-chosen] :as opts}]
   (let [*q (::q state)
         *result (::result state)
@@ -370,7 +372,7 @@
                  :class (util/classnames [{:active active?} "tab-item"])
                  :on-click #(reset! *tab id)}
                 label)))]
-         (when (= :icon @*tab)
+         (when (not= :emoji @*tab)
            (color-picker *color))]
 
         ;; preview

src/main/frontend/components/icon.css

@@ -1,4 +1,6 @@
 .cp__emoji-icon-picker {
+  --ls-color-icon-preset: "inherit";
+
   @apply w-[380px] max-h-[396px] relative flex flex-col overflow-hidden;
   @apply pt-14 pb-[40px];
 
@@ -54,6 +56,8 @@
   .pane-section {
     @apply pl-2 overflow-y-scroll h-full;
 
+    color: var(--ls-color-icon-preset);
+
     > .its, .icons-row {
       @apply flex gap-1 py-1 flex-wrap;
 

src/main/frontend/components/imports.cljs

@@ -326,9 +326,10 @@
                    ;; doc file options
                    ;; Write to frontend first as writing to worker first is poor ux with slow streaming changes
                    :export-file (fn export-file [conn m opts]
-                                  (let [tx-report
+                                  (let [tx-reports
                                         (gp-exporter/add-file-to-db-graph conn (:file/path m) (:file/content m) opts)]
-                                    (db-browser/transact! @db-browser/*worker repo (:tx-data tx-report) (:tx-meta tx-report))))}
+                                    (doseq [tx-report tx-reports]
+                                      (db-browser/transact! @db-browser/*worker repo (:tx-data tx-report) (:tx-meta tx-report)))))}
           {:keys [files import-state]} (gp-exporter/export-file-graph repo db-conn config-file *files options)]
     (log/info :import-file-graph {:msg (str "Import finished in " (/ (t/in-millis (t/interval start-time (t/now))) 1000) " seconds")})
     (state/set-state! :graph/importing nil)

src/main/frontend/components/page.cljs

@@ -129,14 +129,14 @@
                                 (js/setTimeout #(let [target-block page]
                                                   (dnd/move-blocks event blocks target-block nil :sibling))
                                                0)))]
-        [:div.ls-block.flex-1.flex-col.rounded-sm
+        [:div.ls-dummy-block
          {:style {:width "100%"
                 ;; The same as .dnd-separator
                   :border-top (if hover
                                 "3px solid #ccc"
                                 nil)}}
-         [:div.flex.flex-row
-          [:div.flex.flex-row.items-center.mr-2.ml-1 {:style {:height 24}}
+         [:div.flex.items-center
+          [:div.flex.items-center.mx-1.pr-1 {:style {:height 24}}
            [:span.bullet-container.cursor
             [:span.bullet]]]
           (shui/trigger-as :div.flex.flex-1
@@ -146,7 +146,7 @@
                             :on-drag-over #(util/stop %)
                             :on-drop drop-handler-fn
                             :on-drag-leave #(set-hover! false)}
-                           [:span.opacity-70
+                           [:span.opacity-70.text
                             "Click here to edit..."])]]))))
 
 (rum/defc add-button
@@ -407,7 +407,7 @@
                                                                                            (:block/format page))))
                        :else title))])]
 
-           (when (and db-based? @*hover?)
+           (when (and db-based? @*hover? (not preview?))
              (page-title-configure *show-page-info?))])))))
 
 (defn- page-mouse-over
@@ -516,7 +516,7 @@
 
          (if (and whiteboard-page? (not sidebar?))
            [:div ((state/get-component :whiteboard/tldraw-preview) (:block/uuid page))] ;; FIXME: this is not reactive
-           [:div.relative.grid.gap-2
+           [:div.relative.grid.gap-2.page-inner
             (when (and (not sidebar?) (not block?))
               [:div.flex.flex-row.space-between
                (when (or (mobile-util/native-platform?) (util/mobile?))
@@ -565,7 +565,7 @@
                (page-blocks-cp repo page (merge option {:sidebar? sidebar?
                                                         :whiteboard? whiteboard?}))])])
 
-         (when @(::main-ready? state)
+         (when (and (not preview?) @(::main-ready? state))
            [:div {:style {:padding-left 9}}
             (when today?
               (today-queries repo today? sidebar?))

src/main/frontend/components/page.css

@@ -241,3 +241,31 @@ html.is-native-ios {
 .no-ring {
   @apply focus:ring-0 focus:ring-offset-0;
 }
+
+.ls-dummy-block {
+  @apply pb-[27px];
+
+  .bullet {
+    @apply relative top-[4px] left-[-2px];
+  }
+
+  .text {
+    @apply relative top-[3px];
+  }
+}
+
+.ls-preview-popup {
+  @apply pl-6;
+
+  .tippy-wrapper {
+    @apply p-2;
+
+    &.as-page {
+      @apply -ml-5 pl-7 outline-none;
+    }
+
+    &.as-block {
+      @apply -ml-5;
+    }
+  }
+}

src/main/frontend/components/reference.css

@@ -15,9 +15,9 @@
 }
 
 .ls-filters {
-    max-width: 704px;
+  max-width: 704px;
 }
 
 .custom-query-page-result {
-    @apply p-4 my-1.5 -mx-1 rounded;
+  @apply p-4 my-1.5 -mx-1 rounded bg-gray-03;
 }

src/main/frontend/components/reference_filters.cljs

@@ -6,6 +6,7 @@
             [frontend.search :as search]
             [frontend.ui :as ui]
             [frontend.util :as util]
+            [promesa.core :as p]
             [rum.core :as rum]
             [frontend.config :as config]
             [frontend.state :as state]
@@ -84,11 +85,12 @@
       [:input.cp__filters-input.w-full.bg-transparent
        {:placeholder (t :linked-references/filter-search)
         :autofocus true
+        :ref (fn [^js el] (-> (p/delay 32) (p/then #(.focus el))))
         :on-change (fn [e]
                      (reset! filter-search (util/evalue e)))}]]
      (let [all-filters (set
-                        (concat (map :block/name included)
-                                (map :block/name excluded)))
+                         (concat (map :block/name included)
+                           (map :block/name excluded)))
            refs (remove (fn [[page _]] (all-filters (util/page-name-sanity-lc page)))
                         filtered-references)]
        (when (seq refs)

src/main/frontend/components/settings.cljs

@@ -1146,7 +1146,7 @@
          [:div.flex.flex-row.items-center.gap-2 {:key (str "user-" user-name)}
           [:div user-name]
           (when user-email [:div.opacity-50.text-sm user-email])
-          (when graph<->user-user-type [:div.opacity-50.text-sm graph<->user-user-type])])]
+          (when graph<->user-user-type [:div.opacity-50.text-sm (name graph<->user-user-type)])])]
       [:div.flex.flex-col.gap-4.mt-4
        (shui/input
         {:placeholder   "Email address"

src/main/frontend/components/shortcut.cljs

@@ -204,6 +204,7 @@
              [:code [:small (str id')]]]]))]])])
 
 (rum/defc ^:large-vars/cleanup-todo customize-shortcut-dialog-inner
+  "user-binding: empty vector is for the unset state, nil is for the default binding"
   [k action-name binding user-binding {:keys [saved-cb modal-id]}]
   (let [*ref-el (rum/use-ref nil)
         [modal-life _] (r/use-atom *customize-modal-life-sentry)
@@ -214,7 +215,6 @@
         handler-id (rum/use-memo #(dh/get-group k))
         dirty? (not= (or user-binding binding) current-binding)
         keypressed? (not= "" keystroke)
-
         save-keystroke-fn!
         (fn []
           ;; parse current binding conflicts
@@ -323,11 +323,11 @@
 
      [:div.action-btns.text-right.mt-6.flex.justify-between.items-center
       ;; restore default
-      (if (and dirty? (or user-binding binding))
+      (if (and (not= current-binding binding) (seq binding))
         [:a.flex.items-center.space-x-1.text-sm.fade-link
-         {:on-click #(set-current-binding! (or user-binding binding))}
+         {:on-click #(set-current-binding! binding)}
          (t :keymap/restore-to-default)
-         (for [it (some->> (or binding user-binding) (map #(some->> % (dh/mod-key) (shortcut-utils/decorate-binding))))]
+         (for [it (some->> binding (map #(some->> % (dh/mod-key) (shortcut-utils/decorate-binding))))]
            [:span.keyboard-shortcut.ml-1 [:code it]])]
         [:div])
 

src/main/frontend/components/table.css

@@ -54,3 +54,25 @@
     width: auto !important;
   }
 }
+
+.query-table {
+  @apply my-2 rounded;
+
+  table {
+    @apply w-full border-none;
+  }
+
+  th {
+    @apply bg-gray-03;
+  }
+
+  tr {
+    &:nth-child(even) {
+      @apply bg-gray-02;
+    }
+  }
+
+  th, td {
+    @apply p-1.5 border border-collapse;
+  }
+}

src/main/frontend/db/rtc/debug_ui.cljs

@@ -82,7 +82,13 @@
                                                      :graph<->user-user-type
                                                      :graph<->user-grant-by-user])))
                       graph-list)))))}
-       (shui/tabler-icon "download") "graph-list")]
+       (shui/tabler-icon "download") "graph-list")
+      (shui/button
+       {:size :sm
+        :on-click #(c.m/run-task
+                    (user/new-task--upload-user-avatar "TEST_AVATAR")
+                    :upload-test-avatar)}
+       (shui/tabler-icon "upload") "upload-test-avatar")]
 
      [:div.pb-4
       [:pre.select-text

src/main/frontend/extensions/srs.cljs

@@ -771,12 +771,14 @@
 (commands/register-slash-command ["Cards"
                                   [[:editor/input "{{cards }}" {:backward-pos 2}]]
                                   "Create a cards query"
-                                  {:db-graph? false}])
+                                  {:db-graph? false
+                                   :icon :icon/cards}])
 
 (commands/register-slash-command ["Cloze"
                                   [[:editor/input "{{cloze }}" {:backward-pos 2}]]
                                   "Create a cloze"
-                                  {:db-graph? false}])
+                                  {:db-graph? false
+                                   :icon :icon/eye-question}])
 
 ;; handlers
 (defn add-card-tag-to-block

src/main/frontend/handler/editor.cljs

@@ -3244,10 +3244,17 @@
   (some-> (first (dom/by-class "reveal"))
           (dom/has-class? "focused")))
 
+(defn- in-page-preview?
+  []
+  (some-> js/document.activeElement
+    (.closest ".ls-preview-popup")
+    (nil?) (not)))
+
 (defn shortcut-up-down [direction]
   (fn [e]
     (when (and (not (auto-complete?))
-               (not (in-shui-popup?))
+               (or (in-page-preview?)
+                 (not (in-shui-popup?)))
                (not (slide-focused?))
                (not (state/get-timestamp-block)))
       (util/stop e)
@@ -3287,17 +3294,14 @@
 (defn open-selected-block!
   [direction e]
   (let [selected-blocks (state/get-selection-blocks)
-        f (case direction
-            :left first
-            :right last)]
-    (when-let [block-id (some-> selected-blocks
-                                f
-                                (dom/attr "blockid")
-                                uuid)]
+        f (case direction :left first :right last)
+        node (some-> selected-blocks f)]
+    (when-let [block-id (some-> node (dom/attr "blockid") uuid)]
       (util/stop e)
-      (let [block    {:block/uuid block-id}
-            left?    (= direction :left)]
-        (edit-block! block (if left? 0 :max))))))
+      (let [block {:block/uuid block-id}
+            left? (= direction :left)
+            opts {:container-id (some-> node (dom/attr "containerid") (parse-long))}]
+        (edit-block! block (if left? 0 :max) opts)))))
 
 (defn shortcut-left-right [direction]
   (fn [e]

src/main/frontend/handler/user.cljs

@@ -6,6 +6,7 @@
             [cljs-time.core :as t]
             [cljs.core.async :as async :refer [<! go]]
             [clojure.string :as string]
+            [frontend.common.missionary-util :as c.m]
             [frontend.config :as config]
             [frontend.debug :as debug]
             [frontend.handler.config :as config-handler]
@@ -13,7 +14,8 @@
             [frontend.state :as state]
             [goog.crypt :as crypt]
             [goog.crypt.Hmac]
-            [goog.crypt.Sha256]))
+            [goog.crypt.Sha256]
+            [missionary.core :as m]))
 
 (defn set-preferred-format!
   [format]
@@ -290,6 +292,23 @@
     [repo]
     (= (get-user-type repo) "member"))
 
+(defn new-task--upload-user-avatar
+  [avatar-str]
+  (m/sp
+    (when-let [token (state/get-auth-id-token)]
+      (let [{:keys [status body] :as resp}
+            (c.m/<?
+             (http/post
+              (str "https://" config/API-DOMAIN "/logseq/get_presigned_user_avatar_put_url")
+              {:oauth-token token
+               :with-credentials? false}))]
+        (when-not (http/unexceptional-status? status)
+          (throw (ex-info "failed to get presigned url" {:resp resp})))
+        (let [presigned-url (:presigned-url body)
+              {:keys [status]} (c.m/<? (http/put presigned-url {:body avatar-str :with-credentials? false}))]
+          (when-not (http/unexceptional-status? status)
+            (throw (ex-info "failed to upload avatar" {:resp resp}))))))))
+
 (comment
   ;; We probably need this for some new features later
   (defonce feature-matrix {:file-sync :beta})

src/main/frontend/tippy-tooltip.css

@@ -74,15 +74,6 @@
   text-align: left;
 }
 
-.tippy-wrapper {
-  border-radius: 8px;
-  margin: 0 -6px;
-}
-
-.tippy-hover {
-  cursor: pointer;
-}
-
 .tippy-popper .tippy-tooltip.monospace-theme {
   font-family: 'Fira Code', Monaco, Menlo, Consolas, 'COURIER NEW', monospace;
 }

src/main/frontend/util.cljc

@@ -883,10 +883,12 @@
       (when-let [blocks (if container
                           (get-blocks-noncollapse container)
                           (get-blocks-noncollapse))]
-        (when-let [index (.indexOf blocks block)]
-          (let [idx (dec index)]
-            (when (>= idx 0)
-              (nth-safe blocks idx))))))))
+        (let [block-id (.-id block)
+              block-ids (mapv #(.-id %) blocks)]
+          (when-let [index (.indexOf block-ids block-id)]
+            (let [idx (dec index)]
+              (when (>= idx 0)
+                (nth-safe blocks idx)))))))))
 
 #?(:cljs
    (defn get-prev-block-non-collapsed-non-embed
@@ -903,11 +905,13 @@
 #?(:cljs
    (defn get-next-block-non-collapsed
      [block]
-     (when-let [blocks (get-blocks-noncollapse)]
-       (when-let [index (.indexOf blocks block)]
-         (let [idx (inc index)]
-           (when (>= (count blocks) idx)
-             (nth-safe blocks idx)))))))
+     (when-let [blocks (and block (get-blocks-noncollapse))]
+       (let [block-id (.-id block)
+             block-ids (mapv #(.-id %) blocks)]
+         (when-let [index (.indexOf block-ids block-id)]
+           (let [idx (inc index)]
+             (when (>= (count blocks) idx)
+               (nth-safe blocks idx))))))))
 
 #?(:cljs
    (defn get-next-block-non-collapsed-skip

src/main/frontend/worker/rtc/asset.cljs

@@ -63,12 +63,12 @@
         (doseq [[asset-uuid put-url] asset-uuid->url]
           (assert (uuid? asset-uuid) asset-uuid)
           (let [{:keys [status] :as r}
-                (m/? (c.m/<! (http/put put-url {:headers {"x-amz-meta-checksum" "TEST-CHECKSUM"}
-                                                :body (js/JSON.stringify
-                                                       (clj->js {:TEST-ASSET true
-                                                                 :asset-uuid (str asset-uuid)
-                                                                 :graph-uuid (str graph-uuid)}))
-                                                :with-credentials? false})))]
+                (c.m/<? (http/put put-url {:headers {"x-amz-meta-checksum" "TEST-CHECKSUM"}
+                                           :body (js/JSON.stringify
+                                                  (clj->js {:TEST-ASSET true
+                                                            :asset-uuid (str asset-uuid)
+                                                            :graph-uuid (str graph-uuid)}))
+                                           :with-credentials? false}))]
             (if (not= 200 status)
               (prn :debug-failed-upload-asset {:resp r :asset-uuid asset-uuid :graph-uuid graph-uuid})
 
@@ -88,7 +88,7 @@
                  :asset-uuid->url)]
         (doseq [[asset-uuid get-url] asset-uuid->url]
           (assert (uuid? asset-uuid) asset-uuid)
-          (let [{:keys [status _body] :as r} (m/? (c.m/<! (http/get get-url {:with-credentials? false})))]
+          (let [{:keys [status _body] :as r} (c.m/<? (http/get get-url {:with-credentials? false}))]
             (if (not= 200 status)
               (prn :debug-failed-download-asset {:resp r :asset-uuid asset-uuid :graph-uuid graph-uuid})
               (when (d/entity @conn [:block/uuid asset-uuid])

src/main/frontend/worker/rtc/client.cljs

@@ -61,8 +61,9 @@
                 (take 5 (drop 2 c.m/delays)) ;retry 5 times if remote-graph is creating (4000 8000 16000 32000 64000)
                 (register-graph-updates get-ws-create-task graph-uuid repo)))
           (let [t (op-mem-layer/get-local-tx repo)]
-            (when (or (nil? @*last-calibrate-t)
-                      (< 500 (- t @*last-calibrate-t)))
+            (when (and (zero? (op-mem-layer/get-unpushed-block-update-count repo))
+                       (or (nil? @*last-calibrate-t)
+                           (< 500 (- t @*last-calibrate-t))))
               (m/? (r.skeleton/new-task--calibrate-graph-skeleton get-ws-create-task graph-uuid conn t))
               (reset! *last-calibrate-t t)))
           (swap! *sent assoc ws true))

src/main/frontend/worker/rtc/const.cljs

@@ -78,10 +78,20 @@
    [:failed-ops {:optional true} [:sequential to-ws-op-schema]]
    [:s3-presign-url {:optional true} :string]
    [:diff-data {:optional true} [:map-of :keyword :any]]
-   [:online-users {:optional true} [:sequential [:map
-                                                 [:user/uuid :uuid]
-                                                 [:user/name :string]
-                                                 [:user/email :string]]]]
+   [:users {:optional true} [:sequential
+                             [:map {:closed true}
+                              [:user/uuid :uuid]
+                              [:user/name :string]
+                              [:user/email :string]
+                              [:user/online? :boolean]
+                              [:user/avatar {:optional true} :string]
+                              [:graph<->user/user-type :keyword]]]]
+   [:online-users {:optional true} [:sequential
+                                    [:map {:closed true}
+                                     [:user/uuid :uuid]
+                                     [:user/name :string]
+                                     [:user/email :string]
+                                     [:user/avatar {:optional true} :string]]]]
    [:refed-blocks {:optional true}
     [:maybe
      [:sequential

src/main/frontend/worker/rtc/db_listener.cljs

@@ -1,11 +1,11 @@
 (ns frontend.worker.rtc.db-listener
   "listen datascript changes, infer operations from the db tx-report"
-  (:require [datascript.core :as d]
+  (:require [clojure.string :as string]
+            [datascript.core :as d]
             [frontend.schema-register :include-macros true :as sr]
             [frontend.worker.db-listener :as db-listener]
             [frontend.worker.rtc.op-mem-layer :as op-mem-layer]
-            [logseq.db :as ldb]
-            [clojure.string :as string]))
+            [logseq.db :as ldb]))
 
 (defn- latest-add?->v->t
   [add?->v->t]
@@ -112,33 +112,10 @@
                           [:update t {:block-uuid block-uuid :av-coll av-coll}]))]
         (cond-> ops update-op (conj update-op))))))
 
-(defn- entity-datoms=>asset-op
-  [db-after id->attr->datom entity-datoms]
-  (when-let [e (ffirst entity-datoms)]
-    (let [attr->datom (id->attr->datom e)]
-      (when (seq attr->datom)
-        (let [{[_e _a asset-uuid _t add1?] :asset/uuid
-               [_e _a asset-meta _t add2?] :asset/meta}
-              attr->datom
-              op (cond
-                   (or (and add1? asset-uuid)
-                       (and add2? asset-meta))
-                   [:update-asset]
-
-                   (and (not add1?) asset-uuid)
-                   [:remove-asset asset-uuid])]
-          (when op
-            (let [asset-uuid (some-> (d/entity db-after e) :asset/uuid str)]
-              (case (first op)
-                :update-asset (when asset-uuid ["update-asset" {:asset-uuid asset-uuid}])
-                :remove-asset ["remove-asset" {:asset-uuid (str (second op))}]))))))))
-
 (defn- generate-rtc-ops
-  [repo db-before db-after same-entity-datoms-coll id->attr->datom e->a->v->add?->t]
-  (let [asset-ops (keep (partial entity-datoms=>asset-op db-after id->attr->datom) same-entity-datoms-coll)
-        ops (when (empty asset-ops)
-              (mapcat (partial entity-datoms=>ops db-before db-after e->a->v->add?->t)
-                      same-entity-datoms-coll))]
+  [repo db-before db-after same-entity-datoms-coll e->a->v->add?->t]
+  (let [ops (mapcat (partial entity-datoms=>ops db-before db-after e->a->v->add?->t)
+                    same-entity-datoms-coll)]
     (when (seq ops)
       (op-mem-layer/add-ops! repo ops))))
 
@@ -147,7 +124,7 @@
 
 (defmethod db-listener/listen-db-changes :gen-rtc-ops
   [_ {:keys [_tx-data tx-meta db-before db-after
-             repo id->attr->datom e->a->add?->v->t same-entity-datoms-coll]}]
+             repo _id->attr->datom e->a->add?->v->t same-entity-datoms-coll]}]
   (when (and (op-mem-layer/rtc-db-graph? repo)
              (:persist-op? tx-meta true))
-    (generate-rtc-ops repo db-before db-after same-entity-datoms-coll id->attr->datom e->a->add?->v->t)))
+    (generate-rtc-ops repo db-before db-after same-entity-datoms-coll e->a->add?->v->t)))

src/main/frontend/worker/rtc/full_upload_download_graph.cljs

@@ -121,7 +121,7 @@
                 (ldb/write-transit-str all-blocks)))))]
       (rtc-log-and-state/rtc-log :rtc.log/upload {:sub-type :upload-data
                                                   :message "uploading data"})
-      (m/? (c.m/<! (http/put url {:body all-blocks-str :with-credentials? false})))
+      (c.m/<? (http/put url {:body all-blocks-str :with-credentials? false}))
       (rtc-log-and-state/rtc-log :rtc.log/upload {:sub-type :request-upload-graph
                                                   :message "requesting upload-graph"})
       (let [upload-resp
@@ -314,7 +314,7 @@
                                                   :message "downloading graph data"
                                                   :graph-uuid graph-uuid})
     (let [^js worker-obj              (:worker/object @worker-state/*state)
-          {:keys [status body] :as r} (m/? (c.m/<! (http/get s3-url {:with-credentials? false})))
+          {:keys [status body] :as r} (c.m/<? (http/get s3-url {:with-credentials? false}))
           repo                        (str sqlite-util/db-version-prefix graph-name)]
       (if (not= 200 status)
         (throw (ex-info "download-graph from s3 failed" {:resp r}))

src/main/frontend/worker/rtc/op_mem_layer.cljs

@@ -73,6 +73,7 @@
    [:local-tx {:optional true} :int]
    [:block-uuid->ops [:map-of :uuid
                       [:map-of [:enum :move :remove :update :update-page :remove-page] :any]]]
+   ;; TODO: remove :asset-uuid->ops
    [:asset-uuid->ops [:map-of :uuid
                       [:map-of [:enum :update-asset :remove-asset] :any]]]
    [:t+block-uuid-sorted-set [:set [:cat :int :uuid]]]])

src/main/frontend/worker/rtc/ws.cljs

@@ -177,7 +177,7 @@
           message (assoc message :req-id req-id)
           resp (m/? (send&recv* mws message :timeout-ms timeout-ms))]
       (if-let [s3-presign-url (:s3-presign-url resp)]
-        (let [{:keys [status body]} (m/? (c.m/<! (http/get s3-presign-url {:with-credentials? false})))]
+        (let [{:keys [status body]} (c.m/<? (http/get s3-presign-url {:with-credentials? false}))]
           (if (http/unexceptional-status? status)
             (rtc-const/data-from-ws-coercer (js->clj (js/JSON.parse body) :keywordize-keys true))
             {:req-id req-id

src/resources/dicts/en.edn

@@ -620,7 +620,7 @@
  :keymap/keystroke-filter "Keystroke filter"
  :keymap/keystroke-record-desc "Press any sequence of keys to filter shortcuts"
  :keymap/keystroke-record-setup-label "Press any sequence of keys to set a shortcut"
- :keymap/restore-to-default "Restore to system default"
+ :keymap/restore-to-default "Restore to the default"
  :keymap/customize-for-label "Customize shortcuts"
  :keymap/conflicts-for-label "Keymap conflicts for"
 

src/test/frontend/handler/repo_test.cljs

@@ -25,7 +25,7 @@
 
     (docs-graph-helper/docs-graph-assertions db graph-dir (map :file/path files))
     (testing "Additional Counts"
-      (is (= 64476 (count (d/datoms db :eavt))) "Correct datoms count")
+      (is (= 64477 (count (d/datoms db :eavt))) "Correct datoms count")
 
       (is (= 5946
              (ffirst