Merge branch 'master' into feat/hnswlib+transformer-js

.github/workflows/build-android.yml

@@ -50,7 +50,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out Git repository
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c  # v3.3.0
+        uses: actions/checkout@v4
         with:
           ref: ${{ github.event.inputs.git-ref }}
 

.github/workflows/build-desktop-release.yml

@@ -62,7 +62,7 @@ jobs:
           exit 1
 
       - name: Check out Git repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: ${{ github.event.inputs.git-ref }}
 

.github/workflows/build-docker.yml

@@ -16,7 +16,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c  # v3.3.0
+        uses: actions/checkout@v4
         with:
           fetch-depth: 1
           submodules: 'true'

.github/workflows/build-ios-release.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: macos-15
     steps:
       - name: Check out Git repository
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c  # v3.3.0
+        uses: actions/checkout@v4
         with:
           ref: ${{ github.event.inputs.git-ref }}
       - uses: maxim-lobanov/setup-xcode@v1

.github/workflows/build-ios.yml

@@ -25,7 +25,7 @@ jobs:
     runs-on: macos-14
     steps:
       - name: Check out Git repository
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c  # v3.3.0
+        uses: actions/checkout@v4
 
       - name: Install Node.js, NPM and Yarn
         uses: actions/setup-node@v3

.github/workflows/db.yml

@@ -8,12 +8,16 @@ on:
       - 'deps/db/**'
       - '.github/workflows/db.yml'
       - '!deps/db/**.md'
+      # Deps that logseq/db depends on should trigger this workflow
+      - 'deps/common/**'
   pull_request:
     branches: [master]
     paths:
       - 'deps/db/**'
       - '.github/workflows/db.yml'
       - '!deps/db/**.md'
+      # Deps that logseq/db depends on should trigger this workflow
+      - 'deps/common/**'
 
 defaults:
   run:
@@ -32,7 +36,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c  # v3.3.0
+        uses: actions/checkout@v4
 
       - name: Set up Node
         uses: actions/setup-node@v3
@@ -68,7 +72,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c  # v3
+        uses: actions/checkout@v4
 
       - name: Set up Java
         uses: actions/setup-java@v3

.github/workflows/deploy-db-test-pages.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Setup Java JDK
         uses: actions/setup-java@v3

.github/workflows/graph-parser.yml

@@ -7,17 +7,20 @@ on:
     branches: [master]
     paths:
       - 'deps/graph-parser/**'
-      # db is a local dep that could break functionality in this lib and should trigger this
-      - 'deps/db/**'
       - '.github/workflows/graph-parser.yml'
       - '!deps/graph-parser/**.md'
+      # Deps that logseq/graph-parser depends on should trigger this workflow
+      - 'deps/db/**'
+      - 'deps/common/**'
   pull_request:
     branches: [master]
     paths:
       - 'deps/graph-parser/**'
-      - 'deps/db/**'
       - '.github/workflows/graph-parser.yml'
       - '!deps/graph-parser/**.md'
+      # Deps that logseq/graph-parser depends on should trigger this workflow
+      - 'deps/db/**'
+      - 'deps/common/**'
 
 defaults:
   run:
@@ -37,7 +40,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c  # v3.3.0
+        uses: actions/checkout@v4
 
       - name: Set up Node
         uses: actions/setup-node@v3
@@ -90,7 +93,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c  # v3.3.0
+        uses: actions/checkout@v4
 
       - name: Set up Java
         uses: actions/setup-java@v3

.github/workflows/logseq-common.yml

@@ -32,7 +32,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Node
         uses: actions/setup-node@v3
@@ -85,7 +85,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Java
         uses: actions/setup-java@v3

.github/workflows/outliner.yml

@@ -7,17 +7,22 @@ on:
     branches: [master]
     paths:
       - 'deps/outliner/**'
-      # db is a local dep that could break functionality in this lib and should trigger this
-      - 'deps/db/**'
       - '.github/workflows/outliner.yml'
       - '!deps/outliner/**.md'
+      # Deps that logseq/outliner depends on should trigger this workflow
+      - 'deps/graph-parser/**'
+      - 'deps/db/**'
+      - 'deps/common/**'
   pull_request:
     branches: [master]
     paths:
       - 'deps/outliner/**'
-      - 'deps/db/**'
       - '.github/workflows/outliner.yml'
       - '!deps/outliner/**.md'
+      # Deps that logseq/outliner depends on should trigger this workflow
+      - 'deps/graph-parser/**'
+      - 'deps/db/**'
+      - 'deps/common/**'
 
 defaults:
   run:
@@ -37,7 +42,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Node
         uses: actions/setup-node@v3
@@ -74,7 +79,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Java
         uses: actions/setup-java@v3

.github/workflows/publishing.yml

@@ -7,17 +7,20 @@ on:
     branches: [master]
     paths:
       - 'deps/publishing/**'
-      # db is a local dep that could break functionality in this lib and should trigger this
-      - 'deps/db/**'
       - '.github/workflows/publishing.yml'
       - '!deps/publishing/**.md'
+      # Deps that logseq/publishing depends on should trigger this workflow
+      - 'deps/db/**'
+      - 'deps/common/**'
   pull_request:
     branches: [master]
     paths:
       - 'deps/publishing/**'
-      - 'deps/db/**'
       - '.github/workflows/publishing.yml'
       - '!deps/publishing/**.md'
+      # Deps that logseq/publishing depends on should trigger this workflow
+      - 'deps/db/**'
+      - 'deps/common/**'
 
 defaults:
   run:
@@ -37,7 +40,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Node
         uses: actions/setup-node@v3
@@ -74,7 +77,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Java
         uses: actions/setup-java@v3

README.md

@@ -74,7 +74,7 @@ The DB version is in beta status while the new mobile app and RTC is in alpha. T
 
 To get started with the DB version:
 * To try the latest web version, go to https://test.logseq.com/.
-* To try the latest desktop version, go to https://github.com/logseq/logseq/actions/workflows/build-desktop-release.yml and click on the latest release. Scroll to the bottom and under the `Artifacts` section download the artifact for your operating system.
+* To try the latest desktop version, login to Github and go to https://github.com/logseq/logseq/actions/workflows/build-desktop-release.yml and click on the latest release. Scroll to the bottom and under the `Artifacts` section download the artifact for your operating system.
 * To report bugs, please file them at https://github.com/logseq/db-test/issues.
 * For feature or enhancement requests, please file them on Discord on the `#db-feedback` channel.
 * For discussion, see the `#db-chat` channel in Discord.

deps/db/src/logseq/db/test/helper.cljs

@@ -63,7 +63,7 @@
        (mapv (fn [[k v]]
                [k
                 (cond
-                  (= :block/tags k)
+                  (#{:block/tags :logseq.property.class/extends} k)
                   (mapv :db/ident v)
                   (and (set? v) (every? de/entity? v))
                   (set (map db-property/property-value-content v))

deps/graph-parser/package.json

@@ -7,7 +7,8 @@
     "better-sqlite3": "11.10.0"
   },
   "dependencies": {
-    "mldoc": "^1.5.9"
+    "mldoc": "^1.5.9",
+    "sanitize-filename": "1.6.3"
   },
   "scripts": {
     "test": "nbb-logseq -cp test:../outliner/src -m nextjournal.test-runner",

deps/graph-parser/script/db_import.cljs

@@ -57,14 +57,15 @@
            {:size (.-length buffer)
             :checksum checksum
             :type (db-asset/asset-path->type (:path file))
-            :path (:path file)})))
+            :path (:path file)})
+    buffer))
 
 (defn- <copy-asset-file [asset-m db-graph-dir]
   (p/let [parent-dir (node-path/join db-graph-dir common-config/local-assets-dir)
           _ (fsp/mkdir parent-dir #js {:recursive true})]
     (if (:block/uuid asset-m)
       (fsp/copyFile (:path asset-m) (node-path/join parent-dir (str (:block/uuid asset-m) "." (:type asset-m))))
-      (do
+      (when-not (:pdf-annotation? asset-m)
         (println "[INFO]" "Copied asset" (pr-str (node-path/basename (:path asset-m)))
                  "by its name since it was unused.")
         (fsp/copyFile (:path asset-m) (node-path/join parent-dir (node-path/basename (:path asset-m))))))))

deps/graph-parser/src/logseq/graph_parser/exporter.cljs

@@ -2,6 +2,7 @@
   "Exports a file graph to DB graph. Used by the File to DB graph importer and
   by nbb-logseq CLIs"
   (:require ["path" :as node-path]
+            ["sanitize-filename" :as sanitizeFilename]
             [borkdude.rewrite-edn :as rewrite]
             [cljs-time.coerce :as tc]
             [cljs.pprint]
@@ -35,6 +36,7 @@
             [logseq.graph-parser.block :as gp-block]
             [logseq.graph-parser.extract :as extract]
             [logseq.graph-parser.property :as gp-property]
+            [logseq.graph-parser.utf8 :as utf8]
             [promesa.core :as p]))
 
 (defn- add-missing-timestamps
@@ -1005,9 +1007,137 @@
           block-title
           asset-name-to-uuids))
 
+(defn find-annotation-children-blocks
+  "Given a list of blocks and a set of parent uuids, return all blocks that are
+   descendants via :block/parent of given parent uuids"
+  [blocks parent-uuids]
+  (let [get-descendant-uuids
+        (fn get-descendant-uuids [acc-uuids seen]
+          (let [new-blocks (filter #(contains? acc-uuids (second (:block/parent %))) blocks)
+                new-uuids  (set (map :block/uuid new-blocks))
+                unseen     (set/difference new-uuids seen)]
+            (if (empty? unseen)
+              seen
+              (recur unseen (set/union seen unseen)))))
+        parent-and-descendant-uuids (get-descendant-uuids parent-uuids parent-uuids)
+        only-descendants (set/difference parent-and-descendant-uuids parent-uuids)]
+    (filter #(contains? only-descendants (:block/uuid %)) blocks)))
+
+(defn- build-annotation-block
+  [m color-text-idents parent-asset image-asset-name-to-uuids md-blocks {:keys [log-fn] :or {log-fn prn}}]
+  (let [user-attributes
+        {:logseq.property.pdf/hl-color (get color-text-idents (get-in m [:properties :color]))
+         :logseq.property.pdf/hl-page (:page m)
+         :block/title (get-in m [:content :text])}
+        _ (when (some (comp nil? val) user-attributes)
+            (log-fn :missing-annotation-attributes "Annotation is missing some attributes so set reasonable defaults for them"
+                    {:annotation user-attributes :asset (:block/title parent-asset)}))
+        asset-image-uuid (some (fn [[asset-name image-uuid]]
+                                 (when (string/includes? asset-name
+                                                         (str (:id m)
+                                                              (when (get-in m [:content :image])
+                                                                (str "_" (get-in m [:content :image])))))
+                                   image-uuid))
+                               image-asset-name-to-uuids)
+        md-block (get md-blocks (:id m))
+        annotation (merge
+                     ;; Reasonable defaults for user attributes
+                    {:logseq.property.pdf/hl-color :logseq.property/color.yellow
+                     :logseq.property.pdf/hl-page 1
+                     :block/title ""}
+                    user-attributes
+                    {:block/uuid (:id m)
+                     :block/order (db-order/gen-key)
+                     :logseq.property/ls-type :annotation
+                     :logseq.property.pdf/hl-value m
+                     :logseq.property/asset [:block/uuid (:block/uuid parent-asset)]
+                     :block/tags [:logseq.class/Pdf-annotation]
+                     :block/parent [:block/uuid (:block/uuid parent-asset)]
+                     :block/page :logseq.class/Asset}
+                    (when asset-image-uuid
+                      {:logseq.property.pdf/hl-image [:block/uuid asset-image-uuid]
+                       :logseq.property.pdf/hl-type :area})
+                    (when md-block
+                      (select-keys md-block [:block/title])))]
+    (sqlite-util/block-with-timestamps annotation)))
+
+(defn- build-pdf-annotations-tx*
+  "Creates annotations for a pdf asset given the asset's edn map and parsed markdown file"
+  [asset-edn-map parsed-md parent-asset image-asset-name-to-uuids opts]
+  (let [color-text-idents
+        (->> (get-in db-property/built-in-properties [:logseq.property.pdf/hl-color :closed-values])
+             (map (juxt :value :db-ident))
+             (into {}))
+        md-blocks
+        (->> parsed-md
+             :blocks
+             ;; Currently we can only import text of any md annotation blocks. No tags or properties
+             (map #(vector (:block/uuid %)
+                           (select-keys % [:block/title :block/order :block/parent :block/uuid])))
+             (into {}))
+        annotation-blocks
+        (mapv #(build-annotation-block % color-text-idents parent-asset image-asset-name-to-uuids md-blocks opts)
+              (get-in asset-edn-map [:edn-content :highlights]))
+        md-children-blocks*
+        (find-annotation-children-blocks (vals md-blocks) (set (map :id (get-in asset-edn-map [:edn-content :highlights]))))
+        md-children-blocks (keep #(sqlite-util/block-with-timestamps (merge % {:block/page :logseq.class/Asset}))
+                                 md-children-blocks*)]
+    (into annotation-blocks md-children-blocks)))
+
+(defn- build-new-asset [asset-data]
+  (merge (sqlite-util/block-with-timestamps
+          {:block/uuid (d/squuid)
+           :block/order (db-order/gen-key)
+           :block/page :logseq.class/Asset
+           :block/parent :logseq.class/Asset})
+         {:block/tags [:logseq.class/Asset]
+          :logseq.property.asset/type (:type asset-data)
+          :logseq.property.asset/checksum (:checksum asset-data)
+          :logseq.property.asset/size (:size asset-data)}))
+
+(defn- build-annotation-images
+  "Builds tx for annotation images and provides a map for mapping image asset names
+   to their new uuids"
+  [parent-asset-path assets]
+  (let [image-dir (string/replace-first parent-asset-path #"(?i)\.pdf$" "")
+        image-paths (filter #(= image-dir (node-path/dirname %)) (keys @assets))
+        txs (mapv #(let [new-asset (merge (build-new-asset (get @assets %))
+                                          {:block/title "pdf area highlight"})]
+                     (swap! assets assoc-in [% :block/uuid] (:block/uuid new-asset))
+                     new-asset)
+                  image-paths)]
+    {:txs txs
+     :image-asset-name-to-uuids
+     (->> (map (fn [image-path tx]
+                 [(node-path/basename image-path) (:block/uuid tx)]) image-paths txs)
+          (into {}))}))
+
+;; Reference same default class in cljs + nbb without needing .cljc
+(def sanitizeFilename' (if (find-ns 'nbb.core) (aget sanitizeFilename "default") sanitizeFilename))
+
+(defn safe-sanitize-file-name
+  "Sanitizes filenames for pdf assets"
+  [s]
+  (sanitizeFilename' (str s)))
+
+(defn- build-pdf-annotations-tx
+  "Builds tx for pdf annotations when a pdf has an annotations EDN file under assets/"
+  [parent-asset-path assets parent-asset pdf-annotation-pages opts]
+  (let [asset-edn-path (node-path/join common-config/local-assets-dir
+                                       (safe-sanitize-file-name
+                                        (node-path/basename (string/replace-first parent-asset-path #"(?i)\.pdf$" ".edn"))))
+        asset-md-name (str "hls__" (safe-sanitize-file-name
+                                    (node-path/basename (string/replace-first parent-asset-path #"(?i)\.pdf$" ".md"))))]
+    (when-let [asset-edn-map (get @assets asset-edn-path)]
+      ;; Mark edn asset so it isn't treated like a normal asset later
+      (swap! assets assoc-in [asset-edn-path :pdf-annotation?] true)
+      (let [{:keys [txs image-asset-name-to-uuids]} (build-annotation-images parent-asset-path assets)]
+        (concat txs
+                (build-pdf-annotations-tx* asset-edn-map (get @pdf-annotation-pages asset-md-name) parent-asset image-asset-name-to-uuids opts))))))
+
 (defn- handle-assets-in-block
   "If a block contains assets, creates them as #Asset nodes in the Asset page and references them in the block."
-  [block {:keys [asset-links]} {:keys [assets ignored-assets]}]
+  [block {:keys [asset-links]} {:keys [assets ignored-assets pdf-annotation-pages]} opts]
   (if (seq asset-links)
     (let [asset-maps
           (keep
@@ -1016,24 +1146,19 @@
                (if-let [asset-data (and asset-name (get @assets asset-name))]
                  (if (:block/uuid asset-data)
                    {:asset-name-uuid [asset-name (:block/uuid asset-data)]}
-                   (let [new-block (sqlite-util/block-with-timestamps
-                                    {:block/uuid (d/squuid)
-                                     :block/order (db-order/gen-key)
-                                     :block/page :logseq.class/Asset
-                                     :block/parent :logseq.class/Asset})
-                         new-asset (merge new-block
-                                          {:block/tags [:logseq.class/Asset]
-                                           :logseq.property.asset/type (:type asset-data)
-                                           :logseq.property.asset/checksum (:checksum asset-data)
-                                           :logseq.property.asset/size (:size asset-data)
-                                           :block/title (db-asset/asset-name->title (node-path/basename asset-name))}
+                   (let [new-asset (merge (build-new-asset asset-data)
+                                          {:block/title (db-asset/asset-name->title (node-path/basename asset-name))}
                                           (when-let [metadata (not-empty (common-util/safe-read-map-string (:metadata (second asset-link))))]
-                                            {:logseq.property.asset/resize-metadata metadata}))]
-                      ;;  (prn :asset-added! (node-path/basename asset-name) #_(get @assets asset-name))
-                      ;;  (cljs.pprint/pprint asset-link)
-                     (swap! assets assoc-in [asset-name :block/uuid] (:block/uuid new-block))
+                                            {:logseq.property.asset/resize-metadata metadata}))
+                         pdf-annotations-tx (when (= "pdf" (path/file-ext asset-name))
+                                              (build-pdf-annotations-tx asset-name assets new-asset pdf-annotation-pages opts))
+                         asset-tx (concat [new-asset]
+                                          (when pdf-annotations-tx pdf-annotations-tx))]
+                    ;;  (prn :asset-added! (node-path/basename asset-name))
+                    ;;  (cljs.pprint/pprint asset-link)
+                     (swap! assets assoc-in [asset-name :block/uuid] (:block/uuid new-asset))
                      {:asset-name-uuid [asset-name (:block/uuid new-asset)]
-                      :asset new-asset}))
+                      :asset-tx asset-tx}))
                  (do
                    (swap! ignored-assets conj
                           {:reason "No asset data found for this asset path"
@@ -1041,7 +1166,7 @@
                            :location {:block (:block/title block)}})
                    nil))))
            asset-links)
-          asset-blocks (keep :asset asset-maps)
+          asset-blocks (mapcat :asset-tx asset-maps)
           asset-names-to-uuids
           (into {} (map :asset-name-uuid asset-maps))]
       (cond-> {:block
@@ -1094,7 +1219,7 @@
         {block-after-built-in-props :block deadline-properties-tx :properties-tx}
         (update-block-deadline-and-scheduled block page-names-to-uuids options)
         {block-after-assets :block :keys [asset-blocks-tx]}
-        (handle-assets-in-block block-after-built-in-props walked-ast-blocks (select-keys import-state [:assets :ignored-assets]))
+        (handle-assets-in-block block-after-built-in-props walked-ast-blocks import-state (select-keys options [:log-fn]))
         ;; :block/page should be [:block/page NAME]
         journal-page-created-at (some-> (:block/page block*) second journal-created-ats)
         prepared-block (cond-> block-after-assets
@@ -1112,7 +1237,7 @@
                    add-missing-timestamps
                    ;; old whiteboards may have :block/left
                    (dissoc :block/left :block/format :block.temp/ast-blocks)
-                  ;;  ((fn [x] (prn :block-out x) x))
+                  ;;  ((fn [x] (prn ::block-out x) x))
                    )]
     ;; Order matters as previous txs are referenced in block
     (concat properties-tx deadline-properties-tx asset-blocks-tx [block'])))
@@ -1350,6 +1475,8 @@
    :ignored-files (atom [])
    ;; Vec of maps with keys :path, :reason and :location (optional).
    :ignored-assets (atom [])
+   ;; Map of annotation page paths and their parsed contents
+   :pdf-annotation-pages (atom {})
    ;; Map of property names (keyword) and their current schemas (map of qualified properties).
    ;; Used for adding schemas to properties and detecting changes across a property's usage
    :property-schemas (atom {})
@@ -1492,8 +1619,6 @@
   "Main fn which calls graph-parser to convert markdown into data"
   [db file content {:keys [extract-options import-state]}]
   (let [format (common-util/get-format file)
-        ;; TODO: Remove once pdf highlights are supported
-        ignored-highlight-file? (string/starts-with? (str (path/basename file)) "hls__")
         extract-options' (merge {:block-pattern (common-config/get-block-pattern format)
                                  :date-formatter "MMM do, yyyy"
                                  :uri-encoded? false
@@ -1501,30 +1626,34 @@
                                  :export-to-db-graph? true
                                  :filename-format :legacy}
                                 extract-options
-                                {:db db})]
-    (cond (and (contains? common-config/mldoc-support-formats format) (not ignored-highlight-file?))
-          (-> (extract/extract file content extract-options')
-              (update :pages (fn [pages]
-                               (map #(dissoc % :block.temp/original-page-name) pages)))
-              (update :blocks fix-extracted-block-tags-and-refs))
-
-          (common-config/whiteboard? file)
-          (-> (extract/extract-whiteboard-edn file content extract-options')
-              (update :pages (fn [pages]
-                               (->> pages
-                                    ;; migrate previous attribute for :block/title
-                                    (map #(-> %
-                                              (assoc :block/title (or (:block/original-name %) (:block/title %))
-                                                     :block/tags #{:logseq.class/Whiteboard})
-                                              (dissoc :block/type :block/original-name))))))
-              (update :blocks update-whiteboard-blocks format))
+                                {:db db})
+        extracted
+        (cond (contains? common-config/mldoc-support-formats format)
+              (-> (extract/extract file content extract-options')
+                  (update :pages (fn [pages]
+                                   (map #(dissoc % :block.temp/original-page-name) pages)))
+                  (update :blocks fix-extracted-block-tags-and-refs))
+
+              (common-config/whiteboard? file)
+              (-> (extract/extract-whiteboard-edn file content extract-options')
+                  (update :pages (fn [pages]
+                                   (->> pages
+                                        ;; migrate previous attribute for :block/title
+                                        (map #(-> %
+                                                  (assoc :block/title (or (:block/original-name %) (:block/title %))
+                                                         :block/tags #{:logseq.class/Whiteboard})
+                                                  (dissoc :block/type :block/original-name))))))
+                  (update :blocks update-whiteboard-blocks format))
 
-          :else
-          (if ignored-highlight-file?
-            (swap! (:ignored-files import-state) conj
-                   {:path file :reason :pdf-highlight})
-            (swap! (:ignored-files import-state) conj
-                   {:path file :reason :unsupported-file-format})))))
+              :else
+              (swap! (:ignored-files import-state) conj
+                     {:path file :reason :unsupported-file-format}))]
+    ;; Annotation markdown pages are saved for later as they are dependant on the asset being annotated
+    (if (string/starts-with? (str (path/basename file)) "hls__")
+      (do
+        (swap! (:pdf-annotation-pages import-state) assoc (node-path/basename file) extracted)
+        nil)
+      extracted)))
 
 (defn- build-journal-created-ats
   "Calculate created-at timestamps for journals"
@@ -1683,7 +1812,10 @@
   (set-ui-state [:graph/importing-state :total] (count *doc-files))
   (let [doc-files (mapv #(assoc %1 :idx %2)
                         ;; Sort files to ensure reproducible import behavior
-                        (sort-by :path *doc-files)
+                        ;; pdf annotation pages sort first because other pages depend on them
+                        (sort-by (fn [{:keys [path]}]
+                                   [(not (string/starts-with? (node-path/basename path) "hls__")) path])
+                                 *doc-files)
                         (range 0 (count *doc-files)))]
     (-> (p/loop [_file-map (export-doc-file (get doc-files 0) conn <read-file options)
                  i 0]
@@ -1793,7 +1925,11 @@
                           (sort-by :path *asset-files)
                           (range 0 (count *asset-files)))
         read-asset (fn read-asset [{:keys [path] :as file}]
-                     (-> (<read-asset-file file assets)
+                     (-> (p/let [byte-array (<read-asset-file file assets)]
+                           (when (= "edn" (path/file-ext (:path file)))
+                             (swap! assets assoc-in
+                                    [(asset-path->name path) :edn-content]
+                                    (common-util/safe-read-map-string (utf8/decode byte-array)))))
                          (p/catch
                           (fn [error]
                             (notify-user {:msg (str "Import failed to read " (pr-str path) " with error:\n" (.-message error))
@@ -1916,10 +2052,11 @@
                        (set/rename-keys {:<save-config-file :<save-file})))]
      (let [files (common-config/remove-hidden-files *files config rpath-key)
            logseq-file? #(string/starts-with? (get % rpath-key) "logseq/")
+           asset-file? #(string/starts-with? (get % rpath-key) "assets/")
            doc-files (->> files
-                          (remove logseq-file?)
+                          (remove #(or (logseq-file? %) (asset-file? %)))
                           (filter #(contains? #{"md" "org" "markdown" "edn"} (path/file-ext (:path %)))))
-           asset-files (filter #(string/starts-with? (get % rpath-key) "assets/") files)
+           asset-files (filter asset-file? files)
            doc-options (build-doc-options config options)]
        (log-fn "Importing" (count doc-files) "files ...")
        ;; These export* fns are all the major export/import steps
@@ -1928,7 +2065,7 @@
                              (-> (select-keys options [:notify-user :<save-logseq-file])
                                  (set/rename-keys {:<save-logseq-file :<save-file})))
         ;; Assets are read first as doc-files need data from them to make Asset blocks.
-        ;; Assets are copied after after doc-files as they need block/uuid's from them to name assets
+        ;; Assets are copied after doc-files as they need block/uuid's from them to name assets
         (read-asset-files asset-files <read-asset (merge (select-keys options [:notify-user :set-ui-state])
                                                          {:assets (get-in doc-options [:import-state :assets])}))
         (export-doc-files conn doc-files <read-file doc-options)

deps/graph-parser/src/logseq/graph_parser/property.cljs

@@ -66,7 +66,7 @@
      :id :background-color :heading :collapsed
      :created-at :updated-at :last-modified-at
      :query-table :query-properties :query-sort-by :query-sort-desc :ls-type
-     :hl-type :hl-page :hl-stamp :hl-color :hl-value
+     :hl-type :hl-page :hl-stamp :hl-color :hl-value :logseq.macro-name :logseq.macro-arguments
      :logseq.order-list-type :logseq.tldraw.page :logseq.tldraw.shape
      ; task markers
      :todo :doing :now :later :done}

deps/graph-parser/src/logseq/graph_parser/text.cljs

@@ -1,13 +1,13 @@
 (ns logseq.graph-parser.text
   "Miscellaneous text util fns for the parser. Used by file and DB graphs"
-  (:require [goog.string :as gstring]
+  (:require [clojure.set :as set]
             [clojure.string :as string]
-            [clojure.set :as set]
-            [logseq.graph-parser.property :as gp-property]
-            [logseq.graph-parser.mldoc :as gp-mldoc]
+            [goog.string :as gstring]
             [logseq.common.util :as common-util]
+            [logseq.common.util.namespace :as ns-util]
             [logseq.common.util.page-ref :as page-ref]
-            [logseq.common.util.namespace :as ns-util]))
+            [logseq.graph-parser.mldoc :as gp-mldoc]
+            [logseq.graph-parser.property :as gp-property]))
 
 (def get-file-basename page-ref/get-file-basename)
 
@@ -151,4 +151,4 @@
             v'))))))
 
 (def namespace-page? ns-util/namespace-page?)
-(def get-namespace-last-part ns-util/get-last-part)
+(def get-namespace-last-part ns-util/get-last-part)

deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs

@@ -103,7 +103,8 @@
            {:size (.-length buffer)
             :checksum checksum
             :type (db-asset/asset-path->type (:path file))
-            :path (:path file)})))
+            :path (:path file)})
+    buffer))
 
 ;; Copied from db-import script and tweaked for an in-memory import
 (defn- import-file-graph-to-db
@@ -119,10 +120,11 @@
                         ;; asset file options
                          :<read-asset <read-asset-file
                          :<copy-asset (fn copy-asset [m]
-                                        (when-not (:block/uuid m)
-                                          (println "[INFO]" "Asset" (pr-str (node-path/basename (:path m)))
-                                                   "does not have a :block/uuid"))
-                                        (swap! assets conj m))}
+                                        (if (:block/uuid m)
+                                          (swap! assets conj m)
+                                          (when-not (:pdf-annotation? m)
+                                            (println "[INFO]" "Asset" (pr-str (node-path/basename (:path m)))
+                                                     "does not have a :block/uuid"))))}
                         (select-keys options [:verbose]))]
     (gp-exporter/export-file-graph conn conn config-file *files options')))
 
@@ -206,13 +208,14 @@
 
       ;; Counts
       ;; Includes journals as property values e.g. :logseq.property/deadline
-      (is (= 27 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Journal]] @conn))))
+      (is (= 29 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Journal]] @conn))))
 
-      (is (= 3 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Asset]] @conn))))
+      (is (= 5 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Asset]] @conn))))
       (is (= 4 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Task]] @conn))))
       (is (= 4 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Query]] @conn))))
       (is (= 2 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Card]] @conn))))
       (is (= 3 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Quote-block]] @conn))))
+      (is (= 2 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Pdf-annotation]] @conn))))
 
       ;; Properties and tags aren't included in this count as they aren't a Page
       (is (= 10
@@ -235,7 +238,7 @@
       (is (= 0 (count @(:ignored-properties import-state))) "No ignored properties")
       (is (= 0 (count @(:ignored-assets import-state))) "No ignored assets")
       (is (= 1 (count @(:ignored-files import-state))) "Ignore .edn for now")
-      (is (= 3 (count @assets))))
+      (is (= 5 (count @assets))))
 
     (testing "logseq files"
       (is (= ".foo {}\n"
@@ -418,6 +421,32 @@
       (is (= (d/entity @conn :logseq.class/Asset)
              (:block/page (db-test/find-block-by-content @conn "greg-popovich-thumbs-up_1704749687791_0")))
           "Imported into Asset page")
+
+      ;; Annotations
+      (is (= {:logseq.property.pdf/hl-color :logseq.property/color.blue
+              :logseq.property.pdf/hl-page 8
+              :block/tags [:logseq.class/Pdf-annotation]
+              :logseq.property/asset "Sina_de_Capoeria_Batizado_2025_-_Program_Itinerary_1752179325104_0"}
+             (dissoc (db-test/readable-properties (db-test/find-block-by-content @conn #"Duke School - modified"))
+                     :logseq.property.pdf/hl-value :logseq.property/ls-type))
+          "Pdf text highlight has correct properties")
+      (is (= ["note about duke" "sub note"]
+             (mapv :block/title (rest (ldb/get-block-and-children @conn (:block/uuid (db-test/find-block-by-content @conn #"Duke School - modified"))))))
+          "Pdf text highlight has correct children blocks")
+      (is (= {:logseq.property.pdf/hl-color :logseq.property/color.yellow
+              :logseq.property.pdf/hl-page 1
+              :block/tags [:logseq.class/Pdf-annotation]
+              :logseq.property/asset "Sina_de_Capoeria_Batizado_2025_-_Program_Itinerary_1752179325104_0"
+              :logseq.property.pdf/hl-image "pdf area highlight"
+              :logseq.property.pdf/hl-type :area}
+             (dissoc (->> (d/q '[:find [?b ...]
+                                 :where [?b :block/tags :logseq.class/Pdf-annotation] [?b :block/title ""]] @conn)
+                          first
+                          (d/entity @conn)
+                          db-test/readable-properties)
+                     :logseq.property.pdf/hl-value :logseq.property/ls-type))
+          "Pdf area highlight has correct properties")
+
       ;; Quotes
       (is (= {:block/tags [:logseq.class/Quote-block]
               :logseq.property.node/display-type :quote}

deps/graph-parser/test/resources/exporter-test-graph/assets/Sina_de_Capoeria_Batizado_2025_-_Program_Itinerary_1752179325104_0.edn

@@ -0,0 +1,30 @@
+{:highlights [{:id #uuid "687022ae-dac1-42a6-9d4c-39a0dba05918",
+               :page 1,
+               :position {:bounding {:x1 131,
+                                     :y1 336,
+                                     :x2 483,
+                                     :y2 399,
+                                     :width 574.0000000000001,
+                                     :height 573.999986224},
+                          :rects (),
+                          :page 1},
+               :content {:text "", :image 1752179374600},
+               :properties {:color "yellow"}}
+              {:id #uuid "68702394-3613-4bac-85a7-28643d58237f",
+               :page 8,
+               :position {:bounding {:x1 10.680589094758034,
+                                     :y1 183.2645263671875,
+                                     :x2 119.76637782156467,
+                                     :y2 204.954345703125,
+                                     :width 574.0000000000001,
+                                     :height 573.999986224},
+                          :rects ({:x1 10.680589094758034,
+                                   :y1 183.2645263671875,
+                                   :x2 119.76637782156467,
+                                   :y2 204.954345703125,
+                                   :width 574.0000000000001,
+                                   :height 573.999986224}),
+                          :page 8},
+               :content {:text "Duke School"},
+               :properties {:color "blue"}}],
+ :extra {:page 2}}

deps/graph-parser/test/resources/exporter-test-graph/journals/2025_07_10.md

@@ -0,0 +1 @@
+- ![Sina de Capoeria Batizado 2025 - Program Itinerary.pdf](../assets/Sina_de_Capoeria_Batizado_2025_-_Program_Itinerary_1752179325104_0.pdf)

deps/graph-parser/test/resources/exporter-test-graph/journals/2025_07_15.md

@@ -0,0 +1 @@
+- Test ref to an annotation: ((68702394-3613-4bac-85a7-28643d58237f))

deps/graph-parser/test/resources/exporter-test-graph/pages/hls__Sina_de_Capoeria_Batizado_2025_-_Program_Itinerary_1752179325104_0.md

@@ -0,0 +1,16 @@
+file:: [Sina_de_Capoeria_Batizado_2025_-_Program_Itinerary_1752179325104_0.pdf](../assets/Sina_de_Capoeria_Batizado_2025_-_Program_Itinerary_1752179325104_0.pdf)
+file-path:: ../assets/Sina_de_Capoeria_Batizado_2025_-_Program_Itinerary_1752179325104_0.pdf
+
+- id:: 687022ae-dac1-42a6-9d4c-39a0dba05918
+  ls-type:: annotation
+  hl-page:: 1
+  hl-color:: yellow
+  hl-type:: area
+  hl-stamp:: 1752179374600
+- Duke School - modified
+  hl-page:: 8
+  ls-type:: annotation
+  id:: 68702394-3613-4bac-85a7-28643d58237f
+  hl-color:: blue
+	- note about duke
+		- sub note

deps/graph-parser/yarn.lock

@@ -425,6 +425,13 @@ safe-buffer@^5.0.1, safe-buffer@~5.2.0:
   resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6"
   integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==
 
[email protected]:
+  version "1.6.3"
+  resolved "https://registry.yarnpkg.com/sanitize-filename/-/sanitize-filename-1.6.3.tgz#755ebd752045931977e30b2025d340d7c9090378"
+  integrity sha512-y/52Mcy7aw3gRm7IrcGDFx/bCk4AhRh2eI9luHOQM86nZsqwiRkkq2GekHXBBD+SmPidc8i2PqtYZl+pWJ8Oeg==
+  dependencies:
+    truncate-utf8-bytes "^1.0.0"
+
 semver@^5.5.0:
   version "5.7.2"
   resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.2.tgz#48d55db737c3287cd4835e17fa13feace1c41ef8"
@@ -540,6 +547,13 @@ tar-stream@^2.1.4:
     inherits "^2.0.3"
     readable-stream "^3.1.1"
 
+truncate-utf8-bytes@^1.0.0:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/truncate-utf8-bytes/-/truncate-utf8-bytes-1.0.2.tgz#405923909592d56f78a5818434b0b78489ca5f2b"
+  integrity sha512-95Pu1QXQvruGEhv62XCMO3Mm90GscOCClvrIUwCM0PYOXK3kaF3l3sIHxx71ThJfcbM2O5Au6SO3AWCSEfW4mQ==
+  dependencies:
+    utf8-byte-length "^1.0.1"
+
 tunnel-agent@^0.6.0:
   version "0.6.0"
   resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.6.0.tgz#27a5dea06b36b04a0a9966774b290868f0fc40fd"
@@ -547,6 +561,11 @@ tunnel-agent@^0.6.0:
   dependencies:
     safe-buffer "^5.0.1"
 
+utf8-byte-length@^1.0.1:
+  version "1.0.5"
+  resolved "https://registry.yarnpkg.com/utf8-byte-length/-/utf8-byte-length-1.0.5.tgz#f9f63910d15536ee2b2d5dd4665389715eac5c1e"
+  integrity sha512-Xn0w3MtiQ6zoz2vFyUVruaCL53O/DwUvkEeOvj+uulMm0BkUGYWmBYVyElqZaSLhY6ZD0ulfU3aBra2aVT4xfA==
+
 util-deprecate@^1.0.1:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"

deps/outliner/deps.edn

@@ -6,7 +6,7 @@
 
   ;; Any other deps should be added here and to nbb.edn
   logseq/db             {:local/root "../db"}
-  logseq/graph-parser   {:local/root "../db"}
+  logseq/graph-parser   {:local/root "../graph-parser"}
   metosin/malli {:mvn/version "0.16.1"}}
  :aliases
  {:clj-kondo

deps/outliner/src/logseq/outliner/property.cljs

@@ -411,7 +411,9 @@
   (let [eid (->eid eid)
         block (d/entity @conn eid)
         property (d/entity @conn property-id)]
-    (validate-batch-deletion-of-property [block] property-id)
+    ;; Can skip for extends b/c below tx ensures it has a default value
+    (when-not (= :logseq.property.class/extends property-id)
+      (validate-batch-deletion-of-property [block] property-id))
     (when block
       (cond
         (= :logseq.property/empty-placeholder (:db/ident (get block property-id)))

deps/outliner/test/logseq/outliner/property_test.cljs

@@ -351,3 +351,20 @@
            #"Extends cycle"
            (outliner-property/set-block-property! conn (:db/id class3) :logseq.property.class/extends (:db/id class1)))
           "Extends cycle"))))
+
+(deftest delete-property-value!
+  (let [conn (db-test/create-conn-with-blocks
+              {:classes {:C1 {}
+                         :C2 {}
+                         :C3 {:build/class-extends [:C1 :C2]}}})]
+    (outliner-property/delete-property-value! conn :user.class/C3 :logseq.property.class/extends
+                                              (:db/id (d/entity @conn :user.class/C2)))
+    (is (= [:user.class/C1]
+           (:logseq.property.class/extends (db-test/readable-properties (d/entity @conn :user.class/C3))))
+        "Specific property value is deleted")
+
+    (outliner-property/delete-property-value! conn :user.class/C3 :logseq.property.class/extends
+                                              (:db/id (d/entity @conn :user.class/C1)))
+    (is (= [:logseq.class/Root]
+           (:logseq.property.class/extends (db-test/readable-properties (d/entity @conn :user.class/C3))))
+        "Extends property is restored back to Root")))

scripts/src/logseq/tasks/dev/db_and_file_graphs.clj

@@ -110,7 +110,8 @@
                          ;; Use file-entity-util and entity-util when in a single graph context
                          "ldb/whiteboard\\?" "ldb/journal\\?" "ldb/page\\?"]
         res (grep-many multi-graph-fns (into file-graph-paths db-graph-paths))]
-    (when-not (and (= 1 (:exit res)) (= "" (:out res)))
+    (when-not (or (and (= 1 (:exit res)) (= "" (:out res)))
+                  (and (zero? (:exit res)) (string/starts-with? (:out res) "src/main/mobile/components/app.cljs:")))
       (println "The following files should not have fns meant to be used in multi-graph contexts:")
       (println (:out res))
       (System/exit 1))))

src/main/electron/listener.cljs

@@ -130,7 +130,7 @@
                          method' (last ns-method)
                          args    (.-args data)
                          ret-fn! #(ipc/invoke (str :electron.server/sync! sync-id) %)
-                         app? (contains? #{"app" "editor"} ns')
+                         app? (contains? #{"app" "editor" "db"} ns')
                          ^js sdk1 (aget js/window.logseq "api")
                          ^js sdk2 (aget js/window.logseq "sdk")]
 
@@ -141,7 +141,7 @@
                            (throw (js/Error. (str "MethodNotExist: " method))))
                          (-> (p/promise (apply js-invoke methodTarget method' args))
                              (p/then #(ret-fn! %))
-                             (p/catch #(ret-fn! {:error %}))))
+                             (p/catch #(ret-fn! {:error (.-message %)}))))
                        (catch js/Error e
                          (ret-fn! {:error (.-message e)}))))))
 

src/main/frontend/components/block.cljs

@@ -695,7 +695,6 @@
 
 (rum/defcs ^:large-vars/cleanup-todo page-inner <
   (rum/local false ::mouse-down?)
-  (rum/local false ::hover?)
   "The inner div of page reference component
 
    page-name-in-block is the overridable name of the page (legacy)
@@ -707,8 +706,7 @@
     :or {with-parent? true}
     :as config}
    page-entity children label]
-  (let [*hover? (::hover? state)
-        *mouse-down? (::mouse-down? state)
+  (let [*mouse-down? (::mouse-down? state)
         tag? (:tag? config)
         page-name (when (:block/title page-entity)
                     (util/page-name-sanity-lc (:block/title page-entity)))
@@ -728,8 +726,6 @@
        :draggable true
        :on-drag-start (fn [e]
                         (editor-handler/block->data-transfer! page-name e true))
-       :on-mouse-over #(reset! *hover? true)
-       :on-mouse-leave #(reset! *hover? false)
        :on-pointer-down (fn [^js e]
                           (cond
                             (util/link? (.-target e))
@@ -833,26 +829,18 @@
   (let [*el-trigger (hooks/use-ref nil)]
     (hooks/use-effect!
      (fn []
-       (when-not (state/editing?)
-         (when (true? visible?)
-           (shui/popup-show!
-            (hooks/deref *el-trigger) render
-            {:root-props {:onOpenChange (fn [v] (set-visible! v))
-                          :modal false}
-             :content-props {:class "ls-preview-popup"
-                             :onInteractOutside (fn [^js e] (.preventDefault e))
-                             :onEscapeKeyDown (fn [^js e]
-                                                (when (state/editing?)
-                                                  (.preventDefault e)
-                                                  (some-> (hooks/deref *el-popup) (.focus))))}
-             :as-dropdown? false}))
-
-         (when (false? visible?)
-           (shui/popup-hide!)
-           (when (state/get-edit-block)
-             (state/clear-edit!)))
-         (hooks/set-ref! *timer nil)
-         (hooks/set-ref! *timer1 nil))
+       (when (true? visible?)
+         (shui/popup-show!
+          (hooks/deref *el-trigger) render
+          {:root-props {:onOpenChange (fn [v] (set-visible! v))
+                        :modal false}
+           :content-props {:class "ls-preview-popup"
+                           :onInteractOutside (fn [^js e] (.preventDefault e))
+                           :onEscapeKeyDown (fn [^js e]
+                                              (when (state/editing?)
+                                                (.preventDefault e)
+                                                (some-> (hooks/deref *el-popup) (.focus))))}
+           :as-dropdown? false}))
 
         ;; teardown
        (fn []
@@ -862,17 +850,17 @@
 
     [:span.preview-ref-link
      {:ref *el-trigger
-      :on-mouse-enter (fn [^js e]
-                        (when (= (some-> (.-target e) (.closest ".preview-ref-link"))
-                                 (hooks/deref *el-trigger))
-                          (let [timer (hooks/deref *timer)
-                                timer1 (hooks/deref *timer1)]
-                            (when-not timer
-                              (hooks/set-ref! *timer
-                                              (js/setTimeout #(set-visible! true) 1000)))
-                            (when timer1
-                              (js/clearTimeout timer1)
-                              (hooks/set-ref! *timer1 nil)))))
+      :on-mouse-move (fn [^js e]
+                       (when (= (some-> (.-target e) (.closest ".preview-ref-link"))
+                                (hooks/deref *el-trigger))
+                         (let [timer (hooks/deref *timer)
+                               timer1 (hooks/deref *timer1)]
+                           (when-not timer
+                             (hooks/set-ref! *timer
+                                             (js/setTimeout #(set-visible! true) 1000)))
+                           (when timer1
+                             (js/clearTimeout timer1)
+                             (hooks/set-ref! *timer1 nil)))))
       :on-mouse-leave (fn []
                         (let [timer (hooks/deref *timer)
                               timer1 (hooks/deref *timer1)]
@@ -941,8 +929,7 @@
      (if (boolean? in-popup?)
        (if (and (not (:preview? config))
                 (not in-popup?)
-                (or (not manual?) open?)
-                (not (state/editing?)))
+                (or (not manual?) open?))
          (popup-preview-impl children
                              {:visible? visible? :set-visible! set-visible!
                               :*timer *timer :*timer1 *timer1
@@ -1157,23 +1144,24 @@
                (excalidraw uuid-or-title (:block/uuid config))]
 
               :else
-              [:span.page-reference
-               {:data-ref (str uuid-or-title)}
-               (when brackets?
-                 [:span.text-gray-500.bracket page-ref/left-brackets])
-               (when (and (config/db-based-graph?) (ldb/class-instance? (db/entity :logseq.class/Task) block))
-                 [:div.inline-block
-                  {:style {:margin-right 1
-                           :margin-top -2
-                           :vertical-align "middle"}
-                   :on-pointer-down (fn [e]
-                                      (util/stop e))}
-                  (block-positioned-properties config block :block-left)])
-               (page-cp config' (if (uuid? uuid-or-title)
-                                  {:block/uuid uuid-or-title}
-                                  {:block/name uuid-or-title}))
-               (when brackets?
-                 [:span.text-gray-500.bracket page-ref/right-brackets])])))))))
+              (let [blank-title? (string/blank? (:block/title block))]
+                [:span.page-reference
+                 {:data-ref (str uuid-or-title)}
+                 (when (and brackets? (not blank-title?))
+                   [:span.text-gray-500.bracket page-ref/left-brackets])
+                 (when (and (config/db-based-graph?) (ldb/class-instance? (db/entity :logseq.class/Task) block))
+                   [:div.inline-block
+                    {:style {:margin-right 1
+                             :margin-top -2
+                             :vertical-align "middle"}
+                     :on-pointer-down (fn [e]
+                                        (util/stop e))}
+                    (block-positioned-properties config block :block-left)])
+                 (page-cp config' (if (uuid? uuid-or-title)
+                                    {:block/uuid uuid-or-title}
+                                    {:block/name uuid-or-title}))
+                 (when (and brackets? (not blank-title?))
+                   [:span.text-gray-500.bracket page-ref/right-brackets])]))))))))
 
 (defn- latex-environment-content
   [name option content]
@@ -2654,13 +2642,7 @@
         selection-blocks (state/get-selection-blocks)
         starting-block (state/get-selection-start-block-or-first)
         mobile-selection? (and (util/capacitor-new?) (seq selection-blocks))
-        block-dom-element (util/rec-get-node target "ls-block")
-        cursor-range (if (util/ios?)
-                       (:block/title block)
-                       (some-> block-dom-element
-                               (dom/by-class "block-content-inner")
-                               first
-                               util/caret-range))]
+        block-dom-element (util/rec-get-node target "ls-block")]
 
     (if mobile-selection?
       (let [ids (set (state/get-selection-block-ids))]
@@ -2721,7 +2703,13 @@
                                           (->> title
                                                (property-file/remove-built-in-properties-when-file-based
                                                 (state/get-current-repo) format)
-                                               (drawer/remove-logbook)))]
+                                               (drawer/remove-logbook)))
+                                cursor-range (if (util/ios?)
+                                               (:block/title block)
+                                               (some-> block-dom-element
+                                                       (dom/by-class "block-content-inner")
+                                                       first
+                                                       util/caret-range))]
                             (state/set-editing!
                              edit-input-id
                              content
@@ -2844,10 +2832,7 @@
                       (:block/tags block)
                       (remove (fn [t]
                                 (or (ldb/inline-tag? (:block/raw-title block) t)
-                                    (if (contains? t :logseq.property.class/hide-from-node)
-                                      (:logseq.property.class/hide-from-node t)
-                                      ;; Mobile app hides by default while everything else doesn't
-                                      (if (util/capacitor-new?) true false))
+                                    (:logseq.property.class/hide-from-node t)
                                     (contains? hidden-internal-tags (:db/ident t))
                                     (and (util/mobile?) (= (:db/ident t) :logseq.class/Task))))))
           popup-opts {:align :end

src/main/frontend/components/container.cljs

@@ -466,7 +466,8 @@
       [:div.wrap
        [:div.sidebar-header-container
         ;; sidebar graphs
-        (sidebar-graphs)
+        (when (not config/publishing?)
+         (sidebar-graphs))
 
         ;; sidebar sticky navigations
         (sidebar-navigations

src/main/frontend/components/header.cljs

@@ -169,7 +169,7 @@
                      :options {:on-click #(state/toggle-theme!)}
                      :icon (ui/icon "bulb")})
 
-                  (when-not login?
+                  (when-not (or config/publishing? login?)
                     {:title (t :login)
                      :options {:on-click #(state/pub-event! [:user/login])}
                      :icon (ui/icon "user")})

src/main/frontend/components/imports.cljs

@@ -351,26 +351,27 @@
 (defn- read-asset [file assets]
   (-> (.arrayBuffer (:file-object file))
       (p/then (fn [buffer]
-                (p/let [checksum (db-asset/<get-file-array-buffer-checksum buffer)]
+                (p/let [checksum (db-asset/<get-file-array-buffer-checksum buffer)
+                        byte-array (js/Uint8Array. buffer)]
                   (swap! assets assoc
                          (gp-exporter/asset-path->name (:path file))
                          {:size (.-size (:file-object file))
                           :checksum checksum
                           :type (db-asset/asset-path->type (:path file))
                           :path (:path file)
-                          ;; Save buffer to avoid reading asset twice
-                          ::array-buffer buffer}))))))
+                          ;; Save array to avoid reading asset twice
+                          ::byte-array byte-array})
+                  byte-array)))))
 
 (defn- copy-asset [repo repo-dir asset-m]
-  (-> (::array-buffer asset-m)
-      (p/then (fn [buffer]
-                (let [content (js/Uint8Array. buffer)
-                      assets-dir (path/path-join repo-dir common-config/local-assets-dir)]
+  (-> (::byte-array asset-m)
+      (p/then (fn [content]
+                (let [assets-dir (path/path-join repo-dir common-config/local-assets-dir)]
                   (p/do!
                    (fs/mkdir-if-not-exists assets-dir)
                    (if (:block/uuid asset-m)
                      (fs/write-plain-text-file! repo assets-dir (str (:block/uuid asset-m) "." (:type asset-m)) content {:skip-transact? true})
-                     (do
+                     (when-not (:pdf-annotation? asset-m)
                        (println "Copied asset" (pr-str (node-path/basename (:path asset-m)))
                                 "by its name since it was unused.")
                        (fs/write-plain-text-file! repo assets-dir (node-path/basename (:path asset-m)) content {:skip-transact? true})))))))))

src/main/frontend/components/repo.cljs

@@ -126,7 +126,9 @@
                                            (repo-handler/remove-repo! repo)
                                            (state/pub-event! [:graph/unlinked repo (state/get-current-repo)]))))))}
               "Delete local graph"))
-           (when (and db-based? root (not remote?))
+           (when (and db-based? root
+                      (not remote?)
+                      (= url (state/get-current-repo)))
              (shui/dropdown-menu-item
               {:key "logseq-sync"
                :class "use-logseq-sync-menu-item"
@@ -141,11 +143,14 @@
                                  (shui/popup-show! nil
                                                    (fn []
                                                      (rtc-indicator/uploading-logs))
-                                                   {:id :rtc-graph-upload-log})
-                                 (rtc-indicator/on-upload-finished-task
-                                  (fn []
-                                    (shui/popup-hide! :rtc-graph-upload-log)
-                                    (rtc-flows/trigger-rtc-start repo)))))))}
+                                                   {:id :rtc-graph-upload-log}))
+
+                               (rtc-indicator/on-upload-finished-task
+                                (fn []
+                                  (when (util/mobile?) (shui/popup-hide! :rtc-graph-upload-log))
+                                  (p/do!
+                                   (rtc-flows/trigger-rtc-start repo)
+                                   (rtc-handler/<get-remote-graphs)))))))}
               "Use Logseq sync (Beta testing)"))
            (when (and remote? (or (and db-based? manager?) (not db-based?)))
              (shui/dropdown-menu-item
@@ -165,9 +170,12 @@
                                                             (fn [graph-uuid _graph-schema-version]
                                                               (async-util/c->p (file-sync/<delete-graph graph-uuid))))]
                                         (state/set-state! [:file-sync/remote-graphs :loading] true)
+                                        (when (= (state/get-current-repo) repo)
+                                          (state/<invoke-db-worker :thread-api/rtc-stop))
                                         (p/do! (<delete-graph GraphUUID GraphSchemaVersion)
                                                (state/delete-remote-graph! repo)
-                                               (state/set-state! [:file-sync/remote-graphs :loading] false)))))))))}
+                                               (state/set-state! [:file-sync/remote-graphs :loading] false)
+                                               (rtc-handler/<get-remote-graphs)))))))))}
               "Delete from server")))))]]]))
 
 (rum/defc repos-cp < rum/reactive
@@ -228,9 +236,7 @@
             :on-click (fn []
                         (when-not (util/capacitor-new?)
                           (file-sync/load-session-graphs))
-                        (p/do!
-                         (rtc-handler/<get-remote-graphs)
-                         (repo-handler/refresh-repos!))))]]
+                        (rtc-handler/<get-remote-graphs)))]]
          (repos-inner remote-graphs)])]]))
 
 (defn- repos-dropdown-links [repos current-repo downloading-graph-id & {:as opts}]
@@ -464,13 +470,13 @@
                               (p/do
                                 (state/set-state! :rtc/uploading? true)
                                 (rtc-handler/<rtc-create-graph! repo)
-                                (state/set-state! :rtc/uploading? false)
-                                (rtc-flows/trigger-rtc-start repo))
+                                (rtc-flows/trigger-rtc-start repo)
+                                (rtc-handler/<get-remote-graphs))
                               (p/catch (fn [error]
-                                         (reset! *creating-db? false)
-                                         (state/set-state! :rtc/uploading? false)
-                                         (log/error :create-db-failed error)))))
-                           (reset! *creating-db? false)
+                                         (log/error :create-db-failed error)))
+                              (p/finally (fn []
+                                           (state/set-state! :rtc/uploading? false)
+                                           (reset! *creating-db? false)))))
                            (shui/dialog-close!))))))
         submit! (fn [^js e click?]
                   (when-let [value (and (or click? (= (gobj/get e "key") "Enter"))

src/main/frontend/components/rtc/indicator.cljs

@@ -5,6 +5,7 @@
             [frontend.config :as config]
             [frontend.db :as db]
             [frontend.flows :as flows]
+            [frontend.handler.db-based.rtc :as rtc-handler]
             [frontend.handler.db-based.rtc-flows :as rtc-flows]
             [frontend.state :as state]
             [frontend.ui :as ui]
@@ -138,7 +139,15 @@
                remote-tx (assoc :remote-tx remote-tx)
                rtc-state (assoc :rtc-state rtc-state))
              pprint/pprint
-             with-out-str)]])]))
+             with-out-str)]])
+     (when-not (= rtc-state :open)
+       [:div.mt-4
+        (shui/button {:variant :default
+                      :size :sm
+                      :on-click (fn []
+                                  (rtc-handler/<rtc-start! (state/get-current-repo)
+                                                           {:stop-before-start? true}))}
+                     "Start sync")])]))
 
 (rum/defc indicator
   []

src/main/frontend/config.cljs

@@ -424,7 +424,8 @@
 
     ;; nfs, browser-fs-access
     ;; Format: logseq_local_{dir-name}
-        (local-file-based-graph? repo-url)
+        (or (local-file-based-graph? repo-url)
+            (and publishing? (not db-based?)))
         (string/replace-first repo-url local-db-prefix "")
 
      ;; unit test

src/main/frontend/extensions/pdf/assets.cljs

@@ -25,6 +25,7 @@
             [frontend.util.ref :as ref]
             [logseq.common.config :as common-config]
             [logseq.common.path :as path]
+            [logseq.graph-parser.exporter :as gp-exporter]
             [logseq.publishing.db :as publish-db]
             [medley.core :as medley]
             [promesa.core :as p]
@@ -52,7 +53,7 @@
                       (some-> url (js/decodeURIComponent)
                               (get-in-repo-assets-full-filename)
                               (string/replace '"/" "_")))
-        filekey   (util/safe-sanitize-file-name
+        filekey   (gp-exporter/safe-sanitize-file-name
                    (subs filename' 0 (- (count filename') (inc (count ext-name)))))]
     (when-let [key (and (not (string/blank? filekey))
                         (if web-link?

src/main/frontend/handler/db_based/rtc.cljs

@@ -5,6 +5,7 @@
             [frontend.db :as db]
             [frontend.handler.db-based.rtc-flows :as rtc-flows]
             [frontend.handler.notification :as notification]
+            [frontend.handler.repo :as repo-handler]
             [frontend.handler.user :as user-handler]
             [frontend.state :as state]
             [frontend.util :as util]
@@ -145,7 +146,8 @@
                                   :GraphUUID (:graph-uuid graph)
                                   :rtc-graph? true})
                                (dissoc graph :graph-uuid :graph-name)))))]
-    (state/set-state! :rtc/graphs result)))
+    (state/set-state! :rtc/graphs result)
+    (repo-handler/refresh-repos!)))
 
 (defn <rtc-get-users-info
   []

src/main/frontend/handler/db_based/rtc_flows.cljs

@@ -1,6 +1,7 @@
 (ns frontend.handler.db-based.rtc-flows
   "Flows related to RTC"
   (:require [frontend.common.missionary :as c.m]
+            [frontend.common.thread-api :as thread-api :refer [def-thread-api]]
             [frontend.flows :as flows]
             [frontend.mobile.flows :as mobile-flows]
             [frontend.state :as state]
@@ -94,6 +95,10 @@ conditions:
   (assert (some? repo))
   (reset! *rtc-start-trigger repo))
 
+(def-thread-api :thread-api/rtc-start-request
+  [repo]
+  (trigger-rtc-start repo))
+
 (def ^:private document-visible&rtc-not-running-flow
   (m/ap
     (let [visibility (m/?< flows/document-visibility-state-flow)]

src/main/frontend/handler/editor.cljs

@@ -1475,7 +1475,10 @@
      (for [[_index ^js file] (map-indexed vector files)]
       ;; WARN file name maybe fully qualified path when paste file
        (p/let [file-name (node-path/basename (.-name file))
-               file-name-without-ext (db-asset/asset-name->title file-name)
+               file-name-without-ext* (db-asset/asset-name->title file-name)
+               file-name-without-ext (if (= file-name-without-ext* "image")
+                                       (date/get-date-time-string-2)
+                                       file-name-without-ext*)
                checksum (assets-handler/get-file-checksum file)
                existing-asset (db-async/<get-asset-with-checksum repo checksum)]
          (if existing-asset

src/main/frontend/handler/events.cljs

@@ -188,9 +188,10 @@
        (state/pub-event! [:mobile/post-init]))
      ;; FIXME: an ugly implementation for redirecting to page on new window is restored
      (repo-handler/graph-ready! repo)
-     (if db-based?
-       (export/auto-db-backup! repo {:backup-now? true})
-       (fs-watcher/load-graph-files! repo)))))
+     (when-not config/publishing?
+       (if db-based?
+         (export/auto-db-backup! repo {:backup-now? true})
+         (fs-watcher/load-graph-files! repo))))))
 
 (defmethod handle :instrument [[_ {:keys [type payload] :as opts}]]
   (when-not (empty? (dissoc opts :type :payload))
@@ -379,6 +380,11 @@
 (defmethod handle :rtc/log [[_ data]]
   (state/set-state! :rtc/log data))
 
+(defmethod handle :rtc/remote-graph-gone [_]
+  (p/do!
+   (notification/show! "This graph has been removed from Logseq Sync." :warning false)
+   (rtc-handler/<get-remote-graphs)))
+
 (defmethod handle :rtc/download-remote-graph [[_ graph-name graph-uuid graph-schema-version]]
   (assert (= (:major (db-schema/parse-schema-version db-schema/version))
              (:major (db-schema/parse-schema-version graph-schema-version)))
@@ -395,6 +401,7 @@
           (indicator/downloading-logs)])
        {:id :download-rtc-graph}))
     (rtc-handler/<rtc-download-graph! graph-name graph-uuid graph-schema-version 60000)
+    (rtc-handler/<get-remote-graphs)
     (when (util/mobile?)
       (shui/popup-hide! :download-rtc-graph)))
    (p/catch (fn [e]

src/main/frontend/handler/file_based/events.cljs

@@ -203,7 +203,8 @@
    {:id :page-histories :label "modal-page-histories"}))
 
 (defmethod events/handle :file-sync/maybe-onboarding-show [[_ type]]
-  (file-sync/maybe-onboarding-show type))
+  (when-not util/web-platform?
+    (file-sync/maybe-onboarding-show type)))
 
 (defmethod events/handle :file-sync/storage-exceed-limit [[_]]
   (notification/show! "file sync storage exceed limit" :warning false)

src/main/frontend/handler/worker.cljs

@@ -61,6 +61,9 @@
 (defmethod handle :notify-existing-file  [_ _worker data]
   (state/pub-event! [:graph/notify-existing-file data]))
 
+(defmethod handle :remote-graph-gone []
+  (state/pub-event! [:rtc/remote-graph-gone]))
+
 (defmethod handle :default [_ _worker data]
   (prn :debug "Worker data not handled: " data))
 

src/main/frontend/log.cljs

@@ -10,4 +10,4 @@
 
 (if config/dev?
   (log/set-levels {:glogi/root :info})
-  (log/set-levels {:glogi/root :warn}))
+  (log/set-levels {:glogi/root :info}))

src/main/frontend/worker/db_worker.cljs

@@ -875,6 +875,7 @@
                       (into {})
                       bean/->js)]
     (glogi-console/install!)
+    (log/set-levels {:glogi/root :info})
     (check-worker-scope!)
     (outliner-register-op-handlers!)
     (<ratelimit-file-writes!)

src/main/frontend/worker/rtc/asset.cljs

@@ -50,49 +50,35 @@
                                        repo (str block-uuid) asset-type))))
 
 (defn- remote-block-ops=>remote-asset-ops
-  [db-before remove-ops]
-  (keep
-   (fn [remove-op]
-     (let [block-uuid (:block-uuid remove-op)]
-       (when-let [ent (d/entity db-before [:block/uuid block-uuid])]
-         (when-let [asset-type (:logseq.property.asset/type ent)]
-           {:op :remove-asset
-            :block/uuid block-uuid
-            :logseq.property.asset/type asset-type}))))
-   remove-ops))
+  [db-before db-after remove-ops update-ops]
+  (concat
+   (keep
+    (fn [remove-op]
+      (let [block-uuid (:block-uuid remove-op)]
+        (when-let [ent (d/entity db-before [:block/uuid block-uuid])]
+          (when-let [asset-type (:logseq.property.asset/type ent)]
+            {:op :remove-asset
+             :block/uuid block-uuid
+             :logseq.property.asset/type asset-type}))))
+    remove-ops)
+   (keep
+    (fn [update-op]
+      (let [block-uuid (:self update-op)]
+        (when-let [ent (d/entity db-after [:block/uuid block-uuid])]
+          (let [remote-metadata (:logseq.property.asset/remote-metadata ent)
+                checksum (:logseq.property.asset/checksum ent)
+                asset-type (:logseq.property.asset/type ent)]
+            (when (and remote-metadata checksum asset-type)
+              {:op :update-asset
+               :block/uuid block-uuid})))))
+    update-ops)))
 
 (defn emit-remote-asset-updates-from-block-ops
-  [db-before remove-ops]
+  [db-before db-after remove-ops update-ops]
   (when-let [asset-update-ops
-             (not-empty (remote-block-ops=>remote-asset-ops db-before remove-ops))]
+             (not-empty (remote-block-ops=>remote-asset-ops db-before db-after remove-ops update-ops))]
     (reset! *remote-asset-updates asset-update-ops)))
 
-(defn new-task--emit-remote-asset-updates-from-push-asset-upload-updates
-  [repo db push-asset-upload-updates-message]
-  (m/sp
-    (let [{:keys [uploaded-assets]} push-asset-upload-updates-message]
-      (when-let [asset-update-ops
-                 (->> uploaded-assets
-                      (map
-                       (fn [[asset-uuid remote-metadata]]
-                         (m/sp
-                           (let [ent (d/entity db [:block/uuid asset-uuid])
-                                 asset-type (:logseq.property.asset/type ent)
-                                 local-checksum (:logseq.property.asset/checksum ent)
-                                 remote-checksum (get remote-metadata "checksum")]
-                             (when (or (and local-checksum remote-checksum
-                                            (not= local-checksum remote-checksum))
-                                       (and asset-type
-                                            (nil? (m/? (new-task--get-asset-file-metadata
-                                                        repo asset-uuid asset-type)))))
-                               {:op :update-asset
-                                :block/uuid asset-uuid})))))
-                      (apply m/join vector)
-                      m/?
-                      (remove nil?)
-                      not-empty)]
-        (reset! *remote-asset-updates asset-update-ops)))))
-
 (defn- create-mixed-flow
   "Return a flow that emits different events:
   - `:local-update-check`: event to notify check if there're some new local-updates on assets
@@ -228,16 +214,25 @@
                   asset-update-ops)
             asset-uuid->asset-type (into {}
                                          (keep (fn [asset-uuid]
-                                                 (when-let [tp (:logseq.property.asset/type
-                                                                (d/entity @conn [:block/uuid asset-uuid]))]
-                                                   [asset-uuid tp])))
+                                                 (when-let [ent (d/entity @conn [:block/uuid asset-uuid])]
+                                                   (let [asset-type (:logseq.property.asset/type ent)]
+                                                     [asset-uuid asset-type]))))
                                          update-asset-uuids)
             asset-uuid->url
-            (when (seq asset-uuid->asset-type)
+            (when-let [asset-uuids
+                       (->> asset-uuid->asset-type
+                            (map
+                             (fn [[asset-uuid asset-type]]
+                               (m/sp
+                                 (when (nil? (m/? (new-task--get-asset-file-metadata repo asset-uuid asset-type)))
+                                   asset-uuid))))
+                            (apply m/join vector)
+                            m/?
+                            (remove nil?))]
               (->> (m/? (ws-util/send&recv get-ws-create-task
                                            {:action "get-assets-download-urls"
                                             :graph-uuid graph-uuid
-                                            :asset-uuids (keys asset-uuid->asset-type)}))
+                                            :asset-uuids asset-uuids}))
                    :asset-uuid->url))]
         (doseq [[asset-uuid asset-type] remove-asset-uuid->asset-type]
           (c.m/<? (worker-state/<invoke-main-thread :thread-api/unlink-asset

src/main/frontend/worker/rtc/client_op.cljs

@@ -92,7 +92,6 @@
   [repo graph-uuid]
   {:pre [(some? graph-uuid)]}
   (when-let [conn (worker-state/get-client-ops-conn repo)]
-    (assert (nil? (first (d/datoms @conn :avet :graph-uuid))))
     (d/transact! conn [[:db/add "e" :graph-uuid graph-uuid]])))
 
 (defn get-graph-uuid
@@ -470,13 +469,3 @@
       (m/ap
         (let [_ (m/?> (c.m/throttle 100 db-updated-flow))]
           (datom-count-fn @conn))))))
-
-(defn reset-client-op-conn
-  [repo]
-  (when-let [conn (worker-state/get-client-ops-conn repo)]
-    (let [tx-data (->> (concat (d/datoms @conn :avet :graph-uuid)
-                               (d/datoms @conn :avet :local-tx)
-                               (d/datoms @conn :avet :aes-key-jwk)
-                               (d/datoms @conn :avet :block/uuid))
-                       (map (fn [datom] [:db/retractEntity (:e datom)])))]
-      (d/transact! conn tx-data))))

src/main/frontend/worker/rtc/core.cljs

@@ -10,6 +10,7 @@
             [frontend.worker.rtc.branch-graph :as r.branch-graph]
             [frontend.worker.rtc.client :as r.client]
             [frontend.worker.rtc.client-op :as client-op]
+            [frontend.worker.rtc.db :as rtc-db]
             [frontend.worker.rtc.exception :as r.ex]
             [frontend.worker.rtc.full-upload-download-graph :as r.upload-download]
             [frontend.worker.rtc.log-and-state :as rtc-log-and-state]
@@ -40,7 +41,7 @@
   and filter messages with :req-id=
   - `push-updates`
   - `online-users-updated`.
-  - `push-asset-upload-updates`"
+  - `push-asset-block-updates`"
   [get-ws-create-task]
   (m/ap
     (loop []
@@ -51,7 +52,7 @@
                                  (contains?
                                   #{"online-users-updated"
                                     "push-updates"
-                                    "push-asset-upload-updates"}
+                                    "push-asset-block-updates"}
                                   (:req-id data))))
                        (ws/recv-flow ws)))
                 (catch js/CloseEvent _
@@ -122,7 +123,7 @@
 (defn- create-mixed-flow
   "Return a flow that emits all kinds of events:
   `:remote-update`: remote-updates data from server
-  `:remote-asset-update`: remote asset-updates from server
+  `:remote-asset-block-update`: remote asset-updates from server
   `:local-update-check`: event to notify to check if there're some new local-updates, then push to remote.
   `:online-users-updated`: online users info updated
   `:pull-remote-updates`: pull remote updates
@@ -133,7 +134,7 @@
                                     (case (:req-id data)
                                       "push-updates" {:type :remote-update :value data}
                                       "online-users-updated" {:type :online-users-updated :value data}
-                                      "push-asset-upload-updates" {:type :remote-asset-update :value data})))
+                                      "push-asset-block-updates" {:type :remote-asset-block-update :value data})))
                              (get-remote-updates get-ws-create-task))
         local-updates-check-flow (m/eduction
                                   (map (fn [data] {:type :local-update-check :value data}))
@@ -246,6 +247,7 @@
       started-dfv
       (m/sp
         (try
+          (log/info :rtc :loop-starting)
           ;; init run to open a ws
           (m/? get-ws-create-task)
           (started-dfv true)
@@ -257,15 +259,12 @@
           (->>
            (let [event (m/?> mixed-flow)]
              (case (:type event)
-               :remote-update
+               (:remote-update :remote-asset-block-update)
                (try (r.remote-update/apply-remote-update graph-uuid repo conn date-formatter event add-log-fn)
                     (catch :default e
                       (when (= ::r.remote-update/need-pull-remote-data (:type (ex-data e)))
                         (m/? (r.client/new-task--pull-remote-data
                               repo conn graph-uuid major-schema-version date-formatter get-ws-create-task add-log-fn)))))
-               :remote-asset-update
-               (m/? (r.asset/new-task--emit-remote-asset-updates-from-push-asset-upload-updates
-                     repo @conn (:value event)))
 
                :local-update-check
                (m/? (r.client/new-task--push-local-ops
@@ -368,10 +367,15 @@
                          rtc-loop-task
                          :fail (fn [e]
                                  (reset! *last-stop-exception e)
-                                 (log/info :rtc-loop-task e)))
+                                 (log/info :rtc-loop-task e)
+                                 (when (= :rtc.exception/ws-timeout (some-> e ex-data :type))
+                                   ;; if fail reason is websocket-timeout, try to restart rtc
+                                   (worker-state/<invoke-main-thread :thread-api/rtc-start-request repo))))
               start-ex (m/? onstarted-task)]
           (if (instance? ExceptionInfo start-ex)
-            start-ex
+            (do
+              (canceler)
+              start-ex)
             (do (reset! *rtc-loop-metadata {:repo repo
                                             :graph-uuid graph-uuid
                                             :local-graph-schema-version schema-version
@@ -443,7 +447,14 @@
                                     {:action "delete-graph"
                                      :graph-uuid graph-uuid
                                      :schema-version (str schema-version)}))]
-        (when ex-data (log/info ::delete-graph-failed {:graph-uuid graph-uuid :ex-data ex-data}))
+        (if ex-data
+          (log/info ::delete-graph-failed {:graph-uuid graph-uuid :ex-data ex-data})
+          ;; Clean up rtc data in existing dbs so that the graph can be uploaded again
+          (when-let [repo (worker-state/get-current-repo)]
+            (when-let [conn (worker-state/get-datascript-conn repo)]
+              (let [graph-id (ldb/get-graph-rtc-uuid @conn)]
+                (when (= (str graph-id) (str graph-uuid))
+                  (rtc-db/remove-rtc-data-in-conn! repo))))))
         (boolean (nil? ex-data))))))
 
 (defn new-task--get-users-info

src/main/frontend/worker/rtc/db.cljs

@@ -0,0 +1,26 @@
+(ns frontend.worker.rtc.db
+  "rtc db ops"
+  (:require [datascript.core :as d]
+            [frontend.worker.state :as worker-state]))
+
+(defn remove-rtc-data-from-local-db!
+  [repo]
+  (when-let [conn (worker-state/get-datascript-conn repo)]
+    (d/transact! conn [[:db/retractEntity :logseq.kv/graph-uuid]
+                       [:db/retractEntity :logseq.kv/graph-local-tx]
+                       [:db/retractEntity :logseq.kv/remote-schema-version]])))
+
+(defn reset-client-op-conn
+  [repo]
+  (when-let [conn (worker-state/get-client-ops-conn repo)]
+    (let [tx-data (->> (concat (d/datoms @conn :avet :graph-uuid)
+                               (d/datoms @conn :avet :local-tx)
+                               (d/datoms @conn :avet :aes-key-jwk)
+                               (d/datoms @conn :avet :block/uuid))
+                       (map (fn [datom] [:db/retractEntity (:e datom)])))]
+      (d/transact! conn tx-data))))
+
+(defn remove-rtc-data-in-conn!
+  [repo]
+  (remove-rtc-data-from-local-db! repo)
+  (reset-client-op-conn repo))

src/main/frontend/worker/rtc/full_upload_download_graph.cljs

@@ -12,6 +12,7 @@
             [frontend.worker.db-metadata :as worker-db-metadata]
             [frontend.worker.rtc.client-op :as client-op]
             [frontend.worker.rtc.const :as rtc-const]
+            [frontend.worker.rtc.db :as rtc-db]
             [frontend.worker.rtc.log-and-state :as rtc-log-and-state]
             [frontend.worker.rtc.ws-util :as ws-util]
             [frontend.worker.shared-service :as shared-service]
@@ -120,14 +121,6 @@
                   (:db/ident block) (update :db/ident ldb/read-transit-str)
                   (:block/order block) (update :block/order ldb/read-transit-str)))))))
 
-(defn- remove-rtc-data-in-conn!
-  [repo]
-  (client-op/reset-client-op-conn repo)
-  (when-let [conn (worker-state/get-datascript-conn repo)]
-    (d/transact! conn [[:db/retractEntity :logseq.kv/graph-uuid]
-                       [:db/retractEntity :logseq.kv/graph-local-tx]
-                       [:db/retractEntity :logseq.kv/remote-schema-version]])))
-
 (defn new-task--upload-graph
   [get-ws-create-task repo conn remote-graph-name major-schema-version]
   (m/sp
@@ -477,7 +470,7 @@
   (m/sp
     (rtc-log-and-state/rtc-log :rtc.log/branch-graph {:sub-type :fetching-presigned-put-url
                                                       :message "fetching presigned put-url"})
-    (remove-rtc-data-in-conn! repo)
+    (rtc-db/remove-rtc-data-in-conn! repo)
     (let [[{:keys [url key]} all-blocks-str]
           (m/?
            (m/join

src/main/frontend/worker/rtc/malli_schema.cljs

@@ -196,6 +196,7 @@
        [:t :int]
        [:max-remote-schema-version {:optional true} :string]]]
      ["apply-ops" apply-ops-response-schema]
+     ["push-asset-block-updates" apply-ops-response-schema]
      ["branch-graph"
       [:map
        [:graph-uuid :uuid]

src/main/frontend/worker/rtc/remote_update.cljs

@@ -610,7 +610,8 @@ so need to pull earlier remote-data from websocket."})
           (worker-util/profile :apply-remote-remove-ops (apply-remote-remove-ops repo conn date-formatter remove-ops))
           ;; wait all remote-ops transacted into db,
           ;; then start to check any asset-updates in remote
-          (r.asset/emit-remote-asset-updates-from-block-ops db-before remove-ops)
+          (let [db-after @conn]
+            (r.asset/emit-remote-asset-updates-from-block-ops db-before db-after remove-ops update-ops))
           (js/console.groupEnd)
 
           (client-op/update-local-tx repo remote-t)

src/main/frontend/worker/rtc/ws_util.cljs

@@ -1,16 +1,21 @@
 (ns frontend.worker.rtc.ws-util
   "Add RTC related logic to the function based on ws."
   (:require [cljs-http-missionary.client :as http]
+            [frontend.worker.rtc.db :as rtc-db]
             [frontend.worker.rtc.exception :as r.ex]
             [frontend.worker.rtc.malli-schema :as rtc-schema]
             [frontend.worker.rtc.ws :as ws]
             [frontend.worker.state :as worker-state]
+            [frontend.worker.util :as worker-util]
             [goog.string :as gstring]
             [logseq.graph-parser.utf8 :as utf8]
             [missionary.core :as m]))
 
 (defn- handle-remote-ex
   [resp]
+  (when (= :graph-not-exist (:type (:ex-data resp)))
+    (rtc-db/remove-rtc-data-in-conn! (worker-state/get-current-repo))
+    (worker-util/post-message :remote-graph-gone []))
   (if-let [e ({:graph-not-exist r.ex/ex-remote-graph-not-exist
                :graph-not-ready r.ex/ex-remote-graph-not-ready
                :bad-request-body r.ex/ex-bad-request-body

src/main/logseq/api.cljs

@@ -965,7 +965,7 @@
             _ (db-async/<get-block (state/get-current-repo) block-uuid :children? false)
             db? (config/db-based-graph? (state/get-current-repo))
             key-ns? (and (keyword? key) (namespace key))
-            key (if key-ns? key (-> (if (keyword? key) (name key) key) (util/safe-lower-case)))
+            key (if key-ns? key (if (keyword? key) (name key) key))
             key (if (and db? (not key-ns?))
                   (api-block/get-db-ident-for-user-property-name
                     key (api-block/resolve-property-prefix-for-db this))
@@ -982,7 +982,7 @@
       (when-let [properties (some-> block-uuid (db-model/get-block-by-uuid) (:block/properties))]
         (when (seq properties)
           (let [key (api-block/sanitize-user-property-name key)
-                property-name (-> (if (keyword? key) (name key) key) (util/safe-lower-case))
+                property-name (if (keyword? key) (name key) key)
                 ident (api-block/get-db-ident-for-user-property-name
                         property-name (api-block/resolve-property-prefix-for-db this))
                 property-value (or (get properties key)
@@ -1106,7 +1106,7 @@
     (p/let [result (query-dsl/query repo query-string
                                     {:disable-reactive? true
                                      :return-promise? true})]
-      (bean/->js (sdk-utils/normalize-keyword-for-json (flatten @result))))))
+      (bean/->js (sdk-utils/normalize-keyword-for-json (flatten result))))))
 
 (defn ^:export datascript_query
   [query & inputs]

src/main/logseq/api/block.cljs

@@ -34,7 +34,9 @@
     (-> k (string/trim)
       (string/replace " " "")
       (string/replace #"^[:_\s]+" "")
-      (string/lower-case))
+      (#(cond-> %
+          (not (string/includes? % "/"))
+          (string/lower-case))))
     k))
 
 (defn resolve-property-prefix-for-db

src/main/mobile/components/app.cljs

@@ -4,11 +4,13 @@
             [clojure.string :as string]
             [frontend.components.journal :as journal]
             [frontend.components.rtc.indicator :as rtc-indicator]
+            [frontend.config :as config]
             [frontend.date :as date]
             [frontend.db :as db]
             [frontend.db.conn :as db-conn]
             [frontend.handler.editor :as editor-handler]
             [frontend.handler.page :as page-handler]
+            [frontend.handler.repo :as repo-handler]
             [frontend.handler.user :as user-handler]
             [frontend.mobile.util :as mobile-util]
             [frontend.rum :as frum]
@@ -34,10 +36,17 @@
             [promesa.core :as p]
             [rum.core :as rum]))
 
-(rum/defc app-graphs-select
+(rum/defc app-graphs-select < rum/reactive
   []
   (let [current-repo (state/get-current-repo)
-        graphs (state/get-repos)
+        graphs (->> (state/sub [:me :repos])
+                    (util/distinct-by :url))
+        remote-graphs (state/sub :rtc/graphs)
+        graphs (->>
+                (if (seq remote-graphs)
+                  (repo-handler/combine-local-&-remote-graphs graphs remote-graphs)
+                  graphs)
+                (filter (fn [item] (config/db-based-graph? (:url item)))))
         short-repo-name (if current-repo
                           (db-conn/get-short-repo-name current-repo)
                           "Select a Graph")]
@@ -48,9 +57,11 @@
        :class "border-none w-full rounded-lg"
        :on-click (fn []
                    (let [buttons (concat
-                                  (for [repo graphs]
-                                    {:text (some-> (:url repo) (string/replace #"^logseq_db_" ""))
-                                     :role (:url repo)})
+                                  (->>
+                                   (for [repo graphs]
+                                     {:text (some-> (:url repo) (string/replace #"^logseq_db_" ""))
+                                      :role (:url repo)})
+                                   (remove (fn [{:keys [text]}] (string/blank? text))))
                                   [{:text "Add new graph"
                                     :role "add-new-graph"}])]
                      (ui-component/open-modal! "Switch graph"