Merge branch 'master' into refactor/page-parent

deps/db/.carve/config.edn

@@ -9,6 +9,7 @@
                   logseq.db.common.order
                   logseq.db.sqlite.create-graph
                   logseq.db.frontend.malli-schema
+                  logseq.db.frontend.asset
                   ;; Some fns are used by frontend but not worth moving over yet
                   logseq.db.frontend.schema
                   logseq.db.frontend.validate

deps/db/src/logseq/db/common/entity_plus.cljc

@@ -22,7 +22,7 @@
     :block/pre-block? :block/scheduled :block/deadline :block/type :block/name :block/marker
 
     :block.temp/ast-title
-    :block.temp/fully-loaded? :block.temp/has-children? :block.temp/ast-body
+    :block.temp/load-status :block.temp/has-children? :block.temp/ast-body
 
     :db/valueType :db/cardinality :db/ident :db/index
 

deps/db/src/logseq/db/common/initial_data.cljs

@@ -1,7 +1,6 @@
 (ns logseq.db.common.initial-data
   "Provides db helper fns for graph initialization and lazy loading entities"
-  (:require [clojure.set :as set]
-            [clojure.string :as string]
+  (:require [clojure.string :as string]
             [datascript.core :as d]
             [datascript.impl.entity :as de]
             [logseq.common.config :as common-config]
@@ -87,10 +86,6 @@
       (update block :block/link (fn [link] (d/pull db '[*] (:db/id link))))
       block)))
 
-(defn- mark-block-fully-loaded
-  [b]
-  (assoc b :block.temp/fully-loaded? true))
-
 (comment
   (defn- property-without-db-attrs
     [property]
@@ -133,27 +128,30 @@
 
 (defn get-block-children-ids
   "Returns children UUIDs"
-  [db block-uuid]
+  [db block-uuid & {:keys [include-collapsed-children?]
+                    :or {include-collapsed-children? true}}]
   (when-let [eid (:db/id (d/entity db [:block/uuid block-uuid]))]
-    (let [seen   (volatile! [])]
-      (loop [steps          100      ;check result every 100 steps
-             eids-to-expand [eid]]
+    (let [seen (volatile! #{})]
+      (loop [eids-to-expand [eid]]
         (when (seq eids-to-expand)
-          (let [eids-to-expand*
-                (mapcat (fn [eid] (map first (d/datoms db :avet :block/parent eid))) eids-to-expand)
-                uuids-to-add (remove nil? (map #(:block/uuid (d/entity db %)) eids-to-expand*))]
-            (when (and (zero? steps)
-                       (seq (set/intersection (set @seen) (set uuids-to-add))))
-              (throw (ex-info "bad outliner data, need to re-index to fix"
-                              {:seen @seen :eids-to-expand eids-to-expand})))
+          (let [children
+                (mapcat (fn [eid]
+                          (let [e (d/entity db eid)]
+                            (when (or include-collapsed-children?
+                                      (not (:block/collapsed? e))
+                                      (common-entity-util/page? e))
+
+                              (:block/_parent e)))) eids-to-expand)
+                uuids-to-add (keep :block/uuid children)]
             (vswap! seen (partial apply conj) uuids-to-add)
-            (recur (if (zero? steps) 100 (dec steps)) eids-to-expand*))))
+            (recur (keep :db/id children)))))
       @seen)))
 
 (defn get-block-children
   "Including nested children."
-  [db block-uuid]
-  (let [ids (get-block-children-ids db block-uuid)]
+  {:arglists '([db block-uuid & {:keys [include-collapsed-children?]}])}
+  [db block-uuid & {:as opts}]
+  (let [ids (get-block-children-ids db block-uuid opts)]
     (when (seq ids)
       (map (fn [id] (d/entity db [:block/uuid id])) ids))))
 
@@ -164,10 +162,36 @@
     m))
 
 (defn- entity->map
-  [entity]
-  (-> (into {} entity)
-      (with-raw-title entity)
-      (assoc :db/id (:db/id entity))))
+  [entity & {:keys [level]
+             :or {level 0}}]
+  (let [opts {:level (inc level)}
+        f (if (> level 0)
+            identity
+            (fn [e]
+              (keep (fn [[k v]]
+                      (when-not (contains? #{:block/path-refs} k)
+                        (let [v' (cond
+                                   (= k :block/parent)
+                                   (:db/id v)
+                                   (= k :block/tags)
+                                   (set (map :db/id v))
+                                   (= k :logseq.property/created-by-ref)
+                                   (:db/id v)
+                                   (= k :block/refs)
+                                   (map #(select-keys % [:db/id :block/uuid :block/title]) v)
+                                   (de/entity? v)
+                                   (entity->map v opts)
+                                   (and (coll? v) (every? de/entity? v))
+                                   (map #(entity->map % opts) v)
+                                   :else
+                                   v)]
+                          [k v'])))
+                    e)))
+        m (->> (f entity)
+               (into {}))]
+    (-> m
+        (with-raw-title entity)
+        (assoc :db/id (:db/id entity)))))
 
 (defn hidden-ref?
   "Whether ref-block (for block with the `id`) should be hidden."
@@ -204,7 +228,8 @@
    0))
 
 (defn ^:large-vars/cleanup-todo get-block-and-children
-  [db id-or-page-name {:keys [children? children-only? nested-children? properties children-props]}]
+  [db id-or-page-name {:keys [children? properties include-collapsed-children?]
+                       :or {include-collapsed-children? false}}]
   (let [block (let [eid (cond (uuid? id-or-page-name)
                               [:block/uuid id-or-page-name]
                               (integer? id-or-page-name)
@@ -217,64 +242,29 @@
                   (d/entity db (get-first-page-by-name db (name id-or-page-name)))
                   :else
                   nil))
-        block-refs-count? (some #{:block.temp/refs-count} properties)
-        whiteboard? (common-entity-util/whiteboard? block)]
+        block-refs-count? (some #{:block.temp/refs-count} properties)]
     (when block
-      (let [children (when (or children? children-only?)
-                       (let [page? (common-entity-util/page? block)
-                             children (->>
-                                       (cond
-                                         (and nested-children? (not page?))
-                                         (get-block-children db (:block/uuid block))
-                                         nested-children?
-                                         (:block/_page block)
-                                         :else
-                                         (let [short-page? (when page?
-                                                             (<= (count (:block/_page block)) 100))]
-                                           (if short-page?
-                                             (:block/_page block)
-                                             (:block/_parent block))))
-                                       (remove (fn [e] (or (:logseq.property/created-from-property e)
-                                                           (:block/closed-value-property e)))))
-                             children-props (if whiteboard?
-                                              '[*]
-                                              (or children-props
-                                                  [:db/id :block/uuid :block/parent :block/order :block/collapsed? :block/title
-                                                   ;; pre-loading feature-related properties to avoid UI refreshing
-                                                   :logseq.property/status :logseq.property.node/display-type]))]
+      (let [children (when children?
+                       (let [children (let [children (get-block-children db (:block/uuid block) {:include-collapsed-children? include-collapsed-children?})
+                                            children' (if (>= (count children) 100)
+                                                        (:block/_parent block)
+                                                        children)]
+                                        (->> children'
+                                             (remove (fn [e] (:block/closed-value-property e)))))]
                          (map
                           (fn [block]
-                            (if (= children-props '[*])
-                              (entity->map block)
-                              (-> (select-keys block children-props)
-                                  (with-raw-title block)
-                                  (assoc :block.temp/has-children? (some? (:block/_parent block))))))
-                          children)))]
-        (if children-only?
-          {:children children}
-          (let [block' (if (seq properties)
-                         (-> (select-keys block properties)
-                             (with-raw-title block)
-                             (assoc :db/id (:db/id block)))
-                         (entity->map block))
-                block' (cond->
-                        (mark-block-fully-loaded block')
-                         true
-                         (update-vals (fn [v]
-                                        (cond
-                                          (de/entity? v)
-                                          (entity->map v)
-                                          (and (coll? v) (every? de/entity? v))
-                                          (map entity->map v)
-
-                                          :else
-                                          v)))
-                         block-refs-count?
-                         (assoc :block.temp/refs-count (get-block-refs-count db (:db/id block))))]
-            (cond->
-             {:block block'}
-              children?
-              (assoc :children children))))))))
+                            (-> block
+                                (assoc :block.temp/has-children? (some? (:block/_parent block)))
+                                (dissoc :block/tx-id :block/created-at :block/updated-at)
+                                entity->map))
+                          children)))
+            block' (cond-> (entity->map block)
+                     block-refs-count?
+                     (assoc :block.temp/refs-count (get-block-refs-count db (:db/id block))))]
+        (cond->
+         {:block block'}
+          children?
+          (assoc :children children))))))
 
 (defn get-latest-journals
   [db]

deps/db/src/logseq/db/frontend/asset.cljs

@@ -0,0 +1,27 @@
+(ns logseq.db.frontend.asset
+  "Asset fns used in node and browser contexts"
+  (:require ["path" :as node-path]
+            [clojure.string :as string]))
+
+(defn- decode-digest
+  [^js/Uint8Array digest]
+  (.. (js/Array.from digest)
+      (map (fn [s] (.. s (toString 16) (padStart 2 "0"))))
+      (join "")))
+
+(defn <get-file-array-buffer-checksum
+  "Given a file's ArrayBuffer, returns its checksum in a promise"
+  [file-array-buffer]
+  (-> (js/crypto.subtle.digest "SHA-256" file-array-buffer)
+      (.then (fn [dig] (js/Uint8Array. dig)))
+      (.then decode-digest)))
+
+(defn asset-path->type
+  "Create asset type given asset path"
+  [path]
+  (string/lower-case (.substr (node-path/extname path) 1)))
+
+(defn asset-name->title
+  "Create asset title given asset path's basename"
+  [path-basename]
+  (.-name (node-path/parse path-basename)))

deps/db/src/logseq/db/frontend/malli_schema.cljs

@@ -40,6 +40,11 @@
                             {:error/message "should be a valid user property namespace"}
                             user-property?]])
 
+(def plugin-property-ident
+  [:and :qualified-keyword [:fn
+                            {:error/message "should be a valid plugin property namespace"}
+                            db-property/plugin-property?]])
+
 (def logseq-ident-namespaces
   "Set of all namespaces Logseq uses for :db/ident except for
   db-attribute-ident. It's important to grow this list purposefully and have it
@@ -315,19 +320,40 @@
   (vec
    (concat
     [:map
-     ;; class-ident allows for a class to be used as a property
-     [:db/ident [:or user-property-ident class-ident]]
+     [:db/ident user-property-ident]
      [:logseq.property/type (apply vector :enum db-property-type/user-built-in-property-types)]]
     property-common-schema-attrs
     property-attrs
     page-attrs
     page-or-block-attrs)))
 
+(def plugin-property
+  (vec
+   (concat
+    [:map
+     [:db/ident plugin-property-ident]
+     [:logseq.property/type (apply vector :enum (conj db-property-type/user-built-in-property-types :string))]]
+    property-common-schema-attrs
+    property-attrs
+    page-attrs
+    page-or-block-attrs)))
+
 (def property-page
   [:multi {:dispatch (fn [m]
-                       (or (some->> (:db/ident m) db-property/logseq-property?)
-                           (contains? db-property/db-attribute-properties (:db/ident m))))}
-   [true internal-property]
+                       (let [ident (:db/ident m)]
+
+                         (cond
+                           (or (some->> ident db-property/logseq-property?)
+                               (contains? db-property/db-attribute-properties (:db/ident m)))
+                           :internal
+
+                           (some->> ident db-property/plugin-property?)
+                           :plugin
+
+                           :else
+                           :user)))}
+   [:internal internal-property]
+   [:plugin plugin-property]
    [:malli.core/default user-property]])
 
 (def hidden-page

deps/db/src/logseq/db/frontend/property.cljs

@@ -623,6 +623,11 @@
   [s]
   (string/includes? s ".property"))
 
+(defn plugin-property?
+  "Determines if keyword is a plugin property"
+  [kw]
+  (string/starts-with? (namespace kw) "plugin.property."))
+
 (defn internal-property?
   "Determines if ident kw is an internal property. This includes db-attribute properties
    unlike logseq-property? and doesn't include non-property idents unlike internal-ident?"

deps/db/src/logseq/db/frontend/validate.cljs

@@ -97,7 +97,7 @@
         ent-maps* (db-malli-schema/datoms->entities datoms)
         ent-maps (mapv
                   ;; Remove some UI interactions adding this e.g. import
-                  #(dissoc % :block.temp/fully-loaded? :block.temp/has-children?)
+                  #(dissoc % :block.temp/load-status :block.temp/has-children?)
                   (db-malli-schema/update-properties-in-ents db ent-maps*))
         errors (binding [db-malli-schema/*db-for-validate-fns* db]
                  (-> (map (fn [e]

deps/db/test/logseq/db_test.cljs

@@ -2,31 +2,24 @@
   (:require [cljs.test :refer [deftest is]]
             [datascript.core :as d]
             [logseq.db :as ldb]
-            [logseq.db.file-based.schema :as file-schema]
             [logseq.db.test.helper :as db-test]))
 
 ;;; datoms
 ;;; - 1 <----+
 ;;;   - 2    |
 ;;;     - 3 -+
-(def broken-outliner-data-with-cycle
-  [{:db/id 1
-    :block/uuid #uuid"e538d319-48d4-4a6d-ae70-c03bb55b6fe4"
-    :block/parent 3}
-   {:db/id 2
-    :block/uuid #uuid"c46664c0-ea45-4998-adf0-4c36486bb2e5"
-    :block/parent 1}
-   {:db/id 3
-    :block/uuid #uuid"2b736ac4-fd49-4e04-b00f-48997d2c61a2"
-    :block/parent 2}])
-
-(deftest get-block-children-ids-on-bad-outliner-data
-  (let [db (d/db-with (d/empty-db file-schema/schema)
-                      broken-outliner-data-with-cycle)]
-    (is (= "bad outliner data, need to re-index to fix"
-           (try (ldb/get-block-children-ids db #uuid "e538d319-48d4-4a6d-ae70-c03bb55b6fe4")
-                (catch :default e
-                  (ex-message e)))))))
+(comment
+  ;; TODO: throw error or fix broken data when cycle detected
+  (def broken-outliner-data-with-cycle
+    [{:db/id 1
+      :block/uuid #uuid"e538d319-48d4-4a6d-ae70-c03bb55b6fe4"
+      :block/parent 3}
+     {:db/id 2
+      :block/uuid #uuid"c46664c0-ea45-4998-adf0-4c36486bb2e5"
+      :block/parent 1}
+     {:db/id 3
+      :block/uuid #uuid"2b736ac4-fd49-4e04-b00f-48997d2c61a2"
+      :block/parent 2}]))
 
 (def class-parents-data
   [{:block/tags :logseq.class/Tag
@@ -82,4 +75,4 @@
         "Class pages correctly found for given class")
     (is (= nil
            (ldb/page-exists? @conn "movie" #{:logseq.class/Property}))
-        "Class pages correctly not found for given class")))
+        "Class pages correctly not found for given class")))

deps/graph-parser/script/db_import.cljs

@@ -10,8 +10,10 @@
             [clojure.set :as set]
             [clojure.string :as string]
             [datascript.core :as d]
+            [logseq.common.config :as common-config]
             [logseq.common.graph :as common-graph]
             [logseq.db.common.sqlite-cli :as sqlite-cli]
+            [logseq.db.frontend.asset :as db-asset]
             [logseq.graph-parser.exporter :as gp-exporter]
             [logseq.outliner.cli :as outliner-cli]
             [logseq.outliner.pipeline :as outliner-pipeline]
@@ -47,11 +49,25 @@
   (p/let [s (fsp/readFile (:path file))]
     (str s)))
 
-(defn- <copy-asset-file [file db-graph-dir file-graph-dir]
-  (p/let [parent-dir (node-path/dirname
-                      (node-path/join db-graph-dir (node-path/relative file-graph-dir (:path file))))
+(defn- <read-asset-file [file assets]
+  (p/let [buffer (fs/readFileSync (:path file))
+          checksum (db-asset/<get-file-array-buffer-checksum buffer)]
+    (swap! assets assoc
+           (gp-exporter/asset-path->name (:path file))
+           {:size (.-length buffer)
+            :checksum checksum
+            :type (db-asset/asset-path->type (:path file))
+            :path (:path file)})))
+
+(defn- <copy-asset-file [asset-m db-graph-dir]
+  (p/let [parent-dir (node-path/join db-graph-dir common-config/local-assets-dir)
           _ (fsp/mkdir parent-dir #js {:recursive true})]
-    (fsp/copyFile (:path file) (node-path/join parent-dir (node-path/basename (:path file))))))
+    (if (:block/uuid asset-m)
+      (fsp/copyFile (:path asset-m) (node-path/join parent-dir (str (:block/uuid asset-m) "." (:type asset-m))))
+      (do
+        (println "[INFO]" "Copied asset" (pr-str (node-path/basename (:path asset-m)))
+                 "by its name since it was unused.")
+        (fsp/copyFile (:path asset-m) (node-path/join parent-dir (node-path/basename (:path asset-m))))))))
 
 (defn- notify-user [{:keys [continue debug]} m]
   (println (:msg m))
@@ -103,7 +119,8 @@
                        (default-export-options options)
                         ;; asset file options
                        {:<copy-asset (fn copy-asset [file]
-                                       (<copy-asset-file file db-graph-dir file-graph-dir))})]
+                                       (<copy-asset-file file db-graph-dir))
+                        :<read-asset <read-asset-file})]
     (p/with-redefs [d/transact! dev-transact!]
       (gp-exporter/export-file-graph conn conn config-file *files options))))
 
@@ -184,10 +201,12 @@
 
       (when-let [ignored-props (seq @(:ignored-properties import-state))]
         (println "Ignored properties:" (pr-str ignored-props)))
+      (when-let [ignored-assets (seq @(:ignored-assets import-state))]
+        (println "Ignored assets:" (pr-str ignored-assets)))
       (when-let [ignored-files (seq @(:ignored-files import-state))]
         (println (count ignored-files) "ignored file(s):" (pr-str (vec ignored-files))))
       (when (:verbose options') (println "Transacted" (count (d/datoms @conn :eavt)) "datoms"))
       (println "Created graph" (str db-name "!")))))
 
 (when (= nbb/*file* (nbb/invoked-file))
-  (-main *command-line-args*))
+  (-main *command-line-args*))

deps/graph-parser/src/logseq/graph_parser/block.cljs

@@ -132,7 +132,7 @@
     (when (some-> block-id parse-uuid)
       block-id)))
 
-(defn- paragraph-block?
+(defn paragraph-block?
   [block]
   (and
    (vector? block)
@@ -711,36 +711,36 @@
 
 (defn extract-blocks
   "Extract headings from mldoc ast. Args:
-  *`blocks`: mldoc ast.
-  *  `content`: markdown or org-mode text.
-  *  `format`: content's format, it could be either :markdown or :org-mode.
-  *  `options`: Options are :user-config, :block-pattern, :parse-block, :date-formatter, :db and
+  * `ast`: mldoc ast.
+  * `content`: markdown or org-mode text.
+  * `format`: content's format, it could be either :markdown or :org-mode.
+  * `options`: Options are :user-config, :block-pattern, :parse-block, :date-formatter, :db and
      * :db-graph-mode? : Set when a db graph in the frontend
      * :export-to-db-graph? : Set when exporting to a db graph"
-  [blocks content format {:keys [user-config db-graph-mode? export-to-db-graph?] :as options}]
-  {:pre [(seq blocks) (string? content) (contains? #{:markdown :org} format)]}
+  [ast content format {:keys [user-config db-graph-mode? export-to-db-graph?] :as options}]
+  {:pre [(seq ast) (string? content) (contains? #{:markdown :org} format)]}
   (let [encoded-content (utf8/encode content)
-        all-blocks (vec (reverse blocks))
+        all-blocks (vec (reverse ast))
         [blocks body pre-block-properties]
         (loop [headings []
-               blocks (reverse blocks)
+               ast-blocks (reverse ast)
                block-idx 0
                timestamps {}
                properties {}
                body []]
-          (if (seq blocks)
-            (let [[block pos-meta] (first blocks)]
+          (if (seq ast-blocks)
+            (let [[ast-block pos-meta] (first ast-blocks)]
               (cond
-                (paragraph-timestamp-block? block)
-                (let [timestamps (extract-timestamps block)
+                (paragraph-timestamp-block? ast-block)
+                (let [timestamps (extract-timestamps ast-block)
                       timestamps' (merge timestamps timestamps)]
-                  (recur headings (rest blocks) (inc block-idx) timestamps' properties body))
+                  (recur headings (rest ast-blocks) (inc block-idx) timestamps' properties body))
 
-                (gp-property/properties-ast? block)
-                (let [properties (extract-properties (second block) (assoc user-config :format format))]
-                  (recur headings (rest blocks) (inc block-idx) timestamps properties body))
+                (gp-property/properties-ast? ast-block)
+                (let [properties (extract-properties (second ast-block) (assoc user-config :format format))]
+                  (recur headings (rest ast-blocks) (inc block-idx) timestamps properties body))
 
-                (heading-block? block)
+                (heading-block? ast-block)
                 ;; for db-graphs cut multi-line when there is property, deadline/scheduled or logbook text in :block/title
                 (let [cut-multiline? (and export-to-db-graph?
                                           (when-let [prev-block (first (get all-blocks (dec block-idx)))]
@@ -762,14 +762,19 @@
                                       (and export-to-db-graph?
                                            (and (gp-property/properties-ast? (first (get all-blocks (dec block-idx))))
                                                 (= "Custom" (ffirst (get all-blocks (- block-idx 2)))))))
-                      block' (construct-block block properties timestamps body encoded-content format pos-meta' options')
-                      block'' (if (or db-graph-mode? export-to-db-graph?)
+                      block' (construct-block ast-block properties timestamps body encoded-content format pos-meta' options')
+                      block'' (cond
+                                db-graph-mode?
                                 block'
-                                (assoc block' :macros (extract-macros-from-ast (cons block body))))]
-                  (recur (conj headings block'') (rest blocks) (inc block-idx) {} {} []))
+                                export-to-db-graph?
+                                (assoc block' :block.temp/ast-blocks (cons ast-block body))
+                                :else
+                                (assoc block' :macros (extract-macros-from-ast (cons ast-block body))))]
+
+                  (recur (conj headings block'') (rest ast-blocks) (inc block-idx) {} {} []))
 
                 :else
-                (recur headings (rest blocks) (inc block-idx) timestamps properties (conj body block))))
+                (recur headings (rest ast-blocks) (inc block-idx) timestamps properties (conj body ast-block))))
             [(-> (reverse headings)
                  sanity-blocks-data)
              body

deps/graph-parser/src/logseq/graph_parser/exporter.cljs

@@ -1,12 +1,14 @@
 (ns logseq.graph-parser.exporter
   "Exports a file graph to DB graph. Used by the File to DB graph importer and
   by nbb-logseq CLIs"
-  (:require [borkdude.rewrite-edn :as rewrite]
+  (:require ["path" :as node-path]
+            [borkdude.rewrite-edn :as rewrite]
             [cljs-time.coerce :as tc]
             [cljs.pprint]
             [clojure.edn :as edn]
             [clojure.set :as set]
             [clojure.string :as string]
+            [clojure.walk :as walk]
             [datascript.core :as d]
             [logseq.common.config :as common-config]
             [logseq.common.path :as path]
@@ -19,6 +21,7 @@
             [logseq.db :as ldb]
             [logseq.db.common.order :as db-order]
             [logseq.db.common.property-util :as db-property-util]
+            [logseq.db.frontend.asset :as db-asset]
             [logseq.db.frontend.class :as db-class]
             [logseq.db.frontend.content :as db-content]
             [logseq.db.frontend.db-ident :as db-ident]
@@ -889,6 +892,89 @@
     (:block/name (:block/parent block))
     (assoc :block/parent {:block/uuid (get-page-uuid page-names-to-uuids (:block/name (:block/parent block)) {:block block :block/parent (:block/parent block)})})))
 
+(defn asset-path->name
+  "Given an asset's relative or full path, create a unique name for identifying an asset.
+   Must handle to paths as ../assets/*, assets/* and with subdirectories"
+  [path]
+  (re-find #"assets/.*$" path))
+
+(defn- find-all-asset-links
+  "Walks each ast block in order to its full depth as Link asts can be in different
+   locations e.g. a Heading vs a Paragraph ast block"
+  [ast-blocks]
+  (let [results (atom [])]
+    (walk/prewalk
+     (fn [x]
+       (when (and (vector? x)
+                  (= "Link" (first x))
+                  (common-config/local-asset? (second (:url (second x)))))
+         (swap! results conj x))
+       x)
+     ast-blocks)
+    @results))
+
+(defn- update-asset-links-in-block-title [block-title asset-name-to-uuids ignored-assets]
+  (reduce (fn [acc [asset-name asset-uuid]]
+            (let [new-title (string/replace acc
+                                            (re-pattern (str "!?\\[[^\\]]*?\\]\\([^\\)]*?"
+                                                             asset-name
+                                                             "\\)(\\{[^}]*\\})?"))
+                                            (page-ref/->page-ref asset-uuid))]
+              (when (string/includes? new-title asset-name)
+                (swap! ignored-assets conj
+                       {:reason "Some asset links were not updated to block references"
+                        :path asset-name
+                        :location {:block new-title}}))
+              new-title))
+          block-title
+          asset-name-to-uuids))
+
+(defn- handle-assets-in-block
+  [block* {:keys [assets ignored-assets]}]
+  (let [block (dissoc block* :block.temp/ast-blocks)
+        asset-links (find-all-asset-links (:block.temp/ast-blocks block*))]
+    (if (seq asset-links)
+      (let [asset-maps
+            (keep
+             (fn [asset-link]
+               (let [asset-name (-> asset-link second :url second asset-path->name)]
+                 (if-let [asset-data (and asset-name (get @assets asset-name))]
+                   (if (:block/uuid asset-data)
+                     {:asset-name-uuid [asset-name (:block/uuid asset-data)]}
+                     (let [new-block (sqlite-util/block-with-timestamps
+                                      {:block/uuid (d/squuid)
+                                       :block/order (db-order/gen-key)
+                                       :block/page :logseq.class/Asset
+                                       :block/parent :logseq.class/Asset})
+                           new-asset (merge new-block
+                                            {:block/tags [:logseq.class/Asset]
+                                             :logseq.property.asset/type (:type asset-data)
+                                             :logseq.property.asset/checksum (:checksum asset-data)
+                                             :logseq.property.asset/size (:size asset-data)
+                                             :block/title (db-asset/asset-name->title (node-path/basename asset-name))}
+                                            (when-let [metadata (not-empty (common-util/safe-read-map-string (:metadata (second asset-link))))]
+                                              {:logseq.property.asset/resize-metadata metadata}))]
+                      ;;  (prn :asset-added! (node-path/basename asset-name) #_(get @assets asset-name))
+                      ;;  (cljs.pprint/pprint asset-link)
+                       (swap! assets assoc-in [asset-name :block/uuid] (:block/uuid new-block))
+                       {:asset-name-uuid [asset-name (:block/uuid new-asset)]
+                        :asset new-asset}))
+                   (do
+                     (swap! ignored-assets conj
+                            {:reason "No asset data found for this asset path"
+                             :path (-> asset-link second :url second)
+                             :location {:block (:block/title block)}})
+                     nil))))
+             asset-links)
+            asset-blocks (keep :asset asset-maps)
+            asset-names-to-uuids
+            (into {} (map :asset-name-uuid asset-maps))]
+        (cond-> {:block
+                 (update block :block/title update-asset-links-in-block-title asset-names-to-uuids ignored-assets)}
+          (seq asset-blocks)
+          (assoc :asset-blocks-tx asset-blocks)))
+      {:block block})))
+
 (defn- build-block-tx
   [db block* pre-blocks {:keys [page-names-to-uuids] :as per-file-state} {:keys [import-state journal-created-ats] :as options}]
   ;; (prn ::block-in block*)
@@ -896,9 +982,11 @@
         {:keys [block properties-tx]}
         (handle-block-properties block* db page-names-to-uuids (:block/refs block*) options)
         {block-after-built-in-props :block deadline-properties-tx :properties-tx} (update-block-deadline block page-names-to-uuids options)
+        {block-after-assets :block :keys [asset-blocks-tx]}
+        (handle-assets-in-block block-after-built-in-props (select-keys import-state [:assets :ignored-assets]))
         ;; :block/page should be [:block/page NAME]
         journal-page-created-at (some-> (:block/page block*) second journal-created-ats)
-        prepared-block (cond-> block-after-built-in-props
+        prepared-block (cond-> block-after-assets
                          journal-page-created-at
                          (assoc :block/created-at journal-page-created-at))
         block' (-> prepared-block
@@ -913,8 +1001,8 @@
                    (dissoc :block/left :block/format)
                    ;; ((fn [x] (prn :block-out x) x))
                    )]
-    ;; Order matters as properties are referenced in block
-    (concat properties-tx deadline-properties-tx [block'])))
+    ;; Order matters as previous txs are referenced in block
+    (concat properties-tx deadline-properties-tx asset-blocks-tx [block'])))
 
 (defn- update-page-alias
   [m page-names-to-uuids]
@@ -1147,6 +1235,8 @@
    :ignored-properties (atom [])
    ;; Vec of maps with keys :path and :reason
    :ignored-files (atom [])
+   ;; Vec of maps with keys :path, :reason and :location (optional).
+   :ignored-assets (atom [])
    ;; Map of property names (keyword) and their current schemas (map of qualified properties).
    ;; Used for adding schemas to properties and detecting changes across a property's usage
    :property-schemas (atom {})
@@ -1158,7 +1248,9 @@
    :classes-from-property-parents (atom #{})
    ;; Map of block uuids to their :block/properties-text-values value.
    ;; Used if a property value changes to :default
-   :block-properties-text-values (atom {})})
+   :block-properties-text-values (atom {})
+   ;; Track asset data for use across asset and doc import steps
+   :assets (atom {})})
 
 (defn- build-tx-options [{:keys [user-options] :as options}]
   (merge
@@ -1555,32 +1647,58 @@
                  class-to-prop-uuids)]
     (ldb/transact! repo-or-conn tx)))
 
-(defn- export-asset-files
-  "Exports files under assets/"
-  [*asset-files <copy-asset-file {:keys [notify-user set-ui-state]
+(defn- <safe-async-loop
+  "Calls async-fn with each element in args-to-loop. Catches an unexpected error in loop and notifies user"
+  [async-fn args-to-loop notify-user]
+  (-> (p/loop [_ (async-fn (get args-to-loop 0))
+               i 0]
+        (when-not (>= i (dec (count args-to-loop)))
+          (p/recur (async-fn (get args-to-loop (inc i)))
+                   (inc i))))
+      (p/catch (fn [e]
+                 (notify-user {:msg (str "Import has an unexpected error:\n" (.-message e))
+                               :level :error
+                               :ex-data {:error e}})))))
+
+(defn- read-asset-files
+  "Reads files under assets/"
+  [*asset-files <read-asset-file {:keys [notify-user set-ui-state assets]
                                   :or {set-ui-state (constantly nil)}}]
+  (assert <read-asset-file "read-asset-file fn required")
   (let [asset-files (mapv #(assoc %1 :idx %2)
                           ;; Sort files to ensure reproducible import behavior
                           (sort-by :path *asset-files)
                           (range 0 (count *asset-files)))
-        copy-asset (fn copy-asset [{:keys [path] :as file}]
+        read-asset (fn read-asset [{:keys [path] :as file}]
+                     (-> (<read-asset-file file assets)
+                         (p/catch
+                          (fn [error]
+                            (notify-user {:msg (str "Import failed to read " (pr-str path) " with error:\n" (.-message error))
+                                          :level :error
+                                          :ex-data {:path path :error error}})))))]
+    (when (seq asset-files)
+      (set-ui-state [:graph/importing-state :current-page] "Read asset files")
+      (<safe-async-loop read-asset asset-files notify-user))))
+
+(defn- copy-asset-files
+  "Copy files under assets/"
+  [asset-maps* <copy-asset-file {:keys [notify-user set-ui-state]
+                                 :or {set-ui-state (constantly nil)}}]
+  (assert <copy-asset-file "copy-asset-file fn required")
+  (let [asset-maps (mapv #(assoc %1 :idx %2)
+                          ;; Sort files to ensure reproducible import behavior
+                         (sort-by :path asset-maps*)
+                         (range 0 (count asset-maps*)))
+        copy-asset (fn copy-asset [{:keys [path] :as asset-m}]
                      (p/catch
-                      (<copy-asset-file file)
+                      (<copy-asset-file asset-m)
                       (fn [error]
-                        (notify-user {:msg (str "Import failed on " (pr-str path) " with error:\n" (.-message error))
+                        (notify-user {:msg (str "Import failed to copy " (pr-str path) " with error:\n" (.-message error))
                                       :level :error
                                       :ex-data {:path path :error error}}))))]
-    (when (seq asset-files)
-      (set-ui-state [:graph/importing-state :current-page] "Asset files")
-      (-> (p/loop [_ (copy-asset (get asset-files 0))
-                   i 0]
-            (when-not (>= i (dec (count asset-files)))
-              (p/recur (copy-asset (get asset-files (inc i)))
-                       (inc i))))
-          (p/catch (fn [e]
-                     (notify-user {:msg (str "Import has an unexpected error:\n" (.-message e))
-                                   :level :error
-                                   :ex-data {:error e}})))))))
+    (when (seq asset-maps)
+      (set-ui-state [:graph/importing-state :current-page] "Copy asset files")
+      (<safe-async-loop copy-asset asset-maps notify-user))))
 
 (defn- insert-favorites
   "Inserts favorited pages as uuids into a new favorite page"
@@ -1612,7 +1730,7 @@
        (log-fn :no-favorites-found {:favorites favorites})))))
 
 (defn build-doc-options
-  "Builds options for use with export-doc-files"
+  "Builds options for use with export-doc-files and assets"
   [config options]
   (-> {:extract-options {:date-formatter (common-config/get-date-formatter config)
                          ;; Remove config keys that break importing
@@ -1661,9 +1779,10 @@
    * :<save-config-file - fn which saves a config file
    * :<save-logseq-file - fn which saves a logseq file
    * :<copy-asset - fn which copies asset file
+   * :<read-asset - fn which reads asset file
 
    Note: See export-doc-files for additional options that are only for it"
-  [repo-or-conn conn config-file *files {:keys [<read-file <copy-asset rpath-key log-fn]
+  [repo-or-conn conn config-file *files {:keys [<read-file <copy-asset <read-asset rpath-key log-fn]
                                          :or {rpath-key :path log-fn println}
                                          :as options}]
   (reset! gp-block/*export-to-db-graph? true)
@@ -1685,12 +1804,20 @@
         (export-logseq-files repo-or-conn (filter logseq-file? files) <read-file
                              (-> (select-keys options [:notify-user :<save-logseq-file])
                                  (set/rename-keys {:<save-logseq-file :<save-file})))
-        (export-asset-files asset-files <copy-asset (select-keys options [:notify-user :set-ui-state]))
+        ;; Assets are read first as doc-files need data from them to make Asset blocks.
+        ;; Assets are copied after after doc-files as they need block/uuid's from them to name assets
+        (read-asset-files asset-files <read-asset (merge (select-keys options [:notify-user :set-ui-state])
+                                                         {:assets (get-in doc-options [:import-state :assets])}))
         (export-doc-files conn doc-files <read-file doc-options)
+        (copy-asset-files (vals @(get-in doc-options [:import-state :assets]))
+                          <copy-asset
+                          (select-keys options [:notify-user :set-ui-state]))
         (export-favorites-from-config-edn conn repo-or-conn config {})
         (export-class-properties conn repo-or-conn)
         (move-top-parent-pages-to-library conn repo-or-conn)
-        {:import-state (:import-state doc-options)
+        {:import-state (-> (:import-state doc-options)
+                           ;; don't leak full asset content (which could be large) out of this ns
+                           (dissoc :assets))
          :files files})))
    (p/finally (fn [_]
                 (reset! gp-block/*export-to-db-graph? false)))

deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs

@@ -1,7 +1,7 @@
 (ns ^:node-only logseq.graph-parser.exporter-test
   (:require ["fs" :as fs]
             ["path" :as node-path]
-            [cljs.test :refer [testing is]]
+            [cljs.test :refer [testing is are deftest]]
             [clojure.set :as set]
             [clojure.string :as string]
             [datascript.core :as d]
@@ -10,6 +10,7 @@
             [logseq.common.util.date-time :as date-time-util]
             [logseq.db :as ldb]
             [logseq.db.common.entity-plus :as entity-plus]
+            [logseq.db.frontend.asset :as db-asset]
             [logseq.db.frontend.content :as db-content]
             [logseq.db.frontend.malli-schema :as db-malli-schema]
             [logseq.db.frontend.rules :as rules]
@@ -93,6 +94,17 @@
    ;; TODO: Add actual default
    :default-config {}})
 
+;; Copied from db-import
+(defn- <read-asset-file [file assets]
+  (p/let [buffer (fs/readFileSync (:path file))
+          checksum (db-asset/<get-file-array-buffer-checksum buffer)]
+    (swap! assets assoc
+           (gp-exporter/asset-path->name (:path file))
+           {:size (.-length buffer)
+            :checksum checksum
+            :type (db-asset/asset-path->type (:path file))
+            :path (:path file)})))
+
 ;; Copied from db-import script and tweaked for an in-memory import
 (defn- import-file-graph-to-db
   "Import a file graph dir just like UI does. However, unlike the UI the
@@ -105,7 +117,12 @@
         options' (merge default-export-options
                         {:user-options (merge {:convert-all-tags? false} (dissoc options :assets :verbose))
                         ;; asset file options
-                         :<copy-asset #(swap! assets conj %)}
+                         :<read-asset <read-asset-file
+                         :<copy-asset (fn copy-asset [m]
+                                        (when-not (:block/uuid m)
+                                          (println "[INFO]" "Asset" (pr-str (node-path/basename (:path m)))
+                                                   "does not have a :block/uuid"))
+                                        (swap! assets conj m))}
                         (select-keys options [:verbose]))]
     (gp-exporter/export-file-graph conn conn config-file *files options')))
 
@@ -128,6 +145,24 @@
 ;; Tests
 ;; =====
 
+(deftest update-asset-links-in-block-title
+  (are [x y]
+       (= y (@#'gp-exporter/update-asset-links-in-block-title (first x) {(second x) "UUID"} (atom {})))
+    ;; Standard image link with metadata
+    ["![greg-popovich-thumbs-up.png](../assets/greg-popovich-thumbs-up_1704749687791_0.png){:height 288, :width 100} says pop"
+     "assets/greg-popovich-thumbs-up_1704749687791_0.png"]
+    "[[UUID]] says pop"
+
+    ;; Image link with no metadata
+    ["![some-title](../assets/CleanShot_2022-10-12_at_15.53.20@2x_1665561216083_0.png)"
+     "assets/CleanShot_2022-10-12_at_15.53.20@2x_1665561216083_0.png"]
+    "[[UUID]]"
+
+    ;; 2nd link
+    ["[[FIRST UUID]] and ![dino!](assets/subdir/partydino.gif)"
+     "assets/subdir/partydino.gif"]
+    "[[FIRST UUID]] and [[UUID]]"))
+
 (deftest-async ^:integration export-docs-graph-with-convert-all-tags
   (p/let [file-graph-dir "test/resources/docs-0.10.12"
           start-time (cljs.core/system-time)
@@ -146,6 +181,7 @@
     (is (empty? (map :entity (:errors (db-validate/validate-db! @conn))))
         "Created graph has no validation errors")
     (is (= 0 (count @(:ignored-properties import-state))) "No ignored properties")
+    (is (= 0 (count @(:ignored-assets import-state))) "No ignored assets")
     (is (= []
            (->> (d/q '[:find (pull ?b [:block/title {:block/tags [:db/ident]}])
                        :where [?b :block/tags :logseq.class/Tag]]
@@ -170,8 +206,9 @@
 
       ;; Counts
       ;; Includes journals as property values e.g. :logseq.property/deadline
-      (is (= 25 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Journal]] @conn))))
+      (is (= 26 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Journal]] @conn))))
 
+      (is (= 3 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Asset]] @conn))))
       (is (= 4 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Task]] @conn))))
       (is (= 4 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Query]] @conn))))
       (is (= 2 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Card]] @conn))))
@@ -195,8 +232,9 @@
           "Correct number of user classes")
       (is (= 4 (count (d/datoms @conn :avet :block/tags :logseq.class/Whiteboard))))
       (is (= 0 (count @(:ignored-properties import-state))) "No ignored properties")
+      (is (= 0 (count @(:ignored-assets import-state))) "No ignored assets")
       (is (= 1 (count @(:ignored-files import-state))) "Ignore .edn for now")
-      (is (= 1 (count @assets))))
+      (is (= 3 (count @assets))))
 
     (testing "logseq files"
       (is (= ".foo {}\n"
@@ -365,7 +403,19 @@
       ;; Cards
       (is (= {:block/tags [:logseq.class/Card]}
              (db-test/readable-properties (db-test/find-block-by-content @conn "card 1")))
-          "None of the card properties are imported since they are deprecated"))
+          "None of the card properties are imported since they are deprecated")
+
+      ;; Assets
+      (is (= {:block/tags [:logseq.class/Asset]
+              :logseq.property.asset/type "png"
+              :logseq.property.asset/checksum "3d5e620cac62159d8196c118574bfea7a16e86fa86efd1c3fa15a00a0a08792d"
+              :logseq.property.asset/size 753471
+              :logseq.property.asset/resize-metadata {:height 288, :width 252}}
+             (db-test/readable-properties (db-test/find-block-by-content @conn "greg-popovich-thumbs-up_1704749687791_0")))
+          "Asset has correct properties")
+      (is (= (d/entity @conn :logseq.class/Asset)
+             (:block/page (db-test/find-block-by-content @conn "greg-popovich-thumbs-up_1704749687791_0")))
+          "Imported into Asset page"))
 
     (testing "tags convert to classes"
       (is (= :user.class/Quotes___life

deps/graph-parser/test/resources/exporter-test-graph/journals/2024_01_08.md

@@ -1,3 +1,3 @@
 - [[some page]] tests page ref being parsed before page
 - [[prop-num]] tests page existing before being used as and converted into a property
-- ![greg-popovich-thumbs-up.png](../assets/greg-popovich-thumbs-up_1704749687791_0.png){:height 288, :width 252}
+- ![greg-popovich-thumbs-up.png](../assets/greg-popovich-thumbs-up_1704749687791_0.png){:height 288, :width 252}

deps/graph-parser/test/resources/exporter-test-graph/journals/2025_06_12.md

@@ -0,0 +1,5 @@
+- Test paragraph before an asset
+  
+  
+  ![dino!](assets/subdir/partydino.gif){:width 105} tests an asset with a manual link, custom title and in a subdirectory
+- ![greg-popovich-thumbs-up.png](../assets/greg-popovich-thumbs-up_1704749687791_0.png){:height 288, :width 252} and ![HEART Teams](../assets/HEART_Teams.png){:width 335.99774169921875}

deps/outliner/src/logseq/outliner/core.cljs

@@ -240,7 +240,7 @@
           m* (cond->
               (-> data'
                   (dissoc :block/children :block/meta :block/unordered
-                          :block.temp/ast-title :block.temp/ast-body :block/level :block.temp/fully-loaded?
+                          :block.temp/ast-title :block.temp/ast-body :block/level :block.temp/load-status
                           :block.temp/has-children?)
                   common-util/remove-nils
 

src/main/frontend/common_keywords.cljs

@@ -9,4 +9,8 @@
   :block/raw-title {:doc "like `:block/title`,
                           but when eval `(:block/raw-title block-entity)`, return raw title of this block"}
   :kv/value        {:doc "Used to store key-value, the value could be anything,
-                          e.g. {:db/ident :logseq.kv/xxx :kv/value value}"})
+                          e.g. {:db/ident :logseq.kv/xxx :kv/value value}"}
+
+;; :block.temp/xxx keywords
+  :block.temp/load-status {:doc "`:full` means the block and its children have been loaded,
+                                 `:self` means the block itself has been loaded."})

src/main/frontend/components/block.cljs

@@ -2656,8 +2656,7 @@
               (editor-handler/clear-selection!)
               (editor-handler/unhighlight-blocks!)
               (let [f #(p/do!
-                        (when-not (:block.temp/fully-loaded? (db/entity (:db/id block)))
-                          (db-async/<get-block (state/get-current-repo) (:db/id block) {:children? false}))
+                        (db-async/<get-block (state/get-current-repo) (:db/id block) {:children? false})
                         (let [cursor-range (some-> (gdom/getElement block-id)
                                                    (dom/by-class "block-content-inner")
                                                    first

src/main/frontend/components/imports.cljs

@@ -1,6 +1,7 @@
 (ns frontend.components.imports
   "Import data into Logseq."
-  (:require [cljs-time.core :as t]
+  (:require ["path" :as node-path]
+            [cljs-time.core :as t]
             [cljs.pprint :as pprint]
             [clojure.string :as string]
             [frontend.components.onboarding.setups :as setups]
@@ -26,7 +27,9 @@
             [goog.functions :refer [debounce]]
             [goog.object :as gobj]
             [lambdaisland.glogi :as log]
+            [logseq.common.config :as common-config]
             [logseq.common.path :as path]
+            [logseq.db.frontend.asset :as db-asset]
             [logseq.db.frontend.validate :as db-validate]
             [logseq.graph-parser.exporter :as gp-exporter]
             [logseq.shui.dialog.core :as shui-dialog]
@@ -297,6 +300,13 @@
                         :info false)
     (log/error :import-ignored-files {:msg (str "Import ignored " (count ignored-files) " file(s)")})
     (pprint/pprint ignored-files))
+  (when-let [ignored-assets (seq @(:ignored-assets import-state))]
+    (notification/show! (str "Import ignored " (count ignored-assets) " "
+                             (if (= 1 (count ignored-assets)) "asset" "assets")
+                             ". See the javascript console for more details.")
+                        :info false)
+    (log/error :import-ignored-assets {:msg (str "Import ignored " (count ignored-assets) " asset(s)")})
+    (pprint/pprint ignored-assets))
   (when-let [ignored-props (seq @(:ignored-properties import-state))]
     (notification/show!
      [:.mb-2
@@ -338,14 +348,32 @@
         (log/error :import-error ex-data)))
     (notification/show! msg :warning false)))
 
-(defn- copy-asset [repo repo-dir file]
+(defn- read-asset [file assets]
   (-> (.arrayBuffer (:file-object file))
+      (p/then (fn [buffer]
+                (p/let [checksum (db-asset/<get-file-array-buffer-checksum buffer)]
+                  (swap! assets assoc
+                         (gp-exporter/asset-path->name (:path file))
+                         {:size (.-size (:file-object file))
+                          :checksum checksum
+                          :type (db-asset/asset-path->type (:path file))
+                          :path (:path file)
+                          ;; Save buffer to avoid reading asset twice
+                          ::array-buffer buffer}))))))
+
+(defn- copy-asset [repo repo-dir asset-m]
+  (-> (::array-buffer asset-m)
       (p/then (fn [buffer]
                 (let [content (js/Uint8Array. buffer)
-                      parent-dir (path/path-join repo-dir (path/dirname (:path file)))]
+                      assets-dir (path/path-join repo-dir common-config/local-assets-dir)]
                   (p/do!
-                   (fs/mkdir-if-not-exists parent-dir)
-                   (fs/write-plain-text-file! repo repo-dir (:path file) content {:skip-transact? true})))))))
+                   (fs/mkdir-if-not-exists assets-dir)
+                   (if (:block/uuid asset-m)
+                     (fs/write-plain-text-file! repo assets-dir (str (:block/uuid asset-m) "." (:type asset-m)) content {:skip-transact? true})
+                     (do
+                       (println "Copied asset" (pr-str (node-path/basename (:path asset-m)))
+                                "by its name since it was unused.")
+                       (fs/write-plain-text-file! repo assets-dir (node-path/basename (:path asset-m)) content {:skip-transact? true})))))))))
 
 (defn- import-file-graph
   [*files
@@ -375,6 +403,7 @@
                    :<save-logseq-file (fn save-logseq-file [_ path content]
                                         (db-editor-handler/save-file! path content))
                    ;; asset file options
+                   :<read-asset read-asset
                    :<copy-asset #(copy-asset repo (config/get-repo-dir repo) %)
                    ;; doc file options
                    ;; Write to frontend first as writing to worker first is poor ux with slow streaming changes

src/main/frontend/components/page.cljs

@@ -697,7 +697,7 @@
                  *loading? (atom true)
                  page (db/get-page page-id-uuid-or-name)
                  *page (atom page)]
-             (when (:block.temp/fully-loaded? page) (reset! *loading? false))
+             (when (:block.temp/load-status page) (reset! *loading? false))
              (p/let [page-block (db-async/<get-block (state/get-current-repo) page-id-uuid-or-name)]
                (reset! *loading? false)
                (reset! *page (db/entity (:db/id page-block)))

src/main/frontend/components/views.cljs

@@ -909,7 +909,7 @@
 (rum/defc table-row < rum/reactive db-mixins/query
   [table row props option]
   (let [block (db/sub-block (:db/id row))
-        row' (if (:block.temp/fully-loaded? block)
+        row' (if (:block.temp/load-status block)
                (assoc block :block.temp/refs-count (:block.temp/refs-count row))
                row)]
     (table-row-inner table row' props option)))

src/main/frontend/db/async.cljs

@@ -80,11 +80,11 @@
                              (assoc opts :property-ident property-id))))
 
 (defn <get-block
-  [graph id-uuid-or-name & {:keys [children? nested-children? skip-transact? skip-refresh? children-only? properties]
+  [graph id-uuid-or-name & {:keys [children? skip-transact? skip-refresh? children-only? properties]
                             :or {children? true}
                             :as opts}]
 
-  ;; (prn :debug :<get-block id-uuid-or-name)
+  ;; (prn :debug :<get-block id-uuid-or-name :children? children?)
   ;; (js/console.trace)
   (let [name' (str id-uuid-or-name)
         opts (assoc opts :children? children?)
@@ -97,12 +97,11 @@
             (db/get-page name'))
         id (or (and (:block/uuid e) (str (:block/uuid e)))
                (and (util/uuid-string? name') name')
-               id-uuid-or-name)]
+               id-uuid-or-name)
+        load-status (:block.temp/load-status e)]
     (cond
-      (and (:block.temp/fully-loaded? e) ; children may not be fully loaded
-           (not children-only?)
-           (not children?)
-           (not nested-children?)
+      (and (or (= load-status :full)
+               (and (= load-status :self) (not children?) (not children-only?)))
            (not (some #{:block.temp/refs-count} properties)))
       (p/promise e)
 
@@ -113,11 +112,16 @@
                {:keys [block children]} (first result)]
          (when-not skip-transact?
            (let [conn (db/get-db graph false)
+                 load-status' (if (or children? children-only?) :full :self)
+                 block-load-status-tx (when block
+                                        [{:db/id (:db/id block)
+                                          :block.temp/load-status load-status'}])
                  block-and-children (if block (cons block children) children)
                  affected-keys [[:frontend.worker.react/block (:db/id block)]]
-                 tx-data (->> (remove (fn [b] (:block.temp/fully-loaded? (db/entity (:db/id b)))) block-and-children)
+                 tx-data (->> (remove (fn [b] (:block.temp/load-status (db/entity (:db/id b)))) block-and-children)
                               (common-util/fast-remove-nils)
-                              (remove empty?))]
+                              (remove empty?)
+                              (concat block-load-status-tx))]
              (when (seq tx-data) (d/transact! conn tx-data))
              (when-not skip-refresh?
                (react/refresh-affected-queries! graph affected-keys {:skip-kv-custom-keys? true}))))
@@ -129,7 +133,7 @@
 
 (defn <get-blocks
   [graph ids* & {:as opts}]
-  (let [ids (remove (fn [id] (:block.temp/fully-loaded? (db/entity id))) ids*)]
+  (let [ids (remove (fn [id] (:block.temp/load-status (db/entity id))) ids*)]
     (when (seq ids)
       (p/let [result (state/<invoke-db-worker :thread-api/get-blocks graph
                                               (map (fn [id]
@@ -138,7 +142,7 @@
         (let [conn (db/get-db graph false)
               result' (map :block result)]
           (when (seq result')
-            (let [result'' (map (fn [b] (assoc b :block.temp/fully-loaded? true)) result')]
+            (let [result'' (map (fn [b] (assoc b :block.temp/load-status :self)) result')]
               (d/transact! conn result'')))
           result')))))
 

src/main/frontend/extensions/pdf/assets.cljs

@@ -187,8 +187,7 @@
   [hls-page]
   (p/let [result (db-async/<get-block (state/get-current-repo)
                                       (:block/uuid hls-page)
-                                      {:children? true
-                                       :nested-children? false})]
+                                      {:children? true})]
     {:highlights (keep :logseq.property.pdf/hl-value result)}))
 
 (defn file-based-load-hls-data$

src/main/frontend/handler/assets.cljs

@@ -11,6 +11,7 @@
             [logseq.common.config :as common-config]
             [logseq.common.path :as path]
             [logseq.common.util :as common-util]
+            [logseq.db.frontend.asset :as db-asset]
             [medley.core :as medley]
             [missionary.core :as m]
             [promesa.core :as p])
@@ -177,18 +178,10 @@
                 blob (js/Blob. (array binary) (clj->js {:type "image"}))]
           (when blob (js/URL.createObjectURL blob)))))))
 
-(defn- decode-digest
-  [^js/Uint8Array digest]
-  (.. (js/Array.from digest)
-      (map (fn [s] (.. s (toString 16) (padStart 2 "0"))))
-      (join "")))
-
 (defn get-file-checksum
   [^js/Blob file]
   (-> (.arrayBuffer file)
-      (.then (fn [buf] (js/crypto.subtle.digest "SHA-256" buf)))
-      (.then (fn [dig] (js/Uint8Array. dig)))
-      (.then decode-digest)))
+      (.then db-asset/<get-file-array-buffer-checksum)))
 
 (defn <get-all-assets
   []

src/main/frontend/handler/block.cljs

@@ -178,8 +178,7 @@
     (util/mobile-keep-keyboard-open)
     (let [repo (state/get-current-repo)]
       (p/do!
-       (when-not (:block.temp/fully-loaded? (db/entity (:db/id block)))
-         (db-async/<get-block repo (:db/id block) {:children? false}))
+       (db-async/<get-block repo (:db/id block) {:children? false})
        (when save-code-editor? (state/pub-event! [:editor/save-code-editor]))
        (when (not= (:block/uuid block) (:block/uuid (state/get-edit-block)))
          (state/clear-edit! {:clear-editing-block? false}))

src/main/frontend/handler/editor.cljs

@@ -1,5 +1,6 @@
 (ns ^:no-doc frontend.handler.editor
-  (:require [clojure.set :as set]
+  (:require ["path" :as node-path]
+            [clojure.set :as set]
             [clojure.string :as string]
             [clojure.walk :as w]
             [dommy.core :as dom]
@@ -58,6 +59,7 @@
             [logseq.db :as ldb]
             [logseq.db.common.entity-plus :as entity-plus]
             [logseq.db.file-based.schema :as file-schema]
+            [logseq.db.frontend.asset :as db-asset]
             [logseq.db.frontend.property :as db-property]
             [logseq.graph-parser.block :as gp-block]
             [logseq.graph-parser.mldoc :as gp-mldoc]
@@ -1448,15 +1450,14 @@
     (p/all
      (for [[_index ^js file] (map-indexed vector files)]
       ;; WARN file name maybe fully qualified path when paste file
-       (p/let [file-name (util/node-path.basename (.-name file))
-               file-name-without-ext (.-name (util/node-path.parse file-name))
+       (p/let [file-name (node-path/basename (.-name file))
+               file-name-without-ext (db-asset/asset-name->title file-name)
                checksum (assets-handler/get-file-checksum file)
                existing-asset (db-async/<get-asset-with-checksum repo checksum)]
          (if existing-asset
            existing-asset
            (p/let [block-id (ldb/new-block-id)
-                   ext (when file-name
-                         (string/lower-case (.substr (util/node-path.extname file-name) 1)))
+                   ext (when file-name (db-asset/asset-path->type file-name))
                    _ (when (string/blank? ext)
                        (throw (ex-info "File doesn't have a valid ext."
                                        {:file-name file-name})))
@@ -2116,8 +2117,7 @@
                      (db-async/<get-template-by-name (name db-id)))
              block (when (:block/uuid block)
                      (db-async/<get-block repo (:block/uuid block)
-                                          {:children? true
-                                           :nested-children? true}))]
+                                          {:children? true}))]
        (when (:db/id block)
          (let [journal? (ldb/journal? target)
                target (or target (state/get-edit-block))
@@ -3511,7 +3511,7 @@
             repo (state/get-current-repo)
             result (db-async/<get-block repo (or block-id page-id)
                                         {:children-only? true
-                                         :nested-children? true})
+                                         :include-collapsed-children? true})
             blocks (if page-id
                      result
                      (cons (db/entity [:block/uuid block-id]) result))
@@ -3587,7 +3587,8 @@
 (defn expand-block! [block-id & {:keys [skip-db-collpsing?]}]
   (let [repo (state/get-current-repo)]
     (p/do!
-     (db-async/<get-block repo block-id {:children-only? true})
+     (db-async/<get-block repo block-id {:children-only? true
+                                         :include-collapsed-children? true})
      (when-not (or skip-db-collpsing? (skip-collapsing-in-db?))
        (set-blocks-collapsed! [block-id] false))
      (state/set-collapsed-block! block-id false))))

src/main/frontend/handler/events.cljs

@@ -422,8 +422,7 @@
     ;; load all nested children here for copy/export
     (p/all (map (fn [id]
                   (db-async/<get-block (state/get-current-repo) id
-                                       {:nested-children? true
-                                        :skip-refresh? false})) ids))))
+                                       {:skip-refresh? false})) ids))))
 
 (defn run!
   []

src/main/frontend/modules/outliner/pipeline.cljs

@@ -63,7 +63,7 @@
                             (if (contains? #{:create-property-text-block :insert-blocks} (:outliner-op tx-meta))
                               (let [update-blocks-fully-loaded (keep (fn [datom] (when (= :block/uuid (:a datom))
                                                                                    {:db/id (:e datom)
-                                                                                    :block.temp/fully-loaded? true})) tx-data)]
+                                                                                    :block.temp/load-status :self})) tx-data)]
                                 (concat update-blocks-fully-loaded tx-data))
                               tx-data))]
               (d/transact! conn tx-data' tx-meta))

src/main/frontend/util.cljc

@@ -817,7 +817,7 @@
                              (pr-str
                               {:graph graph
                                :embed-block? embed-block?
-                               :blocks (mapv #(dissoc % :block.temp/fully-loaded? %) blocks)}))}))]
+                               :blocks (mapv #(dissoc % :block.temp/load-status %) blocks)}))}))]
        (if owner-window
          (utils/writeClipboard data owner-window)
          (utils/writeClipboard data)))))

src/main/frontend/worker/db/migrate.cljs

@@ -232,7 +232,9 @@
                                      [:db/retract id :logseq.property/default-value]
                                      [:db/retract id :logseq.property/hide-empty-value]
                                      [:db/retract id :logseq.property/enable-history?]]
-             datoms (d/datoms db :avet ident)]
+             datoms (if (:db/index class)
+                      (d/datoms db :avet ident)
+                      (filter (fn [d] (= ident (:a d))) (d/datoms db :eavt)))]
          (concat [new-property]
                  retract-property-attrs
                  (mapcat

src/main/logseq/api.cljs

@@ -717,9 +717,7 @@
 (defn- <ensure-page-loaded
   [block-uuid-or-page-name]
   (p/let [repo (state/get-current-repo)
-          block (db-async/<get-block repo (str block-uuid-or-page-name)
-                                     {:children-props '[*]
-                                      :nested-children? true})
+          block (db-async/<get-block repo (str block-uuid-or-page-name) {})
           _ (when-let [page-id (:db/id (:block/page block))]
               (when-let [page-uuid (:block/uuid (db/entity page-id))]
                 (db-async/<get-block repo page-uuid)))]

src/main/logseq/api/block.cljs

@@ -149,7 +149,7 @@
                       ;; nested children results
                       (let [blocks (->> (db-model/get-block-and-children repo uuid)
                                         (map (fn [b]
-                                               (dissoc (db-utils/pull (:db/id b)) :block.temp/fully-loaded?))))]
+                                               (dissoc (db-utils/pull (:db/id b)) :block.temp/load-status))))]
                         (first (outliner-tree/blocks->vec-tree blocks uuid)))
                       ;; attached shallow children
                       (assoc block :block/children