enhance: UI imports assets as #Asset

Also made importer and db asset creation use same helper fns

deps/db/src/logseq/db/frontend/asset.cljs

@@ -1,4 +1,7 @@
-(ns logseq.db.frontend.asset)
+(ns logseq.db.frontend.asset
+  "Asset fns used in node and browser contexts"
+  (:require ["path" :as node-path]
+            [clojure.string :as string]))
 
 (defn- decode-digest
   [^js/Uint8Array digest]
@@ -6,10 +9,19 @@
       (map (fn [s] (.. s (toString 16) (padStart 2 "0"))))
       (join "")))
 
-;; TODO: Reuse with frontend
 (defn <get-file-array-buffer-checksum
   "Given a file's ArrayBuffer, returns its checksum in a promise"
   [file-array-buffer]
   (-> (js/crypto.subtle.digest "SHA-256" file-array-buffer)
       (.then (fn [dig] (js/Uint8Array. dig)))
-      (.then decode-digest)))
+      (.then decode-digest)))
+
+(defn asset-path->type
+  "Create asset type given asset path"
+  [path]
+  (string/lower-case (.substr (node-path/extname path) 1)))
+
+(defn asset-name->title
+  "Create asset title given asset path's basename"
+  [path-basename]
+  (.-name (node-path/parse path-basename)))

deps/graph-parser/script/db_import.cljs

@@ -18,7 +18,8 @@
             [logseq.outliner.pipeline :as outliner-pipeline]
             [nbb.classpath :as cp]
             [nbb.core :as nbb]
-            [promesa.core :as p]))
+            [promesa.core :as p]
+            [logseq.common.config :as common-config]))
 
 (def tx-queue (atom cljs.core/PersistentQueue.EMPTY))
 (def original-transact! d/transact!)
@@ -48,22 +49,20 @@
   (p/let [s (fsp/readFile (:path file))]
     (str s)))
 
-(defn- <read-asset-file [file import-state]
+(defn- <read-asset-file [file assets]
   (p/let [buffer (fs/readFileSync (:path file))
-          checksum (db-asset/<get-file-array-buffer-checksum buffer)
-          ext (string/lower-case (.substr (node-path/extname (:path file)) 1))]
-    (swap! (:assets import-state) assoc
+          checksum (db-asset/<get-file-array-buffer-checksum buffer)]
+    (swap! assets assoc
            (node-path/basename (:path file))
            {:size (.-length buffer)
             :checksum checksum
-            :type ext
+            :type (db-asset/asset-path->type (:path file))
             :path (:path file)})))
 
-(defn- <copy-asset-file [file db-graph-dir file-graph-dir]
-  (p/let [parent-dir (node-path/dirname
-                      (node-path/join db-graph-dir (node-path/relative file-graph-dir (:path file))))
+(defn- <copy-asset-file [asset-m db-graph-dir]
+  (p/let [parent-dir (node-path/join db-graph-dir common-config/local-assets-dir)
           _ (fsp/mkdir parent-dir #js {:recursive true})]
-    (fsp/copyFile (:path file) (node-path/join parent-dir (str (:block/uuid file) "." (:type file))))))
+    (fsp/copyFile (:path asset-m) (node-path/join parent-dir (str (:block/uuid asset-m) "." (:type asset-m))))))
 
 (defn- notify-user [{:keys [continue debug]} m]
   (println (:msg m))
@@ -115,7 +114,7 @@
                        (default-export-options options)
                         ;; asset file options
                        {:<copy-asset (fn copy-asset [file]
-                                       (<copy-asset-file file db-graph-dir file-graph-dir))
+                                       (<copy-asset-file file db-graph-dir))
                         :<read-asset <read-asset-file})]
     (p/with-redefs [d/transact! dev-transact!]
       (gp-exporter/export-file-graph conn conn config-file *files options))))

deps/graph-parser/src/logseq/graph_parser/exporter.cljs

@@ -1,7 +1,8 @@
 (ns logseq.graph-parser.exporter
   "Exports a file graph to DB graph. Used by the File to DB graph importer and
   by nbb-logseq CLIs"
-  (:require [borkdude.rewrite-edn :as rewrite]
+  (:require ["path" :as node-path]
+            [borkdude.rewrite-edn :as rewrite]
             [cljs-time.coerce :as tc]
             [cljs.pprint]
             [clojure.edn :as edn]
@@ -31,7 +32,8 @@
             [logseq.graph-parser.block :as gp-block]
             [logseq.graph-parser.extract :as extract]
             [logseq.graph-parser.property :as gp-property]
-            [promesa.core :as p]))
+            [promesa.core :as p]
+            [logseq.db.frontend.asset :as db-asset]))
 
 (defn- add-missing-timestamps
   "Add updated-at or created-at timestamps if they doesn't exist"
@@ -896,7 +898,7 @@
                                 :title
                                 (filter #(and (= "Link" (first %))
                                               (common-config/local-asset? (second (:url (second %))))))))))
-        asset-name (some-> asset-links first second :url second path/basename)]
+        asset-name (some-> asset-links first second :url second node-path/basename)]
     (when (seq asset-links)
       (prn :asset-added! asset-name (get @assets asset-name)))
     ;; TODO: Handle asset metadata
@@ -909,8 +911,7 @@
                 :logseq.property.asset/type (:type asset-data)
                 :logseq.property.asset/checksum (:checksum asset-data)
                 :logseq.property.asset/size (:size asset-data)
-                ;; TODO: Ensure this matches DB version
-                :block/title (path/file-stem asset-name)}))
+                :block/title (db-asset/asset-name->title asset-name)}))
       block)))
 
 (defn- build-block-tx
@@ -1604,7 +1605,7 @@
 
 (defn- read-asset-files
   "Reads files under assets/"
-  [*asset-files <read-asset-file {:keys [notify-user set-ui-state import-state]
+  [*asset-files <read-asset-file {:keys [notify-user set-ui-state assets]
                                   :or {set-ui-state (constantly nil)}}]
   (assert <read-asset-file "read-asset-file fn required")
   (let [asset-files (mapv #(assoc %1 :idx %2)
@@ -1612,7 +1613,7 @@
                           (sort-by :path *asset-files)
                           (range 0 (count *asset-files)))
         read-asset (fn read-asset [{:keys [path] :as file}]
-                     (-> (<read-asset-file file import-state)
+                     (-> (<read-asset-file file assets)
                          (p/catch
                           (fn [error]
                             (notify-user {:msg (str "Import failed to read " (pr-str path) " with error:\n" (.-message error))
@@ -1624,27 +1625,27 @@
 
 (defn- copy-asset-files
   "Copy files under assets/"
-  [*asset-files <copy-asset-file {:keys [notify-user set-ui-state]
-                                  :or {set-ui-state (constantly nil)}}]
+  [asset-maps* <copy-asset-file {:keys [notify-user set-ui-state]
+                                 :or {set-ui-state (constantly nil)}}]
   (assert <copy-asset-file "copy-asset-file fn required")
-  (let [asset-files (mapv #(assoc %1 :idx %2)
+  (let [asset-maps (mapv #(assoc %1 :idx %2)
                           ;; Sort files to ensure reproducible import behavior
-                          (sort-by :path *asset-files)
-                          (range 0 (count *asset-files)))
-        copy-asset (fn copy-asset [{:keys [path] :as file}]
-                     (if (nil? (:block/uuid file))
+                         (sort-by :path asset-maps*)
+                         (range 0 (count asset-maps*)))
+        copy-asset (fn copy-asset [{:keys [path] :as asset-m}]
+                     (if (nil? (:block/uuid asset-m))
                        (notify-user {:msg (str "Import failed to copy " (pr-str path) " because the asset has no :block/uuid")
                                      :level :error
-                                     :ex-data {:file file}})
+                                     :ex-data {:path path}})
                        (p/catch
-                        (<copy-asset-file file)
+                        (<copy-asset-file asset-m)
                         (fn [error]
                           (notify-user {:msg (str "Import failed to copy " (pr-str path) " with error:\n" (.-message error))
                                         :level :error
                                         :ex-data {:path path :error error}})))))]
-    (when (seq asset-files)
+    (when (seq asset-maps)
       (set-ui-state [:graph/importing-state :current-page] "Copy asset files")
-      (<safe-async-loop copy-asset asset-files notify-user))))
+      (<safe-async-loop copy-asset asset-maps notify-user))))
 
 (defn- insert-favorites
   "Inserts favorited pages as uuids into a new favorite page"
@@ -1733,14 +1734,16 @@
         ;; Assets are read first as doc-files need data from them to make Asset blocks.
         ;; Assets are copied after after doc-files as they need block/uuid's from them to name assets
         (read-asset-files asset-files <read-asset (merge (select-keys options [:notify-user :set-ui-state])
-                                                         (select-keys doc-options [:import-state])))
+                                                         {:assets (get-in doc-options [:import-state :assets])}))
         (export-doc-files conn doc-files <read-file doc-options)
         (copy-asset-files (vals @(get-in doc-options [:import-state :assets]))
                           <copy-asset
                           (select-keys options [:notify-user :set-ui-state]))
         (export-favorites-from-config-edn conn repo-or-conn config {})
         (export-class-properties conn repo-or-conn)
-        {:import-state (:import-state doc-options)
+        {:import-state (-> (:import-state doc-options)
+                           ;; don't leak full asset content (which could be large) out of this ns
+                           (dissoc :assets))
          :files files})))
    (p/finally (fn [_]
                 (reset! gp-block/*export-to-db-graph? false)))

deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs

@@ -94,15 +94,15 @@
    ;; TODO: Add actual default
    :default-config {}})
 
-(defn- <read-asset-file [file import-state]
+;; Copied from db-import
+(defn- <read-asset-file [file assets]
   (p/let [buffer (fs/readFileSync (:path file))
-          checksum (db-asset/<get-file-array-buffer-checksum buffer)
-          ext (string/lower-case (.substr (node-path/extname (:path file)) 1))]
-    (swap! (:assets import-state) assoc
+          checksum (db-asset/<get-file-array-buffer-checksum buffer)]
+    (swap! assets assoc
            (node-path/basename (:path file))
            {:size (.-length buffer)
             :checksum checksum
-            :type ext
+            :type (db-asset/asset-path->type (:path file))
             :path (:path file)})))
 
 ;; Copied from db-import script and tweaked for an in-memory import
@@ -379,7 +379,14 @@
       ;; Cards
       (is (= {:block/tags [:logseq.class/Card]}
              (db-test/readable-properties (db-test/find-block-by-content @conn "card 1")))
-          "None of the card properties are imported since they are deprecated"))
+          "None of the card properties are imported since they are deprecated")
+
+      ;; Assets
+      (is (= {:block/tags [:logseq.class/Asset]
+              :logseq.property.asset/type "png"
+              :logseq.property.asset/checksum "3d5e620cac62159d8196c118574bfea7a16e86fa86efd1c3fa15a00a0a08792d"
+              :logseq.property.asset/size 753471}
+             (db-test/readable-properties (db-test/find-block-by-content @conn "greg-popovich-thumbs-up_1704749687791_0")))))
 
     (testing "tags convert to classes"
       (is (= :user.class/Quotes___life

src/main/frontend/components/imports.cljs

@@ -35,7 +35,9 @@
             [logseq.shui.hooks :as hooks]
             [logseq.shui.ui :as shui]
             [promesa.core :as p]
-            [rum.core :as rum]))
+            [rum.core :as rum]
+            ["path" :as node-path]
+            [logseq.common.config :as common-config]))
 
 ;; Can't name this component as `frontend.components.import` since shadow-cljs
 ;; will complain about it.
@@ -339,21 +341,27 @@
         (log/error :import-error ex-data)))
     (notification/show! msg :warning false)))
 
-;; TODO: Wire up UI
-(defn- copy-asset [repo repo-dir file]
-  ;; (prn ::copy-asset (:path file))
-  ;; (prn ::size (.-size (:file-object file)))
+(defn- read-asset [file assets]
   (-> (.arrayBuffer (:file-object file))
-      #_(p/then (fn [buffer]
-                  (p/let [checksum (db-asset/<get-file-array-buffer-checksum buffer)]
-                    (prn ::checksum2 checksum)
-                    buffer)))
+      (p/then (fn [buffer]
+                (p/let [checksum (db-asset/<get-file-array-buffer-checksum buffer)]
+                  (swap! assets assoc
+                         (node-path/basename (:path file))
+                         {:size (.-size (:file-object file))
+                          :checksum checksum
+                          :type (db-asset/asset-path->type (:path file))
+                          :path (:path file)
+                          ;; Save buffer to avoid reading asset twice
+                          ::array-buffer buffer}))))))
+
+(defn- copy-asset [repo repo-dir asset-m]
+  (-> (::array-buffer asset-m)
       (p/then (fn [buffer]
                 (let [content (js/Uint8Array. buffer)
-                      parent-dir (path/path-join repo-dir (path/dirname (:path file)))]
+                      assets-dir (path/path-join repo-dir common-config/local-assets-dir)]
                   (p/do!
-                   (fs/mkdir-if-not-exists parent-dir)
-                   (fs/write-plain-text-file! repo repo-dir (:path file) content {:skip-transact? true})))))))
+                   (fs/mkdir-if-not-exists assets-dir)
+                   (fs/write-plain-text-file! repo assets-dir (str (:block/uuid asset-m) "." (:type asset-m)) content {:skip-transact? true})))))))
 
 (defn- import-file-graph
   [*files
@@ -383,6 +391,7 @@
                    :<save-logseq-file (fn save-logseq-file [_ path content]
                                         (db-editor-handler/save-file! path content))
                    ;; asset file options
+                   :<read-asset read-asset
                    :<copy-asset #(copy-asset repo (config/get-repo-dir repo) %)
                    ;; doc file options
                    ;; Write to frontend first as writing to worker first is poor ux with slow streaming changes

src/main/frontend/handler/assets.cljs

@@ -13,7 +13,8 @@
             [logseq.common.util :as common-util]
             [medley.core :as medley]
             [missionary.core :as m]
-            [promesa.core :as p])
+            [promesa.core :as p]
+            [logseq.db.frontend.asset :as db-asset])
   (:import [missionary Cancelled]))
 
 (defn alias-enabled?
@@ -186,18 +187,10 @@
                 blob (js/Blob. (array binary) (clj->js {:type "image"}))]
           (when blob (js/URL.createObjectURL blob)))))))
 
-(defn- decode-digest
-  [^js/Uint8Array digest]
-  (.. (js/Array.from digest)
-      (map (fn [s] (.. s (toString 16) (padStart 2 "0"))))
-      (join "")))
-
 (defn get-file-checksum
   [^js/Blob file]
   (-> (.arrayBuffer file)
-      (.then (fn [buf] (js/crypto.subtle.digest "SHA-256" buf)))
-      (.then (fn [dig] (js/Uint8Array. dig)))
-      (.then decode-digest)))
+      (.then db-asset/<get-file-array-buffer-checksum)))
 
 (defn <get-all-assets
   []

src/main/frontend/handler/editor.cljs

@@ -1,5 +1,6 @@
 (ns ^:no-doc frontend.handler.editor
-  (:require [clojure.set :as set]
+  (:require ["path" :as node-path]
+            [clojure.set :as set]
             [clojure.string :as string]
             [clojure.walk :as w]
             [dommy.core :as dom]
@@ -58,6 +59,7 @@
             [logseq.db :as ldb]
             [logseq.db.common.entity-plus :as entity-plus]
             [logseq.db.file-based.schema :as file-schema]
+            [logseq.db.frontend.asset :as db-asset]
             [logseq.db.frontend.property :as db-property]
             [logseq.graph-parser.block :as gp-block]
             [logseq.graph-parser.mldoc :as gp-mldoc]
@@ -1422,15 +1424,14 @@
     (p/all
      (for [[_index ^js file] (map-indexed vector files)]
       ;; WARN file name maybe fully qualified path when paste file
-       (p/let [file-name (util/node-path.basename (.-name file))
-               file-name-without-ext (.-name (util/node-path.parse file-name))
+       (p/let [file-name (node-path/basename (.-name file))
+               file-name-without-ext (db-asset/asset-name->title file-name)
                checksum (assets-handler/get-file-checksum file)
                existing-asset (db-async/<get-asset-with-checksum repo checksum)]
          (if existing-asset
            existing-asset
            (p/let [block-id (ldb/new-block-id)
-                   ext (when file-name
-                         (string/lower-case (.substr (util/node-path.extname file-name) 1)))
+                   ext (when file-name (db-asset/asset-path->type file-name))
                    _ (when (string/blank? ext)
                        (throw (ex-info "File doesn't have a valid ext."
                                        {:file-name file-name})))