wip: store assets as blocks

deps/db/src/logseq/db/frontend/class.cljs

@@ -29,6 +29,11 @@
     :properties {:logseq.property/icon {:type :tabler-icon :id "search"}
                  :logseq.property/parent :logseq.class/Query}}
 
+   :logseq.class/Asset
+   {:title "Asset"
+    :properties {:logseq.property/icon {:type :tabler-icon :id "file"}}
+    :schema {:properties [:logseq.property.asset/type :logseq.property.asset/size :logseq.property.asset/checksum]}}
+
    ;; TODO: Add more classes such as :book, :paper, :movie, :music, :project
    })
 

deps/db/src/logseq/db/frontend/malli_schema.cljs

@@ -122,8 +122,8 @@
                          ;; use explicit call to be nbb compatible
                          [(let [closed-values (entity-plus/lookup-kv-then-entity property :property/closed-values)]
                             (cond-> (assoc (select-keys property [:db/ident :db/valueType :db/cardinality])
-                                    :block/schema
-                                    (select-keys (:block/schema property) [:type]))
+                                           :block/schema
+                                           (select-keys (:block/schema property) [:type]))
                               (seq closed-values)
                               (assoc :property/closed-values closed-values)))
                           v])
@@ -254,7 +254,6 @@
     page-attrs
     page-or-block-attrs)))
 
-
 (def property-common-schema-attrs
   "Property :schema attributes common to all properties"
   [[:hide? {:optional true} :boolean]
@@ -400,11 +399,6 @@
    [:file/created-at inst?]
    [:file/last-modified-at inst?]])
 
-(def asset-block
-  [:map
-   [:asset/uuid :uuid]
-   [:asset/meta :map]])
-
 (def db-ident-key-val
   "A key value map with :db/ident and :kv/value"
   [:map
@@ -435,8 +429,6 @@
                           :file-block
                           (:block/uuid d)
                           :block
-                          (:asset/uuid d)
-                          :asset-block
                           (= (:db/ident d) :logseq.property/empty-placeholder)
                           :property-value-placeholder
                           (:db/ident d)
@@ -448,7 +440,6 @@
     :block block
     :file-block file-block
     :db-ident-key-value db-ident-key-val
-    :asset-block asset-block
     :property-value-placeholder property-value-placeholder}))
 
 (def DB
@@ -480,7 +471,7 @@
                     {}))))
 
 (let [malli-non-ref-attrs (->> (concat property-attrs page-attrs block-attrs page-or-block-attrs (rest normal-page))
-                               (concat (rest file-block) (rest asset-block) (rest property-value-block)
+                               (concat (rest file-block) (rest property-value-block)
                                        (rest db-ident-key-val) (rest class-page))
                                (remove #(= (last %) [:set :int]))
                                (map first)

deps/db/src/logseq/db/frontend/property.cljs

@@ -233,6 +233,18 @@
                               {:type :node
                                :hide? true
                                :public? false}}
+   :logseq.property.asset/type {:title "File type"
+                                :schema {:type :string
+                                         :hide? true
+                                         :public? false}}
+   :logseq.property.asset/size {:title "File size"
+                                :schema {:type :raw-number
+                                         :hide? true
+                                         :public? false}}
+   :logseq.property.asset/checksum {:title "File checksum"
+                                    :schema {:type :string
+                                             :hide? true
+                                             :public? false}}
    :logseq.property.asset/remote-metadata {:schema
                                            {:type :map
                                             :hide? true

deps/db/src/logseq/db/frontend/schema.cljs

@@ -2,7 +2,7 @@
   "Main datascript schemas for the Logseq app"
   (:require [clojure.set :as set]))
 
-(def version 27)
+(def version 28)
 ;; A page is a special block, a page can corresponds to multiple files with the same ":block/name".
 (def ^:large-vars/data-var schema
   {:db/ident        {:db/unique :db.unique/identity}
@@ -117,9 +117,7 @@
                                   :db/cardinality :db.cardinality/many}
     :property/schema.classes {:db/valueType :db.type/ref
                               :db/cardinality :db.cardinality/many}
-    :property.value/content {}
-    :asset/uuid {:db/unique :db.unique/identity}
-    :asset/meta {}}))
+    :property.value/content {}}))
 
 (def retract-attributes
   #{:block/refs

src/main/frontend/components/block.cljs

@@ -2994,7 +2994,7 @@
               (when (and (config/local-file-based-graph? repo) (not (state/editing?)))
                 ;; Basically the same logic as editor-handler/upload-asset,
                 ;; does not require edting
-                (-> (editor-handler/save-assets! repo (js->clj files))
+                (-> (editor-handler/file-based-save-assets! repo (js->clj files))
                     (p/then
                      (fn [res]
                        (when-let [[asset-file-name file-obj asset-file-fpath matched-alias] (and (seq res) (first res))]

src/main/frontend/extensions/tldraw.cljs

@@ -77,12 +77,12 @@
                           (-> b
                               (update :block/uuid str)
                               (update :block/title #(->> (text-util/cut-by % "$pfts_2lqh>$" "$<pfts_2lqh$")
-                                                           (apply str))))) blocks)]
+                                                         (apply str))))) blocks)]
       (clj->js {:blocks blocks}))))
 
 (defn save-asset-handler
   [file]
-  (-> (editor-handler/save-assets! (state/get-current-repo) [(js->clj file)])
+  (-> (editor-handler/file-based-save-assets! (state/get-current-repo) [(js->clj file)])
       (p/then
        (fn [res]
          (when-let [[asset-file-name _ full-file-path] (and (seq res) (first res))]

src/main/frontend/handler/assets.cljs

@@ -100,7 +100,6 @@
         (path/path-join "file://" (common-util/safe-decode-uri-component path))
         (path/path-join "file://" path))
 
-
       :else ;; relative path or alias path
       (resolve-asset-real-path-url (state/get-current-repo) path))))
 
@@ -197,3 +196,16 @@
                 (p/let [url (js/URL.createObjectURL file)]
                   (swap! *assets-url-cache assoc (keyword handle-path) url)
                   url)))))))))
+
+(defn- decode-digest
+  [^js/Uint8Array digest]
+  (.. (js/Array.from digest)
+      (map (fn [s] (.. s (toString 16) (padStart 2 "0"))))
+      (join "")))
+
+(defn get-file-checksum
+  [^js/Blob file]
+  (-> (.arrayBuffer file)
+      (.then (fn [buf] (js/crypto.subtle.digest "SHA-256" buf)))
+      (.then (fn [dig] (js/Uint8Array. dig)))
+      (.then decode-digest)))

src/main/frontend/handler/editor.cljs

@@ -1415,40 +1415,41 @@
   (p/let [[repo-dir assets-dir] (ensure-assets-dir! (state/get-current-repo))]
     (path/path-join repo-dir assets-dir filename)))
 
-(defn save-assets!
+(defn file-based-save-assets!
   "Save incoming(pasted) assets to assets directory.
 
    Returns: [file-rpath file-obj file-fpath matched-alias]"
   ([repo files]
    (p/let [[repo-dir assets-dir] (ensure-assets-dir! repo)]
-     (save-assets! repo repo-dir assets-dir files
-                   (fn [index file-stem]
+     (file-based-save-assets! repo repo-dir assets-dir files
+                              (fn [index file-stem]
                      ;; TODO: maybe there're other chars we need to handle?
-                     (let [file-base (-> file-stem
-                                         (string/replace " " "_")
-                                         (string/replace "%" "_")
-                                         (string/replace "/" "_"))
-                           file-name (str file-base "_" (.now js/Date) "_" index)]
-                       (string/replace file-name #"_+" "_"))))))
+                                (let [file-base (-> file-stem
+                                                    (string/replace " " "_")
+                                                    (string/replace "%" "_")
+                                                    (string/replace "/" "_"))
+                                      file-name (str file-base "_" (.now js/Date) "_" index)]
+                                  (string/replace file-name #"_+" "_"))))))
   ([repo repo-dir asset-dir-rpath files gen-filename]
    (p/all
     (for [[index ^js file] (map-indexed vector files)]
       ;; WARN file name maybe fully qualified path when paste file
-      (let [file-name (util/node-path.basename (.-name file))
-            [file-stem ext-full ext-base] (if file-name
-                                            (let [ext-base (util/node-path.extname file-name)
-                                                  ext-full (if-not (config/extname-of-supported? ext-base)
-                                                             (util/full-path-extname file-name) ext-base)]
-                                              [(subs file-name 0 (- (count file-name)
-                                                                    (count ext-full))) ext-full ext-base])
-                                            ["" "" ""])
-            filename  (str (gen-filename index file-stem) ext-full)
-            file-rpath  (str asset-dir-rpath "/" filename)
-            matched-alias (assets-handler/get-matched-alias-by-ext ext-base)
-            file-rpath (cond-> file-rpath
-                         (not (nil? matched-alias))
-                         (string/replace #"^[.\/\\]*assets[\/\\]+" ""))
-            dir (or (:dir matched-alias) repo-dir)]
+      (p/let [file-name (util/node-path.basename (.-name file))
+              [file-stem ext-full ext-base] (if file-name
+                                              (let [ext-base (util/node-path.extname file-name)
+                                                    ext-full (if-not (config/extname-of-supported? ext-base)
+                                                               (util/full-path-extname file-name) ext-base)]
+                                                [(subs file-name 0 (- (count file-name)
+                                                                      (count ext-full))) ext-full ext-base])
+                                              ["" "" ""])
+              filename  (str (gen-filename index file-stem) ext-full)
+              file-rpath  (str asset-dir-rpath "/" filename)
+              matched-alias (assets-handler/get-matched-alias-by-ext ext-base)
+              file-rpath (cond-> file-rpath
+                           (not (nil? matched-alias))
+                           (string/replace #"^[.\/\\]*assets[\/\\]+" ""))
+              dir (or (:dir matched-alias) repo-dir)
+              checksum (assets-handler/get-file-checksum file)]
         (if (util/electron?)
           (let [from (not-empty (.-path file))]
             (js/console.debug "Debug: Copy Asset #" dir file-rpath from)
@@ -1518,39 +1519,135 @@
       (path/get-relative-path current-file-fpath file-path))
     file-path))
 
+(defn file-upload-assets!
+  "Paste asset and insert link to current editing block"
+  [repo id ^js files format uploading? drop-or-paste?]
+  (when (config/local-file-based-graph? repo)
+    (-> (file-based-save-assets! repo (js->clj files))
+          ;; FIXME: only the first asset is handled
+        (p/then
+         (fn [res]
+           (when-let [[asset-file-name file-obj asset-file-fpath matched-alias] (and (seq res) (first res))]
+             (let [image? (config/ext-of-image? asset-file-name)]
+               (insert-command!
+                id
+                (assets-handler/get-asset-file-link format
+                                                    (if matched-alias
+                                                      (str
+                                                       (if image? "../assets/" "")
+                                                       "@" (:name matched-alias) "/" asset-file-name)
+                                                      (resolve-relative-path (or asset-file-fpath asset-file-name)))
+                                                    (if file-obj (.-name file-obj) (if image? "image" "asset"))
+                                                    image?)
+                format
+                {:last-pattern (if drop-or-paste? "" commands/command-trigger)
+                 :restore?     true
+                 :command      :insert-asset})))))
+        (p/catch (fn [e]
+                   (js/console.error e)))
+        (p/finally
+          (fn []
+            (reset! uploading? false)
+            (reset! *asset-uploading? false)
+            (reset! *asset-uploading-process 0))))))
+
+(defn db-based-save-assets!
+  "Save incoming(pasted) assets to assets directory.
+
+   Returns: [file-rpath file-obj file-fpath]"
+  ([repo files]
+   (p/let [[repo-dir assets-dir] (ensure-assets-dir! repo)]
+     (db-based-save-assets! repo repo-dir assets-dir files)))
+  ([repo repo-dir asset-dir-rpath files]
+   (p/all
+    (for [[index ^js file] (map-indexed vector files)]
+      ;; WARN file name maybe fully qualified path when paste file
+      (p/let [file-name (util/node-path.basename (.-name file))
+              checksum (assets-handler/get-file-checksum file)
+              block-id (ldb/new-block-id)
+              ext (when file-name
+                    (string/lower-case (.substr (util/node-path.extname file-name) 1)))
+              _ (when (string/blank? ext)
+                  (throw (ex-info "File doesn't have a valid ext."
+                                  {:file-name file-name})))
+              file-path   (str block-id "." ext)
+              file-rpath  (str asset-dir-rpath "/" file-path)
+              dir repo-dir
+              asset (db/entity :logseq.class/Asset)
+              properties {:logseq.property.asset/type ext
+                          :logseq.property.asset/size (.-size file)
+                          :logseq.property.asset/checksum checksum
+                          :block/tags (:db/id asset)}
+              result (api-insert-new-block! file-name
+                                            {:page (:block/uuid asset)
+                                             :block-uuid block-id
+                                             :edit-block? false
+                                             :properties properties})
+              new-entity (db/entity [:block/uuid (:block/uuid result)])]
+        (if (util/electron?)
+          (let [from (not-empty (.-path file))]
+            (js/console.debug "Debug: Copy Asset #" dir file-rpath from)
+            (-> (js/window.apis.copyFileToAssets dir file-rpath from)
+                (p/then
+                 (fn [_dest]
+                   new-entity))
+                (p/catch #(js/console.error "Debug: Copy Asset Error#" %))))
+
+          (->
+           (p/do! (js/console.debug "Debug: Writing Asset #" dir file-rpath)
+                  (cond
+                    (mobile-util/native-platform?)
+                    ;; capacitor fs accepts Blob, File implements Blob
+                    (p/let [buffer (.arrayBuffer file)
+                            content (base64/encodeByteArray (js/Uint8Array. buffer))
+                            fpath (path/path-join dir file-rpath)]
+                      (capacitor-fs/<write-file-with-base64 fpath content))
+
+                    (config/db-based-graph? repo) ;; memory-fs
+                    (p/let [buffer (.arrayBuffer file)
+                            content (js/Uint8Array. buffer)]
+                      (fs/write-file! repo dir file-rpath content nil))
+
+                    :else
+                    (throw (ex-info "Paste failed"
+                                    {:file-name file-name})))
+                  new-entity)
+           (p/catch (fn [error]
+                      (prn :paste-file-error)
+                      (js/console.error error))))))))))
+
+(defn db-upload-assets!
+  "Paste asset and insert link to current editing block"
+  [repo id ^js files format uploading? drop-or-paste?]
+  (when (or (config/local-file-based-graph? repo)
+            (config/db-based-graph? repo))
+    (-> (db-based-save-assets! repo (js->clj files))
+          ;; FIXME: only the first asset is handled
+        (p/then
+         (fn [entities]
+           (let [entity (first entities)]
+             (insert-command!
+              id
+              (page-ref/->page-ref (:block/uuid entity))
+              format
+              {:last-pattern (if drop-or-paste? "" commands/command-trigger)
+               :restore?     true
+               :command      :insert-asset}))))
+        (p/catch (fn [e]
+                   (js/console.error e)))
+        (p/finally
+          (fn []
+            (reset! uploading? false)
+            (reset! *asset-uploading? false)
+            (reset! *asset-uploading-process 0))))))
+
 (defn upload-asset!
   "Paste asset and insert link to current editing block"
   [id ^js files format uploading? drop-or-paste?]
   (let [repo (state/get-current-repo)]
-    (when (or (config/local-file-based-graph? repo)
-              (config/db-based-graph? repo))
-      (-> (save-assets! repo (js->clj files))
-          ;; FIXME: only the first asset is handled
-          (p/then
-           (fn [res]
-             (when-let [[asset-file-name file-obj asset-file-fpath matched-alias] (and (seq res) (first res))]
-               (let [image? (config/ext-of-image? asset-file-name)]
-                 (insert-command!
-                  id
-                  (assets-handler/get-asset-file-link format
-                                                      (if matched-alias
-                                                        (str
-                                                         (if image? "../assets/" "")
-                                                         "@" (:name matched-alias) "/" asset-file-name)
-                                                        (resolve-relative-path (or asset-file-fpath asset-file-name)))
-                                                      (if file-obj (.-name file-obj) (if image? "image" "asset"))
-                                                      image?)
-                  format
-                  {:last-pattern (if drop-or-paste? "" commands/command-trigger)
-                   :restore?     true
-                   :command      :insert-asset})))))
-          (p/catch (fn [e]
-                     (js/console.error e)))
-          (p/finally
-            (fn []
-              (reset! uploading? false)
-              (reset! *asset-uploading? false)
-              (reset! *asset-uploading-process 0)))))))
+    (if (config/db-based-graph? repo)
+      (db-upload-assets! repo id ^js files format uploading? drop-or-paste?)
+      (file-upload-assets! repo id ^js files format uploading? drop-or-paste?))))
 
 ;; Editor should track some useful information, like editor modes.
 ;; For example:

src/main/frontend/worker/db/migrate.cljs

@@ -200,6 +200,16 @@
         [[:db/add card-id :logseq.property.class/properties :logseq.property.fsrs/due]
          [:db/add card-id :logseq.property.class/properties :logseq.property.fsrs/state]]))))
 
+(defn- add-asset-properties
+  [conn _search-db]
+  (let [db @conn]
+    (when (ldb/db-based-graph? db)
+      (let [e (d/entity db :logseq.class/Asset)
+            eid (:db/id e)]
+        [[:db/add eid :logseq.property.class/properties :logseq.property.asset/type]
+         [:db/add eid :logseq.property.class/properties :logseq.property.asset/size]
+         [:db/add eid :logseq.property.class/properties :logseq.property.asset/checksum]]))))
+
 (defn- add-query-property-to-query-tag
   [conn _search-db]
   (let [db @conn]
@@ -276,7 +286,9 @@
    [25 {:properties [:logseq.property/query]
         :fix add-query-property-to-query-tag}]
    [26 {:properties [:logseq.property.node/type]}]
-   [27 {:properties [:logseq.property.code/mode]}]])
+   [27 {:properties [:logseq.property.code/mode]}]
+   [28 {:classes [:logseq.class/Asset]
+        :properties [:logseq.property.asset/type :logseq.property.asset/size :logseq.property.asset/checksum]}]])
 
 (let [max-schema-version (apply max (map first schema-version->updates))]
   (assert (<= db-schema/version max-schema-version))
@@ -308,7 +320,7 @@
                             schema-version->updates)
               properties (mapcat :properties updates)
               new-properties (->> (select-keys db-property/built-in-properties properties)
-                                ;; property already exists, this should never happen
+                                  ;; property already exists, this should never happen
                                   (remove (fn [[k _]]
                                             (when (d/entity db k)
                                               (assert (str "DB migration: property already exists " k)))))
@@ -317,12 +329,12 @@
                                   (map (fn [b] (assoc b :logseq.property/built-in? true))))
               classes (mapcat :classes updates)
               new-classes (->> (select-keys db-class/built-in-classes classes)
-                             ;; class already exists, this should never happen
+                               ;; class already exists, this should never happen
                                (remove (fn [[k _]]
                                          (when (d/entity db k)
                                            (assert (str "DB migration: class already exists " k)))))
                                (into {})
-                               (#(sqlite-create-graph/build-initial-classes* % {}))
+                               (#(sqlite-create-graph/build-initial-classes* % (zipmap properties properties)))
                                (map (fn [b] (assoc b :logseq.property/built-in? true))))
               fixes (mapcat
                      (fn [update']