enhance: on-demand asset download

src/main/frontend/components/block.cljs

@@ -915,19 +915,39 @@
        [:a.asset-ref {:target "_blank" :href real-path-url}
         title-or-path])]))
 
+(defn- maybe-request-asset-download!
+  [state block]
+  (let [repo (state/get-current-repo)
+        file-exists? @(::file-exists? state)
+        requested? (get state ::download-requested?)
+        asset-file-write-finish @(get @state/state :assets/asset-file-write-finish)
+        asset-file-write-finished? (get-in asset-file-write-finish [repo (str (:block/uuid block))])
+        file-ready? (or file-exists? asset-file-write-finished?)]
+    (when (and (true? @requested?) file-ready?)
+      (reset! requested? false))
+    (when (and (not @requested?)
+               (assets-handler/maybe-request-remote-asset-download! repo block file-ready?))
+      (reset! requested? true)))
+  state)
+
 (rum/defcs asset-cp < rum/reactive
   (rum/local nil ::file-exists?)
+  (rum/local false ::download-requested?)
   {:will-mount (fn [state]
                  (let [block (last (:rum/args state))
                        asset-type (:logseq.property.asset/type block)
                        external-url? (not (string/blank? (:logseq.property.asset/external-url block)))
                        path (path/path-join common-config/local-assets-dir (str (:block/uuid block) "." asset-type))]
                    (p/let [result (if (or external-url? config/publishing?)
-                                                        ;; publishing doesn't have window.pfs defined
+                                    ;; publishing doesn't have window.pfs defined
                                     true
                                     (fs/file-exists? (config/get-repo-dir (state/get-current-repo)) path))]
                      (reset! (::file-exists? state) result))
-                   state))}
+                   state))
+   :did-mount (fn [state]
+                (maybe-request-asset-download! state (last (:rum/args state))))
+   :did-update (fn [state]
+                 (maybe-request-asset-download! state (last (:rum/args state))))}
   [state config block]
   (let [asset-type (:logseq.property.asset/type block)
         file (str (:block/uuid block) "." asset-type)
@@ -935,7 +955,22 @@
         repo (state/get-current-repo)
         asset-file-write-finished? (state/sub :assets/asset-file-write-finish
                                               {:path-in-sub-atom [repo (str (:block/uuid block))]})
-        asset-type (:logseq.property.asset/type block)
+        file-ready? (or file-exists? asset-file-write-finished?)
+        progress-entry (state/sub :rtc/asset-upload-download-progress
+                                  {:path-in-sub-atom [repo (str (:block/uuid block))]})
+        {:keys [direction loaded total]} progress-entry
+        in-progress? (and (number? loaded) (number? total) (pos? total) (not= loaded total))
+        percent (when in-progress?
+                  (int (* 100 (/ loaded total))))
+        label (case direction
+                :upload "Uploading"
+                :download "Downloading"
+                "Syncing")
+        progress-view (when in-progress?
+                        [:div.asset-transfer-progress
+                         [:div.asset-transfer-progress-label (str label " " percent "%")]
+                         [:div.asset-transfer-progress-bar
+                          [:span {:style {:width (str percent "%")}}]]])
         image? (contains? (common-config/img-formats) (keyword asset-type))
         width (get-in block [:logseq.property.asset/resize-metadata :width])
         asset-width (:logseq.property.asset/width block)
@@ -951,18 +986,24 @@
                             {:height (/ width aspect-ratio)}))))
         img-placeholder (when image?
                           [:div.img-placeholder.asset-container
-                           {:style img-metadata}])]
-    (cond
-      (or file-exists? asset-file-write-finished?)
-      (asset-link (assoc config
-                         :asset-block block
-                         :image-placeholder img-placeholder)
-                  (:block/title block)
-                  (path/path-join (str "../" common-config/local-assets-dir) file)
-                  img-metadata
-                  nil)
-      image?
-      img-placeholder)))
+                           {:style img-metadata}])
+        content (cond
+                  file-ready?
+                  (asset-link (assoc config
+                                     :asset-block block
+                                     :image-placeholder img-placeholder)
+                              (:block/title block)
+                              (path/path-join (str "../" common-config/local-assets-dir) file)
+                              img-metadata
+                              nil)
+                  image?
+                  img-placeholder)]
+    (if progress-view
+      [:div.asset-transfer-shell
+       (or content
+           [:div.asset-transfer-placeholder (str label " asset...")])
+       progress-view]
+      content)))
 
 (defn- img-audio-video?
   [block]

src/main/frontend/components/block.css

@@ -1249,3 +1249,23 @@ html.is-mac {
         }
     }
 }
+
+.asset-transfer-shell {
+  @apply relative inline-block w-full;
+}
+
+.asset-transfer-placeholder {
+  @apply text-sm text-gray-11 mt-2;
+}
+
+.asset-transfer-progress {
+  @apply absolute left-2 right-2 bottom-2 rounded-md px-2 py-1 text-xs text-white bg-black/60 flex flex-col gap-1;
+}
+
+.asset-transfer-progress-bar {
+  @apply w-full h-1 rounded bg-white/30 overflow-hidden;
+
+  > span {
+    @apply block h-full bg-white;
+  }
+}

src/main/frontend/handler/assets.cljs

@@ -208,6 +208,35 @@
         {:checksum checksum})
       (p/catch (constantly nil))))
 
+(defn- asset-transfer-in-progress?
+  [progress-entry]
+  (let [{:keys [loaded total]} progress-entry]
+    (and (number? loaded) (number? total) (pos? total) (not= loaded total))))
+
+(defn should-request-remote-asset-download?
+  [repo asset-block file-ready? progress]
+  (let [asset-uuid (:block/uuid asset-block)
+        asset-type (:logseq.property.asset/type asset-block)
+        external-url (:logseq.property.asset/external-url asset-block)
+        progress-entry (get progress (str asset-uuid))]
+    (and (seq repo)
+         asset-uuid
+         (seq asset-type)
+         (string/blank? external-url)
+         (not file-ready?)
+         (not (asset-transfer-in-progress? progress-entry)))))
+
+(defn maybe-request-remote-asset-download!
+  [repo asset-block file-ready?]
+  (let [progress-atom (get @state/state :rtc/asset-upload-download-progress)
+        progress (get (or (some-> progress-atom deref) {}) repo)]
+    (when (should-request-remote-asset-download? repo asset-block file-ready? progress)
+      (state/<invoke-db-worker
+       :thread-api/db-sync-request-asset-download
+       repo
+       (:block/uuid asset-block))
+      true)))
+
 (defn <write-asset
   [repo asset-block-id asset-type data]
   (let [asset-block-id-str (str asset-block-id)

src/main/frontend/worker/db_sync.cljs

@@ -763,6 +763,7 @@
             (take 3 item)
             item)))))
 
+;; Is `ensure-block-parents` really needed?
 (defn- ensure-block-parents
   "Ensure block entities don't lose :block/parent without becoming pages."
   [db tx-data]
@@ -994,7 +995,12 @@
         (if (empty? ops)
           nil
           (p/do!
-           (process-asset-op! repo graph-id (first ops))
+           (-> (process-asset-op! repo graph-id (first ops))
+               (p/catch (fn [e]
+                          (log/error :db-sync/asset-op-failed
+                                     {:repo repo
+                                      :asset-uuid (:block/uuid (first ops))
+                                      :error e}))))
            (p/recur (rest ops)))))
       (p/resolved nil))))
 
@@ -1016,15 +1022,26 @@
                                          ent (when conn (d/entity @conn [:block/uuid asset-uuid]))
                                          asset-type (:logseq.property.asset/type ent)]
                                      (p/do!
-                                      (when (seq asset-type)
-                                        (p/let [meta (worker-state/<invoke-main-thread
-                                                      :thread-api/get-asset-file-metadata
-                                                      repo (str asset-uuid) asset-type)]
-                                          (when (nil? meta)
-                                            (download-remote-asset! repo graph-id asset-uuid asset-type))))
+                                      (-> (p/let [meta (when (seq asset-type)
+                                                         (worker-state/<invoke-main-thread
+                                                          :thread-api/get-asset-file-metadata
+                                                          repo (str asset-uuid) asset-type))]
+                                            (when (and (seq asset-type) (nil? meta))
+                                              (download-remote-asset! repo graph-id asset-uuid asset-type)))
+                                          (p/catch (fn [e]
+                                                     (log/error :db-sync/asset-download-failed
+                                                                {:repo repo
+                                                                 :asset-uuid asset-uuid
+                                                                 :error e}))))
                                       (p/recur (rest uuids))))))
                                (p/resolved nil)))))))
 
+(defn request-asset-download!
+  [repo asset-uuid]
+  (if-let [client (current-client repo)]
+    (enqueue-asset-downloads! repo client [asset-uuid])
+    (p/resolved nil)))
+
 (defn- enqueue-asset-initial-download!
   [repo client]
   (enqueue-asset-task! client
@@ -1270,7 +1287,6 @@
                   (when (> remote-tx local-tx)
                     (send! (:ws client) {:type "pull" :since local-tx}))
                   (enqueue-asset-sync! repo client)
-                  (enqueue-asset-initial-download! repo client)
                   (flush-pending! repo client))
         "online-users" (let [users (:online-users message)]
                          (when (and (some? users) (not (sequential? users)))
@@ -1394,8 +1410,7 @@
             (reset-reconnect! updated)
             (set-ws-state! updated :open)
             (send! ws {:type "hello" :client repo})
-            (enqueue-asset-sync! repo updated)
-            (enqueue-asset-initial-download! repo updated)))
+            (enqueue-asset-sync! repo updated)))
     (start-pull-loop! updated ws)))
 
 (defn stop!

src/main/frontend/worker/db_worker.cljs

@@ -427,6 +427,10 @@
   [editing-block-uuid]
   (db-sync/update-presence! editing-block-uuid))
 
+(def-thread-api :thread-api/db-sync-request-asset-download
+  [repo asset-uuid]
+  (db-sync/request-asset-download! repo asset-uuid))
+
 (def-thread-api :thread-api/db-sync-grant-graph-access
   [repo graph-id target-email]
   (db-sync/grant-graph-access! repo graph-id target-email))

src/main/frontend/worker/rtc/asset.cljs

@@ -135,7 +135,10 @@
                     (when-let [edata (ex-data e)]
                       ;; if download-url return 404, ignore this asset
                       (when (not= 404 (:status (:data edata)))
-                        (throw (ex-info "download asset error(not= 404)" e)))) ()))))
+                        (log/error :rtc/asset-download-failed
+                                   {:repo repo
+                                    :asset-uuid asset-uuid
+                                    :error e}))) ()))))
 
             (c.m/concurrent-exec-flow 5 (m/seed asset-uuid->url))
             (m/reduce (constantly nil)))))))

src/test/frontend/worker/db_sync_test.cljs

@@ -259,6 +259,26 @@
             (is (= "child 3" (:block/title (:block/parent child2'))))
             (is (= "parent" (:block/title (:block/parent child3'))))))))))
 
+(deftest hello-message-does-not-trigger-initial-asset-download-test
+  (let [called? (atom false)
+        client {:repo test-repo
+                :graph-id "graph-id"
+                :asset-queue (atom (p/resolved nil))
+                :inflight (atom [])
+                :ws {}}
+        raw (js/JSON.stringify (clj->js {:type "hello" :t 0}))]
+    (with-redefs [db-sync/enqueue-asset-initial-download!
+                  (fn [& _]
+                    (reset! called? true)
+                    (p/resolved nil))
+                  db-sync/enqueue-asset-sync! (fn [& _] (p/resolved nil))
+                  db-sync/flush-pending! (fn [& _] (p/resolved nil))
+                  db-sync/send! (fn [& _] nil)
+                  db-sync/broadcast-rtc-state! (fn [& _] nil)
+                  client-op/get-local-tx (fn [& _] 0)]
+      (#'db-sync/handle-message! test-repo client raw)
+      (is (false? @called?)))))
+
 (deftest ignore-missing-parent-update-after-local-delete-test
   (testing "remote parent retracted while local adds another child"
     (let [{:keys [conn client-ops-conn parent child1]} (setup-parent-child)