enhance: parsing progress bar

resources/css/common.css

@@ -1163,3 +1163,9 @@ html[data-theme='dark'] .keyboard-shortcut > code {
 .overflow-y-scroll {
     overflow-y: scroll;
 }
+
+.text-ellipsis-wrapper {
+    white-space: nowrap;
+    overflow: hidden;
+    text-overflow: ellipsis;
+}

src/main/frontend/components/sidebar.cljs

@@ -292,7 +292,7 @@
                     :route-match route-match})
 
      [:div#main-content-container.scrollbar-spacing.w-full.flex.justify-center.flex-row
-      
+
       [:div.cp__sidebar-main-content
        {:data-is-global-graph-pages global-graph-pages?
         :data-is-full-width         (or global-graph-pages?
@@ -327,6 +327,23 @@
 (defonce sidebar-inited? (atom false))
 ;; TODO: simplify logic
 
+(rum/defc parsing-state < rum/reactive
+  [current-repo]
+  (let [full-state (state/sub [:graph/parsing-state])
+        parsing-state (state/sub [:graph/parsing-state current-repo])]
+    (when (not= (:total parsing-state) (:finished parsing-state))
+     (let [finished (or (:finished parsing-state) 0)
+           total (:total parsing-state)
+           width (js/Math.round (* (.toFixed (/ finished total) 2) 100))
+           left-label [:div.flex.flex-row.font-bold
+                       (t :parsing-files)
+                       [:div.hidden.md:flex.flex-row
+                        [:span.mr-1 ": "]
+                        [:div.text-ellipsis-wrapper {:style {:max-width 300}}
+                         (util/node-path.basename
+                          (:current-parsing-file parsing-state))]]]]
+       (ui/progress-bar-with-label width left-label (str finished "/" total))))))
+
 (rum/defc main-content < rum/reactive db-mixins/query
   {:init (fn [state]
            (when-not @sidebar-inited?
@@ -346,12 +363,13 @@
            state)}
   []
   (let [default-home (get-default-home-if-valid)
-        parsing? (state/sub :repo/parsing-files?)
         current-repo (state/sub :git/current-repo)
         loading-files? (when current-repo (state/sub [:repo/loading-files? current-repo]))
         journals-length (state/sub :journals-length)
         latest-journals (db/get-latest-journals (state/get-current-repo) journals-length)]
     [:div
+     (parsing-state current-repo)
+
      (cond
        (and default-home
             (= :home (state/get-current-route))
@@ -364,9 +382,6 @@
             (empty? latest-journals))
        (route-handler/redirect! {:to :all-pages})
 
-       parsing?
-       (ui/loading (t :parsing-files))
-
        loading-files?
        (ui/loading (t :loading-files))
 
@@ -507,7 +522,7 @@
                    current-repo
                    (not (state/sub :modal/show?)))
           (footer/footer))]
-       
+
        (right-sidebar/sidebar)
 
        [:div#app-single-container]]

src/main/frontend/fs/watcher_handler.cljs

@@ -92,7 +92,7 @@
           (and (contains? #{"add"} type)
                (= path pages-metadata-path))
           (p/do! (repo-handler/update-pages-metadata! repo content true))
-          
+
           ;; Change is triggered by external changes, so update to the db
           ;; Don't forced update when db is online, but resolving conflicts
           (and (contains? #{"change"} type)

src/main/frontend/handler/repo.cljs

@@ -28,7 +28,8 @@
             [shadow.resource :as rc]
             [frontend.db.persist :as db-persist]
             [electron.ipc :as ipc]
-            [clojure.set :as set]))
+            [clojure.set :as set]
+            [clojure.core.async :as async]))
 
 ;; Project settings should be checked in two situations:
 ;; 1. User changes the config.edn directly in logseq.com (fn: alter-file)
@@ -186,6 +187,37 @@
         file-paths [path]]
     (util/profile "update-pages-metadata!" (load-pages-metadata! repo file-paths files force?))))
 
+(defn- parse-and-load-file!
+  [repo-url file new-graph? metadata]
+  (try
+    (file-handler/alter-file repo-url
+                             (:file/path file)
+                             (:file/content file)
+                             {:new-graph? new-graph?
+                              :re-render-root? false
+                              :from-disk? true
+                              :metadata metadata})
+    (catch :default e
+      (state/set-parsing-state! (fn [m]
+                                  (update m :failed-parsing-files conj [(:file/path file) e])))))
+  (state/set-parsing-state! (fn [m]
+                              (-> (update m :finished inc)
+                                  (assoc :current-parsing-file (:file/path file))))))
+
+(defn- after-parse
+  [repo-url files file-paths first-clone? db-encrypted? re-render? re-render-opts opts graph-added-chan]
+  (state/reset-parsing-state!)
+  (load-pages-metadata! repo-url file-paths files true)
+  (when first-clone?
+    (if (and (not db-encrypted?) (state/enable-encryption? repo-url))
+      (state/pub-event! [:modal/encryption-setup-dialog repo-url
+                         #(create-default-files! repo-url %)])
+      (create-default-files! repo-url db-encrypted?)))
+  (when re-render?
+    (ui-handler/re-render-root! re-render-opts))
+  (state/pub-event! [:graph/added repo-url opts])
+  (async/offer! graph-added-chan true))
+
 (defn- parse-files-and-create-default-files-inner!
   [repo-url files delete-files delete-blocks file-paths first-clone? db-encrypted? re-render? re-render-opts metadata opts]
   (let [support-files (filter
@@ -194,28 +226,33 @@
                            (contains? (set/union #{:edn :css} config/mldoc-support-formats) format)))
                        files)
         support-files (sort-by :file/path support-files)
+        {journals true non-journals false} (group-by (fn [file] (string/includes? (:file/path file) "journals/")) support-files)
+        {built-in true others false} (group-by (fn [file]
+                                                 (or (string/includes? (:file/path file) "contents.")
+                                                     (string/includes? (:file/path file) ".edn")
+                                                     (string/includes? (:file/path file) "custom.css"))) non-journals)
+        support-files' (concat (reverse journals) built-in others)
         new-graph? (:new-graph? opts)
         delete-data (->> (concat delete-files delete-blocks)
-                         (remove nil?))]
+                         (remove nil?))
+        chan (async/to-chan! support-files')
+        graph-added-chan (async/promise-chan)]
     (when (seq delete-data) (db/transact! repo-url delete-data))
-    (doseq [file support-files]
-      (file-handler/alter-file repo-url
-                               (:file/path file)
-                               (:file/content file)
-                               {:new-graph? new-graph?
-                                :re-render-root? false
-                                :from-disk? true
-                                :metadata metadata}))
-    (load-pages-metadata! repo-url file-paths files true)
-    (when first-clone?
-      (if (and (not db-encrypted?) (state/enable-encryption? repo-url))
-        (state/pub-event! [:modal/encryption-setup-dialog repo-url
-                           #(create-default-files! repo-url %)])
-        (create-default-files! repo-url db-encrypted?)))
-    (when re-render?
-      (ui-handler/re-render-root! re-render-opts))
-    (state/pub-event! [:graph/added repo-url opts])
-    (state/set-parsing-files! false)))
+    (state/set-current-repo! repo-url)
+    (state/set-parsing-state! {:total (count support-files')})
+    (if util/node-test?
+      (do
+        (doseq [file support-files']
+          (parse-and-load-file! repo-url file new-graph? metadata))
+        (after-parse repo-url files file-paths first-clone? db-encrypted? re-render? re-render-opts opts graph-added-chan))
+      (async/go-loop []
+        (if-let [file (async/<! chan)]
+          (do
+            (parse-and-load-file! repo-url file new-graph? metadata)
+            (async/<! (async/timeout 1))
+            (recur))
+          (after-parse repo-url files file-paths first-clone? db-encrypted? re-render? re-render-opts opts graph-added-chan))))
+    graph-added-chan))
 
 (defn- parse-files-and-create-default-files!
   [repo-url files delete-files delete-blocks file-paths first-clone? db-encrypted? re-render? re-render-opts metadata opts]
@@ -224,39 +261,34 @@
                    (map (fn [file]
                           (p/let [content (encrypt/decrypt (:file/content file))]
                             (assoc file :file/content content)))
-                        files))]
+                     files))]
       (parse-files-and-create-default-files-inner! repo-url files delete-files delete-blocks file-paths first-clone? db-encrypted? re-render? re-render-opts metadata opts))
     (parse-files-and-create-default-files-inner! repo-url files delete-files delete-blocks file-paths first-clone? db-encrypted? re-render? re-render-opts metadata opts)))
 
 (defn- update-parsing-state!
-  [repo-url refresh?]
-  (state/set-loading-files! repo-url false)
-  (when-not refresh? (state/set-parsing-files! true)))
+  [repo-url]
+  (state/set-loading-files! repo-url false))
 
 (defn parse-files-and-load-to-db!
-  [repo-url files {:keys [first-clone? delete-files delete-blocks re-render? re-render-opts refresh?] :as opts
+  [repo-url files {:keys [first-clone? delete-files delete-blocks re-render? re-render-opts _refresh?] :as opts
                    :or {re-render? true}}]
-  (update-parsing-state! repo-url refresh?)
-
-  (let [f (fn []
-            (let [file-paths (map :file/path files)
-                  metadata-file (config/get-metadata-path)
-                  metadata-content (some #(when (= (:file/path %) metadata-file)
-                                            (:file/content %)) files)
-                  metadata (when metadata-content
-                             (common-handler/read-metadata! metadata-content))
-                  db-encrypted? (:db/encrypted? metadata)
-                  db-encrypted-secret (if db-encrypted? (:db/encrypted-secret metadata) nil)]
-              (if db-encrypted?
-                (let [close-fn #(parse-files-and-create-default-files! repo-url files delete-files delete-blocks file-paths first-clone? db-encrypted? re-render? re-render-opts metadata opts)]
-                  (state/set-state! :encryption/graph-parsing? true)
-                  (state/pub-event! [:modal/encryption-input-secret-dialog repo-url
-                                     db-encrypted-secret
-                                     close-fn]))
-                (parse-files-and-create-default-files! repo-url files delete-files delete-blocks file-paths first-clone? db-encrypted? re-render? re-render-opts metadata opts))))]
-    (if util/node-test?
-      (f)
-      (js/setTimeout f 100))))
+  (update-parsing-state! repo-url)
+
+  (let [file-paths (map :file/path files)
+        metadata-file (config/get-metadata-path)
+        metadata-content (some #(when (= (:file/path %) metadata-file)
+                                  (:file/content %)) files)
+        metadata (when metadata-content
+                   (common-handler/read-metadata! metadata-content))
+        db-encrypted? (:db/encrypted? metadata)
+        db-encrypted-secret (if db-encrypted? (:db/encrypted-secret metadata) nil)]
+    (if db-encrypted?
+      (let [close-fn #(parse-files-and-create-default-files! repo-url files delete-files delete-blocks file-paths first-clone? db-encrypted? re-render? re-render-opts metadata opts)]
+        (state/set-state! :encryption/graph-parsing? true)
+        (state/pub-event! [:modal/encryption-input-secret-dialog repo-url
+                           db-encrypted-secret
+                           close-fn]))
+      (parse-files-and-create-default-files! repo-url files delete-files delete-blocks file-paths first-clone? db-encrypted? re-render? re-render-opts metadata opts))))
 
 (defn load-repo-to-db!
   [repo-url {:keys [first-clone? diffs nfs-files refresh? new-graph? empty-graph?]}]
@@ -684,4 +716,4 @@
 (defn graph-ready!
   "Call electron that the graph is loaded."
   [graph]
-  (ipc/ipc "graphReady" graph))
+  (ipc/ipc "graphReady" graph))

src/main/frontend/handler/web/nfs.cljs

@@ -20,7 +20,8 @@
             [goog.object :as gobj]
             [lambdaisland.glogi :as log]
             [promesa.core :as p]
-            [frontend.mobile.util :as mobile-util]))
+            [frontend.mobile.util :as mobile-util]
+            [clojure.core.async :as async]))
 
 (defn remove-ignore-files
   [files]
@@ -172,16 +173,17 @@
                (p/then (fn [result]
                          (let [files (map #(dissoc % :file/file) result)]
                            (repo-handler/start-repo-db-if-not-exists! repo {:db-type :local-native-fs})
-                           (p/let [_ (repo-handler/load-repo-to-db! repo
-                                                                    {:first-clone? true
-                                                                     :new-graph?   true
-                                                                     :empty-graph? (nil? (seq markup-files))
-                                                                     :nfs-files    files})]
-                             (state/add-repo! {:url repo :nfs? true})
-                             (state/set-loading-files! repo false)
-                             (when ok-handler (ok-handler))
-                             (fs/watch-dir! dir-name)
-                             (db/persist-if-idle! repo)))))
+                           (async/go
+                             (let [_finished? (async/<! (repo-handler/load-repo-to-db! repo
+                                                                                       {:first-clone? true
+                                                                                        :new-graph?   true
+                                                                                        :empty-graph? (nil? (seq markup-files))
+                                                                                        :nfs-files    files}))]
+                               (state/add-repo! {:url repo :nfs? true})
+                               (state/set-loading-files! repo false)
+                               (when ok-handler (ok-handler))
+                               (fs/watch-dir! dir-name)
+                               (db/persist-if-idle! repo))))))
                (p/catch (fn [error]
                           (log/error :nfs/load-files-error repo)
                           (log/error :exception error)))))))

src/main/frontend/state.cljs

@@ -35,7 +35,6 @@
      :repo/cloning?                         false
      ;; :repo/loading-files? is only for github repos
      :repo/loading-files?                   {}
-     :repo/parsing-files?                   nil
      :repo/changed-files                    nil
      :nfs/user-granted?                     {}
      :nfs/refreshing?                       nil
@@ -184,6 +183,8 @@
      ;; all notification contents as k-v pairs
      :notification/contents                 {}
      :graph/syncing?                        false
+     ;; graph -> state
+     :graph/parsing-state                   {}
 
      ;; copied blocks
      :copy/blocks                           {:copy/content nil :copy/block-ids nil}
@@ -1331,10 +1332,6 @@
   [repo]
   (get-in @state [:repo/loading-files? repo]))
 
-(defn set-parsing-files!
-  [value]
-  (set-state! :repo/parsing-files? value))
-
 (defn set-editor-last-input-time!
   [repo time]
   (swap! state assoc-in [:editor/last-input-time repo] time))
@@ -1699,3 +1696,13 @@
   (:file-sync/sync-manager @state))
 (defn get-file-sync-state []
   (:file-sync/sync-state @state))
+
+(defn reset-parsing-state!
+  []
+  (set-state! [:graph/parsing-state (get-current-repo)] {}))
+
+(defn set-parsing-state!
+  [m]
+  (update-state! [:graph/parsing-state (get-current-repo)]
+                 (if (fn? m) m
+                   (fn [old-value] (merge old-value m)))))

src/main/frontend/ui.cljs

@@ -878,3 +878,21 @@
         :arrow       true}
        content)
       content)))
+
+(rum/defc progress-bar
+  [width]
+  {:pre (integer? width)}
+  [:div.w-full.bg-indigo-200.rounded-full.h-2.5
+   [:div.bg-indigo-600.h-2.5.rounded-full {:style {:width (str width "%")}
+                                           :transition "width 1s"}]])
+
+(rum/defc progress-bar-with-label
+  [width label-left label-right]
+  {:pre (integer? width)}
+  [:div
+   [:div.flex.justify-between.mb-1
+    [:span.text-base
+     label-left]
+    [:span.text-sm.font-medium
+     label-right]]
+   (progress-bar width)])