fix: async <get-file and <get-files

deps/db/src/logseq/db/frontend/schema.cljs

@@ -113,6 +113,8 @@
    :file/path {:db/unique :db.unique/identity}
    ;; only store the content of logseq's files
    :file/content {}
+
+   ;; TODO: do we really use this?
    :file/handle {}
    ;; :file/created-at {}
    ;; :file/last-modified-at {}

deps/db/src/logseq/db/sqlite/common_db.cljs

@@ -20,6 +20,13 @@
        (map (fn [e]
               (d/pull db '[*] (:e e))))))
 
+(defn get-all-files
+  [db]
+  (->> (d/datoms db :avet :file/path)
+       (map (fn [e]
+              {:db/id (:e e)
+               :file/path (:v e)}))))
+
 (defn get-block-and-children
   [db name children?]
   (let [get-children (fn [col]

src/main/frontend/components/block.cljs

@@ -813,7 +813,7 @@
 (rum/defc block-embed < rum/reactive
   {:init (fn [state]
            (let [block-id (second (:rum/args state))]
-             (db-async/<get-block-and-children (state/get-current-repo) block-id))
+             (db-async/<get-block (state/get-current-repo) block-id))
            state)}
   [config uuid]
   (if (state/sub-async-query-loading (str uuid))
@@ -837,7 +837,7 @@
   {:init (fn [state]
            (let [page-name (second (:rum/args state))
                  page-name' (util/page-name-sanity-lc (string/trim page-name))]
-             (db-async/<get-block-and-children (state/get-current-repo) page-name'))
+             (db-async/<get-block (state/get-current-repo) page-name'))
            state)}
   [config page-name]
   (let [page-name (util/page-name-sanity-lc (string/trim page-name))
@@ -893,7 +893,7 @@
 (rum/defc block-reference < rum/reactive
   {:init (fn [state]
            (let [block-id (second (:rum/args state))]
-             (db-async/<get-block-and-children (state/get-current-repo) block-id :children? false))
+             (db-async/<get-block (state/get-current-repo) block-id :children? false))
            state)}
   db-mixins/query
   [config id label]
@@ -3114,7 +3114,7 @@
   {:init (fn [state]
            (let [[config block] (:rum/args state)
                  block-id (:block/uuid block)]
-             (db-async/<get-block-and-children (state/get-current-repo) block-id :children? false)
+             (db-async/<get-block (state/get-current-repo) block-id :children? false)
              (cond
                (root-block? config block)
                (state/set-collapsed-block! block-id false)

src/main/frontend/components/cmdk.cljs

@@ -417,7 +417,7 @@
 (defmethod handle-action :open-block [_ state _event]
   (when-let [block-id (some-> state state->highlighted-item :source-block :block/uuid)]
     (p/let [repo (state/get-current-repo)
-            _ (db-async/<get-block-and-children repo block-id)]
+            _ (db-async/<get-block repo block-id)]
       (let [get-block-page (partial model/get-block-page repo)
            block (db/entity [:block/uuid block-id])]
        (when block

src/main/frontend/components/file.cljs

@@ -20,43 +20,48 @@
             [promesa.core :as p]
             [reitit.frontend.easy :as rfe]
             [rum.core :as rum]
-            [logseq.common.path :as path]))
+            [logseq.common.path :as path]
+            [frontend.db.async :as db-async]))
 
 (defn- get-path
   [state]
   (let [route-match (first (:rum/args state))]
     (get-in route-match [:parameters :path :path])))
 
-(rum/defc files-all < rum/reactive
-  []
-  (when-let [current-repo (state/sub :git/current-repo)]
-    (let [files (db/get-files current-repo) ; [[string]]
-          files (sort-by first gstring/intAwareCompare files)
-          mobile? (util/mobile?)]
-      [:table.table-auto
-       [:thead
-        [:tr
-         [:th (t :file/name)]
-         (when-not mobile?
-           [:th (t :file/last-modified-at)])
-         (when-not mobile?
-           [:th ""])]]
-       [:tbody
-        (for [[file modified-at] files]
-          (let [file-id file]
-            [:tr {:key file-id}
-             [:td
-              (let [href (if (common-config/draw? file)
-                           (rfe/href :draw nil {:file (string/replace file (str common-config/default-draw-directory "/") "")})
-                           (rfe/href :file {:path file-id}))]
-                [:a {:href href}
-                 file])]
-             (when-not mobile?
-               [:td [:span.text-gray-500.text-sm
-                     (if (or (nil? modified-at) (zero? modified-at))
-                       (t :file/no-data)
-                       (date/get-date-time-string
-                        (t/to-default-time-zone (tc/to-date-time modified-at))))]])]))]])))
+(rum/defcs files-all < rum/reactive
+  {:init (fn [state]
+           (let [*files (atom nil)]
+             (p/let [result (db-async/<get-files (state/get-current-repo))]
+               (reset! *files result))
+             (assoc state ::files *files)))}
+  [state]
+  (let [files (rum/react (::files state))
+        files (sort-by first gstring/intAwareCompare files)
+        mobile? (util/mobile?)]
+    [:table.table-auto
+     [:thead
+      [:tr
+       [:th (t :file/name)]
+       (when-not mobile?
+         [:th (t :file/last-modified-at)])
+       (when-not mobile?
+         [:th ""])]]
+     [:tbody
+      (for [[file modified-at] files]
+        (let [file-id file]
+          [:tr {:key file-id}
+           [:td
+            (let [href (if (common-config/draw? file)
+                         (rfe/href :draw nil {:file (string/replace file (str common-config/default-draw-directory "/") "")})
+                         (rfe/href :file {:path file-id}))]
+              [:a {:href href}
+               file])]
+           (when-not mobile?
+             [:td [:span.text-gray-500.text-sm
+                   (if (or (nil? modified-at) (zero? modified-at))
+                     (t :file/no-data)
+                     (date/get-date-time-string
+                      (t/to-default-time-zone (tc/to-date-time modified-at))))]])]))]]))
 
 (rum/defc files
   []

src/main/frontend/components/page.cljs

@@ -456,7 +456,7 @@
   {:init (fn [state]
            (let [page-name (:page-name (first (:rum/args state)))
                  page-name' (get-sanity-page-name state page-name)]
-             (db-async/<get-block-and-children (state/get-current-repo) page-name')
+             (db-async/<get-block (state/get-current-repo) page-name')
              (assoc state ::page-name page-name')))}
   [state {:keys [repo page-name preview? sidebar?] :as option}]
   (when-not (state/sub-async-query-loading (::page-name state))
@@ -573,7 +573,7 @@
 
 (rum/defc contents-page < rum/reactive
   {:init (fn [state]
-           (db-async/<get-block-and-children (state/get-current-repo) "contents")
+           (db-async/<get-block (state/get-current-repo) "contents")
            state)}
   [page]
   (when-let [repo (state/get-current-repo)]

src/main/frontend/components/property/value.cljs

@@ -446,7 +446,7 @@
 (rum/defc property-template-value < rum/reactive
   {:init (fn [state]
            (let [block-id (second (:rum/args state))]
-             (db-async/<get-block-and-children (state/get-current-repo) block-id :children? false))
+             (db-async/<get-block (state/get-current-repo) block-id :children? false))
            state)}
   [config value opts]
   (when value
@@ -468,7 +468,7 @@
   (rum/local nil ::template-instance)
   {:init (fn [state]
            (let [block-id (first (:rum/args state))]
-             (db-async/<get-block-and-children (state/get-current-repo) block-id :children? false))
+             (db-async/<get-block (state/get-current-repo) block-id :children? false))
            state)}
   [state value block property block-cp editor-box opts page-cp editor-id]
   (let [*template-instance (::template-instance state)
@@ -504,7 +504,7 @@
 (rum/defc closed-value-item < rum/reactive
   {:init (fn [state]
            (let [block-id (first (:rum/args state))]
-             (db-async/<get-block-and-children (state/get-current-repo) block-id :children? false))
+             (db-async/<get-block (state/get-current-repo) block-id :children? false))
            state)}
   [value {:keys [page-cp inline-text icon?]}]
   (when value

src/main/frontend/components/whiteboard.cljs

@@ -41,7 +41,7 @@
            (p/let [_ (loader/load :tldraw)]
              (reset! tldraw-loaded? true))
            (let [page-name (first (:rum/args state))]
-             (db-async/<get-block-and-children (state/get-current-repo) page-name))
+             (db-async/<get-block (state/get-current-repo) page-name))
            state)}
   [page-name]
   (let [loaded? (rum/react tldraw-loaded?)

src/main/frontend/db.cljs

@@ -36,7 +36,7 @@
   get-block-immediate-children get-block-page
   get-custom-css get-date-scheduled-or-deadlines
   get-file-last-modified-at get-file get-file-page get-file-page-id file-exists?
-  get-files get-files-blocks get-files-full get-journals-length get-pages-with-file
+  get-files-blocks get-files-full get-journals-length get-pages-with-file
   get-latest-journals get-page get-page-alias get-page-alias-names
   get-page-blocks-count get-page-blocks-no-cache get-page-file get-page-format get-page-properties
   get-page-referenced-blocks get-page-referenced-blocks-full get-page-referenced-pages

src/main/frontend/db/async.cljs

@@ -16,6 +16,7 @@
             [frontend.db.react :as react]))
 
 (def <q db-async-util/<q)
+(def <pull db-async-util/<pull)
 (comment
   (def <pull-many db-async-util/<pull-many))
 
@@ -106,7 +107,7 @@
     (<get-db-based-property-values graph property)
     (file-async/<get-file-based-property-values graph property)))
 
-(defn <get-block-and-children
+(defn <get-block
   [graph name-or-uuid & {:keys [children?]
                          :or {children? true}}]
   (let [name' (str name-or-uuid)
@@ -151,8 +152,15 @@
 (defn <get-all-referenced-blocks-uuid
   "Get all uuids of blocks with any back link exists."
   [graph]
-  (<q '[:find [?refed-uuid ...]
+  (<q graph
+      '[:find [?refed-uuid ...]
         :where
            ;; ?referee-b is block with ref towards ?refed-b
         [?refed-b   :block/uuid ?refed-uuid]
         [?referee-b :block/refs ?refed-b]]))
+
+(defn <get-file
+  [graph path]
+  (when (and graph path)
+    (p/let [result (<pull graph [:file/path path])]
+      (:file/content result))))

src/main/frontend/db/async/util.cljs

@@ -12,10 +12,20 @@
       (when result
         (edn/read-string result)))))
 
-(defn <pull-many
-  [graph selector ids]
-  (assert (seq ids))
-  (when-let [^Object sqlite @state/*db-worker]
-    (p/let [result (.pull-many sqlite graph (pr-str selector) (pr-str ids))]
-      (when result
-        (edn/read-string result)))))
+(defn <pull
+  ([graph id]
+   (<pull graph '[*] id))
+  ([graph selector id]
+   (when-let [^Object sqlite @state/*db-worker]
+     (p/let [result (.pull sqlite graph (pr-str selector) (pr-str id))]
+       (when result
+         (edn/read-string result))))))
+
+(comment
+  (defn <pull-many
+   [graph selector ids]
+   (assert (seq ids))
+   (when-let [^Object sqlite @state/*db-worker]
+     (p/let [result (.pull-many sqlite graph (pr-str selector) (pr-str ids))]
+       (when result
+         (edn/read-string result))))))

src/main/frontend/db/model.cljs

@@ -117,18 +117,6 @@
   (when-let [db (conn/get-db repo)]
     (ldb/get-alias-source-page db alias)))
 
-(defn get-files
-  [repo]
-  (when-let [db (conn/get-db repo)]
-    (->> (d/q
-          '[:find ?path ?modified-at
-            :where
-            [?file :file/path ?path]
-            [(get-else $ ?file :file/last-modified-at 0) ?modified-at]]
-          db)
-         (seq)
-         (reverse))))
-
 (defn get-files-blocks
   [repo-url paths]
   (let [paths (set paths)

src/main/frontend/db_worker.cljs

@@ -285,6 +285,14 @@
              result (apply d/q (first inputs) @conn (rest inputs))]
          (pr-str result))))
 
+  (pull
+   [_this repo selector-str id-str]
+   (when-let [conn (worker-state/get-datascript-conn repo)]
+     (let [selector (edn/read-string selector-str)
+           id (edn/read-string id-str)
+           result (d/pull @conn selector id)]
+       (pr-str result))))
+
   (pull-many
    [_this repo selector-str ids-str]
    (when-let [conn (worker-state/get-datascript-conn repo)]
@@ -366,10 +374,11 @@
      (let [data (->> (sqlite-common-db/get-initial-data @conn)
                      pr-str)]
        (async/go
-         (let [all-pages (sqlite-common-db/get-all-pages @conn)]
+         (let [all-pages (sqlite-common-db/get-all-pages @conn)
+               all-files (sqlite-common-db/get-all-files @conn)]
            (worker-util/post-message :sync-db-changes (pr-str
                                                        {:repo repo
-                                                        :tx-data all-pages
+                                                        :tx-data (concat all-files all-pages)
                                                         :tx-meta nil}))))
        data)))
 

src/main/frontend/extensions/tldraw.cljs

@@ -204,7 +204,7 @@
 (rum/defc tldraw-app-inner < rum/reactive
   {:init (fn [state]
            (let [page-name (first (:rum/args state))]
-             (db-async/<get-block-and-children (state/get-current-repo) page-name)
+             (db-async/<get-block (state/get-current-repo) page-name)
              state))}
   [page-name block-id loaded-app set-loaded-app]
   (when-not (state/sub-async-query-loading page-name)

src/main/frontend/fs/watcher_handler.cljs

@@ -21,7 +21,8 @@
             [lambdaisland.glogi :as log]
             [logseq.common.config :as common-config]
             [logseq.common.util.block-ref :as block-ref]
-            [promesa.core :as p]))
+            [promesa.core :as p]
+            [frontend.db.async :as db-async]))
 
 ;; all IPC paths must be normalized! (via common-util/path-normalize)
 
@@ -69,9 +70,9 @@
           {:keys [mtime]} stat
           ext (keyword (path/file-ext path))]
       (when (contains? #{:org :md :markdown :css :js :edn :excalidraw :tldr} ext)
-        (let [db-content (db/get-file repo path)
-              exists-in-db? (not (nil? db-content))
-              db-content (or db-content "")]
+        (p/let [db-content (db-async/<get-file repo path)
+                exists-in-db? (not (nil? db-content))
+                db-content (or db-content "")]
           (when (or content (contains? #{"unlink" "unlinkDir" "addDir"} type))
             (cond
               (and (= "unlinkDir" type) dir)
@@ -172,7 +173,7 @@
                              (:mtime (fs/stat repo-dir file-rpath)))
                 db-empty? (db/page-empty? repo page-name)
                 db-content (if-not db-empty?
-                             (db/get-file repo file-rpath)
+                             (db-async/<get-file repo file-rpath)
                              "")]
           (p/let [_ (cond
                       (and file-exists?
@@ -193,11 +194,11 @@
   [graph exclude-files]
   (when graph
     (let [repo-dir (config/get-repo-dir graph)
-          db-files (->> (db/get-files graph)
-                        (map first))
           exclude-files (set (or exclude-files []))]
       ;; read all files in the repo dir, notify if readdir error
-      (p/let [[files deleted-files]
+      (p/let [db-files' (db-async/<get-files graph)
+              db-files (map first db-files')
+              [files deleted-files]
               (-> (fs/readdir repo-dir :path-only? true)
                   (p/chain (fn [files]
                              (->> files

src/main/frontend/handler.cljs

@@ -11,7 +11,6 @@
             [frontend.components.whiteboard :as whiteboard]
             [frontend.config :as config]
             [frontend.context.i18n :as i18n]
-            [frontend.db :as db]
             [frontend.db.restore :as db-restore]
             [frontend.db.conn :as conn]
             [frontend.db.react :as react]
@@ -41,7 +40,8 @@
             [frontend.mobile.core :as mobile]
             [cljs-bean.core :as bean]
             [frontend.handler.test :as test]
-            [frontend.persist-db.browser :as db-browser]))
+            [frontend.persist-db.browser :as db-browser]
+            [frontend.db.async :as db-async]))
 
 (defn- set-global-error-notification!
   []
@@ -87,17 +87,18 @@
                 ;; install after config is restored
                 (shortcut/refresh!)
 
-                (cond
-                  (and (not (seq (db/get-files config/demo-repo)))
-                       ;; Not native local directory
-                       (not (some config/local-file-based-graph? (map :url repos)))
-                       (not (mobile-util/native-platform?))
-                       (not (config/db-based-graph? repo)))
-                  ;; will execute `(state/set-db-restoring! false)` inside
-                  (repo-handler/setup-demo-repo-if-not-exists!)
-
-                  :else
-                  (state/set-db-restoring! false)))))))
+                (p/let [files (db-async/<get-files config/demo-repo)]
+                  (cond
+                    (and (not (seq files))
+                         ;; Not native local directory
+                         (not (some config/local-file-based-graph? (map :url repos)))
+                         (not (mobile-util/native-platform?))
+                         (not (config/db-based-graph? repo)))
+                    ;; will execute `(state/set-db-restoring! false)` inside
+                    (repo-handler/setup-demo-repo-if-not-exists!)
+
+                    :else
+                    (state/set-db-restoring! false))))))))
         (p/then
          (fn []
            (js/console.log "db restored, setting up repo hooks")