enhance: lazy load all pages

deps/db/src/logseq/db/sqlite/common_db.cljs

@@ -7,12 +7,13 @@
             [logseq.common.util.date-time :as date-time-util]
             [logseq.common.util :as common-util]))
 
-(defn- get-built-in-files
-  [db]
-  (let [files ["logseq/config.edn"
-               "logseq/custom.css"
-               "logseq/custom.js"]]
-    (map #(d/pull db '[*] [:file/path %]) files)))
+(comment
+  (defn- get-built-in-files
+    [db]
+    (let [files ["logseq/config.edn"
+                 "logseq/custom.css"
+                 "logseq/custom.js"]]
+      (map #(d/pull db '[*] [:file/path %]) files))))
 
 (defn get-all-pages
   [db]
@@ -28,15 +29,29 @@
                :file/path (:v e)
                :file/content (:file/content (d/entity db (:e e)))}))))
 
+(defn- with-block-refs
+  [db block]
+  (update block :block/refs (fn [refs]
+                              (map (fn [ref]
+                                     (let [e (d/entity db (:db/id ref))]
+                                       (if (and e (:block/name e))
+                                         (assoc ref
+                                                :block/uuid (:block/uuid e)
+                                                :block/original-name (:block/name e)
+                                                :block/name (:block/name e))
+                                         ref))) refs))))
+
 (defn with-parent-and-left
   [db block]
   (cond
     (:block/name block)
     block
     (:block/page block)
-    (assoc block
-          :block/left (select-keys (d/entity db (:db/id (:block/left block))) [:db/id :block/uuid])
-          :block/parent (select-keys (d/entity db (:db/id (:block/parent block))) [:db/id :block/uuid]))
+    (->>
+     (assoc block
+            :block/left (select-keys (d/entity db (:db/id (:block/left block))) [:db/id :block/uuid])
+            :block/parent (select-keys (d/entity db (:db/id (:block/parent block))) [:db/id :block/uuid]))
+     (with-block-refs db))
     :else
     block))
 
@@ -96,10 +111,10 @@
 (defn get-initial-data
   "Returns initial data"
   [db]
-  (let [all-pages (get-all-pages db)
+  (let [latest-journals (get-latest-journals db 3)
         all-files (get-all-files db)
         closed-values (get-closed-values db)]
-    (concat all-pages all-files closed-values)))
+    (concat latest-journals all-files closed-values)))
 
 (defn restore-initial-data
   "Given initial sqlite data and schema, returns a datascript connection"

src/main/frontend/components/cmdk.cljs

@@ -417,7 +417,7 @@
 (defmethod handle-action :open-block [_ state _event]
   (when-let [block-id (some-> state state->highlighted-item :source-block :block/uuid)]
     (p/let [repo (state/get-current-repo)
-            _ (db-async/<get-block repo block-id)]
+            _ (db-async/<get-block repo block-id :children? false)]
       (let [get-block-page (partial model/get-block-page repo)
            block (db/entity [:block/uuid block-id])]
        (when block
@@ -442,8 +442,10 @@
 
 (defmethod handle-action :open-block-right [_ state _event]
   (when-let [block-uuid (some-> state state->highlighted-item :source-block :block/uuid)]
-    (editor-handler/open-block-in-sidebar! block-uuid)
-    (state/close-modal!)))
+    (p/let [repo (state/get-current-repo)
+            _ (db-async/<get-block repo block-uuid :children? false)]
+      (editor-handler/open-block-in-sidebar! block-uuid)
+      (state/close-modal!))))
 
 (defn- open-file
   [file-path]

src/main/frontend/components/page.cljs

@@ -37,6 +37,7 @@
             [frontend.search :as search]
             [frontend.state :as state]
             [frontend.ui :as ui]
+            [logseq.shui.ui :as shui-ui]
             [frontend.util :as util]
             [frontend.util.text :as text-util]
             [goog.object :as gobj]
@@ -459,114 +460,119 @@
              (db-async/<get-block (state/get-current-repo) page-name')
              (assoc state ::page-name page-name')))}
   [state {:keys [repo page-name preview? sidebar?] :as option}]
-  (when-not (state/sub-async-query-loading (::page-name state))
+  (let [loading? (state/sub-async-query-loading (::page-name state))]
     (when-let [path-page-name (get-path-page-name state page-name)]
       (let [current-repo (state/sub :git/current-repo)
             repo (or repo current-repo)
             page-name (util/page-name-sanity-lc path-page-name)
-            page (get-page-entity page-name)
-            block-id (:block/uuid page)
-            block? (some? (:block/page page))
-            journal? (db/journal-page? page-name)
-            db-based? (config/db-based-graph? repo)
-            fmt-journal? (boolean (date/journal-title->int page-name))
-            whiteboard? (:whiteboard? option) ;; in a whiteboard portal shape?
-            whiteboard-page? (model/whiteboard-page? page-name) ;; is this page a whiteboard?
-            route-page-name path-page-name
-            page-name (:block/name page)
-            page-original-name (:block/original-name page)
-            title (or page-original-name page-name)
-            today? (and
-                    journal?
-                    (= page-name (util/page-name-sanity-lc (date/journal-name))))
-            *control-show? (::control-show? state)
-            *all-collapsed? (::all-collapsed? state)
-            *current-block-page (::current-page state)
-            block-or-whiteboard? (or block? whiteboard?)
-            home? (= :home (state/get-current-route))]
-        (when (or page-name block-or-whiteboard?)
-          [:div.flex-1.page.relative
-           (merge (if (seq (:block/tags page))
-                    (let [page-names (model/get-page-names-by-ids (map :db/id (:block/tags page)))]
-                      {:data-page-tags (text-util/build-data-value page-names)})
-                    {})
-
-                  {:key path-page-name
-                   :class (util/classnames [{:is-journals (or journal? fmt-journal?)}])})
-
-           (if (and whiteboard-page? (not sidebar?))
-             [:div ((state/get-component :whiteboard/tldraw-preview) page-name)] ;; FIXME: this is not reactive
-             [:div.relative
-              (when (and (not sidebar?) (not block?))
-                [:div.flex.flex-row.space-between
-                 (when (or (mobile-util/native-platform?) (util/mobile?))
-                   [:div.flex.flex-row.pr-2
-                    {:style {:margin-left -15}
-                     :on-mouse-over (fn [e]
-                                      (page-mouse-over e *control-show? *all-collapsed?))
-                     :on-mouse-leave (fn [e]
-                                       (page-mouse-leave e *control-show?))}
-                    (page-blocks-collapse-control title *control-show? *all-collapsed?)])
-                 (let [original-name (:block/original-name (db/entity [:block/name (util/page-name-sanity-lc page-name)]))]
-                   (when (and (not whiteboard?) original-name)
-                     (page-title page-name {:journal? journal?
-                                            :fmt-journal? fmt-journal?
-                                            :preview? preview?
-                                            :*hover? (::hover-title? state)})))
-                 (when (not config/publishing?)
-                   (when config/lsp-enabled?
-                     [:div.flex.flex-row
-                      (plugins/hook-ui-slot :page-head-actions-slotted nil)
-                      (plugins/hook-ui-items :pagebar)]))])
-
-              (cond
-                (and db-based? (not block?))
-                [:div.pb-4
-                 (db-page/page-info page (::hover-title? state))]
-
-                (and (not db-based?) (not block?))
-                [:div.pb-4])
-
-              [:div
-               (when (and block? (not sidebar?) (not whiteboard?))
-                 (let [config {:id "block-parent"
-                               :block? true}]
-                   [:div.mb-4
-                    (component-block/breadcrumb config repo block-id {:level-limit 3})]))
-
-               (when (and db-based? (not block?) (not preview?))
-                 (db-page/page-properties-react page {:configure? false}))
-
-           ;; blocks
-               (let [_ (and block? page (reset! *current-block-page (:block/name (:block/page page))))
-                     _ (when (and block? (not page))
-                         (route-handler/redirect-to-page! @*current-block-page))]
-                 (page-blocks-cp repo page {:sidebar? sidebar? :whiteboard? whiteboard?}))]])
-
-           (when today?
-             (today-queries repo today? sidebar?))
-
-           (when today?
-             (scheduled/scheduled-and-deadlines page-name))
-
-           (when-not block?
-             (tagged-pages repo page-name page-original-name))
+            page (get-page-entity page-name)]
+        (when-not (and loading? (nil? page))
+          (let [block-id (:block/uuid page)
+                block? (some? (:block/page page))
+                journal? (db/journal-page? page-name)
+                db-based? (config/db-based-graph? repo)
+                fmt-journal? (boolean (date/journal-title->int page-name))
+                whiteboard? (:whiteboard? option) ;; in a whiteboard portal shape?
+                whiteboard-page? (model/whiteboard-page? page-name) ;; is this page a whiteboard?
+                route-page-name path-page-name
+                page-name (:block/name page)
+                page-original-name (:block/original-name page)
+                title (or page-original-name page-name)
+                today? (and
+                        journal?
+                        (= page-name (util/page-name-sanity-lc (date/journal-name))))
+                *control-show? (::control-show? state)
+                *all-collapsed? (::all-collapsed? state)
+                *current-block-page (::current-page state)
+                block-or-whiteboard? (or block? whiteboard?)
+                home? (= :home (state/get-current-route))]
+            (when (or page-name block-or-whiteboard?)
+              [:div.flex-1.page.relative
+               (merge (if (seq (:block/tags page))
+                        (let [page-names (model/get-page-names-by-ids (map :db/id (:block/tags page)))]
+                          {:data-page-tags (text-util/build-data-value page-names)})
+                        {})
+
+                      {:key path-page-name
+                       :class (util/classnames [{:is-journals (or journal? fmt-journal?)}])})
+
+               (if (and whiteboard-page? (not sidebar?))
+                 [:div ((state/get-component :whiteboard/tldraw-preview) page-name)] ;; FIXME: this is not reactive
+                 [:div.relative
+                  (when (and (not sidebar?) (not block?))
+                    [:div.flex.flex-row.space-between
+                     (when (or (mobile-util/native-platform?) (util/mobile?))
+                       [:div.flex.flex-row.pr-2
+                        {:style {:margin-left -15}
+                         :on-mouse-over (fn [e]
+                                          (page-mouse-over e *control-show? *all-collapsed?))
+                         :on-mouse-leave (fn [e]
+                                           (page-mouse-leave e *control-show?))}
+                        (page-blocks-collapse-control title *control-show? *all-collapsed?)])
+                     (let [original-name (:block/original-name (db/entity [:block/name (util/page-name-sanity-lc page-name)]))]
+                       (when (and (not whiteboard?) original-name)
+                         (page-title page-name {:journal? journal?
+                                                :fmt-journal? fmt-journal?
+                                                :preview? preview?
+                                                :*hover? (::hover-title? state)})))
+                     (when (not config/publishing?)
+                       (when config/lsp-enabled?
+                         [:div.flex.flex-row
+                          (plugins/hook-ui-slot :page-head-actions-slotted nil)
+                          (plugins/hook-ui-items :pagebar)]))])
+
+                  (cond
+                    (and db-based? (not block?))
+                    [:div.pb-4
+                     (db-page/page-info page (::hover-title? state))]
+
+                    (and (not db-based?) (not block?))
+                    [:div.pb-4])
+
+                  [:div
+                   (when (and block? (not sidebar?) (not whiteboard?))
+                     (let [config {:id "block-parent"
+                                   :block? true}]
+                       [:div.mb-4
+                        (component-block/breadcrumb config repo block-id {:level-limit 3})]))
+
+                   (when (and db-based? (not block?) (not preview?))
+                     (db-page/page-properties-react page {:configure? false}))
+
+                   ;; blocks
+                   (if loading?
+                     [:div.space-y-2
+                      (shui-ui/skeleton {:class "h-6 w-full"})
+                      (shui-ui/skeleton {:class "h-6 w-full"})]
+                     (let [_ (and block? page (reset! *current-block-page (:block/name (:block/page page))))
+                           _ (when (and block? (not page))
+                               (route-handler/redirect-to-page! @*current-block-page))]
+                       (page-blocks-cp repo page {:sidebar? sidebar? :whiteboard? whiteboard?})))]])
+
+               (when today?
+                 (today-queries repo today? sidebar?))
+
+               (when today?
+                 (scheduled/scheduled-and-deadlines page-name))
+
+               (when-not block?
+                 (tagged-pages repo page-name page-original-name))
 
            ;; referenced blocks
-           (when-not block-or-whiteboard?
-             (when page
-               [:div {:key "page-references"}
-                (rum/with-key
-                  (reference/references route-page-name)
-                  (str route-page-name "-refs"))]))
-
-           (when-not block-or-whiteboard?
-             (when (not journal?)
-               (hierarchy/structures route-page-name)))
-
-           (when-not (or block-or-whiteboard? sidebar? home?)
-             [:div {:key "page-unlinked-references"}
-              (reference/unlinked-references route-page-name)])])))))
+               (when-not block-or-whiteboard?
+                 (when page
+                   [:div {:key "page-references"}
+                    (rum/with-key
+                      (reference/references route-page-name)
+                      (str route-page-name "-refs"))]))
+
+               (when-not block-or-whiteboard?
+                 (when (not journal?)
+                   (hierarchy/structures route-page-name)))
+
+               (when-not (or block-or-whiteboard? sidebar? home?)
+                 [:div {:key "page-unlinked-references"}
+                  (reference/unlinked-references route-page-name)])])))))))
 
 (rum/defcs page < rum/static
   [state option]

src/main/frontend/db/async.cljs

@@ -236,3 +236,8 @@
             (->> result
                  db-model/sort-by-left-recursive
                  db-utils/group-by-page)))))))
+
+(defn <fetch-all-pages
+  [graph]
+  (when-let [^Object worker @db-browser/*worker]
+    (.fetch-all-pages worker graph)))

src/main/frontend/db/react.cljs

@@ -91,7 +91,8 @@
 (defn- <q-aux
   [repo db query-fn inputs-fn k query inputs]
   (let [kv? (and (vector? k) (= :kv (second k)))
-        q (if util/node-test?
+        journals? (and (vector? k) (= :frontend.worker.react/journals (last k)))
+        q (if (or journals? util/node-test?)
             (fn [query inputs] (apply d/q query db inputs))
             (fn [query inputs] (apply db-async-util/<q repo (cons query inputs))))]
     (when (or query-fn query kv?)

src/main/frontend/db/restore.cljs

@@ -7,7 +7,9 @@
             [promesa.core :as p]
             [cljs-time.core :as t]
             [logseq.db.sqlite.common-db :as sqlite-common-db]
-            [clojure.edn :as edn]))
+            [clojure.edn :as edn]
+            [frontend.db.async :as db-async]
+            [clojure.core.async :as async]))
 
 (comment
   (defn- old-schema?
@@ -47,7 +49,9 @@
     ;; FIXME:
     ;; (db-migrate/migrate attached-db)
 
-    (p/let [_ (p/delay 150)]          ; More time for UI refresh
-      (state/set-state! :graph/loading? false)
-      (react/clear-query-state!)
-      (state/pub-event! [:ui/re-render-root]))))
+    (state/set-state! :graph/loading? false)
+    (react/clear-query-state!)
+    (state/pub-event! [:ui/re-render-root])
+    (async/go
+      (async/<! (async/timeout 100))
+      (db-async/<fetch-all-pages repo))))

src/main/frontend/db_worker.cljs

@@ -377,24 +377,22 @@
   (getInitialData
    [_this repo]
    (when-let [conn (worker-state/get-datascript-conn repo)]
-     (let [data (->> (sqlite-common-db/get-initial-data @conn)
-                     pr-str)]
-       ;; (comment
-       ;;   (async/go
-       ;;   ;; TODO: after UI db transacted
-       ;;    (async/<! (async/timeout 500))
-       ;;    (let [all-pages (sqlite-common-db/get-all-pages @conn)
-       ;;          all-files (sqlite-common-db/get-all-files @conn)
-       ;;          full-data (concat all-files all-pages)
-       ;;          partitioned-data (map-indexed (fn [idx p] [idx p]) (partition-all 2000 full-data))]
-       ;;      (doseq [[idx tx-data] partitioned-data]
-       ;;        (worker-util/post-message :sync-db-changes (pr-str
-       ;;                                                    {:repo repo
-       ;;                                                     :tx-data tx-data
-       ;;                                                     :tx-meta {:initial-pages? true
-       ;;                                                               :end? (= idx (dec (count partitioned-data)))}}))
-       ;;        (async/<! (async/timeout 100))))))
-       data)))
+     (pr-str (sqlite-common-db/get-initial-data @conn))))
+
+  (fetch-all-pages
+   [_this repo]
+   (when-let [conn (worker-state/get-datascript-conn repo)]
+     (async/go
+       (let [all-pages (sqlite-common-db/get-all-pages @conn)
+             partitioned-data (map-indexed (fn [idx p] [idx p]) (partition-all 2000 all-pages))]
+         (doseq [[idx tx-data] partitioned-data]
+           (worker-util/post-message :sync-db-changes (pr-str
+                                                       {:repo repo
+                                                        :tx-data tx-data
+                                                        :tx-meta {:initial-pages? true
+                                                                  :end? (= idx (dec (count partitioned-data)))}}))
+           (async/<! (async/timeout 100)))))
+     nil))
 
   (closeDB
    [_this repo]