Indice being built async + jakob feedback

deps/shui/src/logseq/shui/list_item/v1.cljs

@@ -55,8 +55,8 @@
   (let [[before-text highlighted-text after-text] (split-text-on-highlight text query normal-text normal-query)]
     [:span 
      (when-not (string/blank? before-text) [:span before-text])
-     (when-not (string/blank? after-text) [:span after-text])
-     (when-not (string/blank? highlighted-text) [:span {:class "bg-accent-06 dark:bg-accent-08-alpha"} highlighted-text])]))
+     (when-not (string/blank? highlighted-text) [:span {:class "bg-accent-06 dark:bg-accent-08-alpha"} highlighted-text])
+     (when-not (string/blank? after-text) [:span after-text])]))
 
 (defn span-with-mutliple-highlight-tokens [app-config text query normal-text normal-query]
   (loop [[query-token & more] (string/split normal-query #" ")
@@ -94,22 +94,12 @@
           ;; Otherwise, just return the text
           :else
           [:span text-string])))))
-        
-(defn highlight-query-builder-with-log [props app-config query text]
-  (js/console.log "highlight-query" query text (clj->js props))
-  (try
-    (let [result (highlight-query* app-config query text)]
-      (js/console.log "highlight-query.result" (pr-str result))
-      result)
-    (catch js/Error e
-      (js/console.log "highlight-query.error" (pr-str e) props query text)
-      [:span])))
 
 ;; result-item
-(rum/defc root [{:keys [icon icon-theme query text info shortcut value-label value title highlighted on-highlight on-highlight-dep header on-click] :as props} 
+(rum/defc root [{:keys [icon icon-theme query text info shortcut value-label value title highlighted on-highlight on-highlight-dep header on-click] :as _props} 
                 {:keys [app-config] :as context}]
   (let [ref (rum/create-ref)
-        highlight-query (partial highlight-query-builder-with-log props app-config query)]
+        highlight-query (partial highlight-query* app-config query)]
     (rum/use-effect! 
       (fn [] 
         (when (and highlighted on-highlight) 

src/main/frontend/components/cmdk.cljs

@@ -61,8 +61,8 @@
 (defn create-items [q]
   (if-not (seq q)
     []
-    [{:text "Create page"       :icon "page"       :icon-theme :color :shortcut "cmd+shift+P" :info (str "Create page called '" q "'") :source-create :page} 
-     {:text "Create whiteboard" :icon "whiteboard" :icon-theme :color :shortcut "cmd+shift+W" :info (str "Create whiteboard called '" q "'") :source-create :whiteboard}]))
+    [{:text "Create page"       :icon "new-page"       :icon-theme :color :shortcut "cmd+shift+P" :info (str "Create page called '" q "'") :source-create :page} 
+     {:text "Create whiteboard" :icon "new-whiteboard" :icon-theme :color :shortcut "cmd+shift+W" :info (str "Create whiteboard called '" q "'") :source-create :whiteboard}]))
 
 ;; Take the results, decide how many items to show, and order the results appropriately
 (defn state->results-ordered [state]
@@ -160,8 +160,7 @@
                            :shortcut (:shortcut %)
                            :source-command %))
            (hash-map :status :success :items)
-           (swap! !results assoc group))) 
-    (js/console.log "commands" (clj->js (get-in @!results [:commands :items])))))
+           (swap! !results assoc group))))) 
 
 ;; The pages search action uses an existing handler
 (defmethod load-results :pages [group state]
@@ -180,7 +179,6 @@
                                             :icon-theme :gray 
                                             :text % 
                                             :source-page %) non-boards)]
-      (js/console.log "pages" (clj->js whiteboards) (clj->js non-boards))
       (swap! !results assoc group {:status :success :items non-board-items}
       ; (swap! !results assoc :whiteboards {:status :success :items whiteboard-items}
                             :whiteboards {:status :success :items whiteboard-items}))))
@@ -194,15 +192,15 @@
         opts {:limit 100}]
     (swap! !results assoc-in [group :status] :loading)
     (swap! !results assoc-in [:current-page :status] :loading)
-    (p/let [blocks (search/block-search repo @!input opts)])))
-            ; items (map #(hash-map :icon "block" 
-            ;                       :icon-theme :gray 
-            ;                       :text (:block/content %) 
-            ;                       :header (some-> % :block/page db/entity :block/name)
-            ;                       :current-page? (some-> % :block/page #{current-page})
-            ;                       :source-block %) blocks)
-            ; items-on-other-pages (remove :current-page? items)
-            ; items-on-current-page (filter :current-page? items)]
+    (p/let [blocks (search/block-search repo @!input opts)
+            items (map #(hash-map :icon "block" 
+                                  :icon-theme :gray 
+                                  :text (:block/content %) 
+                                  :header (some-> % :block/page db/entity :block/name)
+                                  :current-page? (some-> % :block/page #{current-page})
+                                  :source-block %) blocks)
+            items-on-other-pages (remove :current-page? items)
+            items-on-current-page (filter :current-page? items)]
       ; (js/console.log "blocks" (clj->js items) current-page) 
       ; ; (js/console.log "blocks" (clj->js items) 
       ; ;                 (pr-str (map (comp pr-str :block/page) blocks)) 
@@ -211,8 +209,8 @@
       ; ; (js/console.log "load-results/blocks" 
       ; ;                 (clj->js blocks) 
       ; ;                 (pr-str (first blocks)))
-      ; (swap! !results assoc group {:status :success :items items-on-other-pages}
-            ;                 :current-page {:status :success :items items-on-current-page}))))
+      (swap! !results assoc group {:status :success :items items-on-other-pages}
+                            :current-page {:status :success :items items-on-current-page}))))
 
 ; (defmethod load-results :whiteboards [group state]
 ;   (let [!input (::input state)
@@ -512,12 +510,13 @@
 
 (defn print-group-name [group]
   (case group
-    :current-page "Search current page"
-    :blocks "Search blocks"
-    :pages "Search pages"
-    :whiteboards "Search whiteboards"
-    :commands "Search commands"
-    (str "Search " (name group))))
+    :current-page "Current page"
+    :blocks "Blocks"
+    :pages "Pages"
+    :whiteboards "Whiteboards"
+    :commands "Commands"
+    :recents "Recents"
+    (string/capitalize (name group))))
 
 (rum/defc filter-row [state filter]
   [:div {:class "pt-3 border-b flex flex-col gap-2 bg-gray-02 border-gray-07" 
@@ -605,10 +604,10 @@
                    (reset! (::keyup-handler state) nil)
                    state)}
   {:did-mount (fn [state] 
-                ; (search-db/make-blocks-indice-non-blocking! (state/get-current-repo))
-                (when-let [ref @(::scroll-container-ref state)]
-                  (js/console.log "scrolling")
-                  (js/setTimeout #(set! (.-scrollTop ref) FILTER-ROW-HEIGHT)))
+                (search-db/make-blocks-indice-non-blocking! (state/get-current-repo))
+                ; (when-let [ref @(::scroll-container-ref state)]
+                ;   (js/console.log "scrolling")
+                ;   (js/setTimeout #(set! (.-scrollTop ref) FILTER-ROW-HEIGHT)))
                 state)}
                   ; (load-results :initial state)))}
   [state {:keys []}]

src/main/frontend/db.cljs

@@ -38,7 +38,7 @@
 
  [frontend.db.model
   blocks-count blocks-count-cache delete-blocks get-pre-block
-  delete-files delete-pages-by-files get-all-block-contents get-all-block-contents-non-blocking get-all-tagged-pages
+  delete-files delete-pages-by-files get-all-block-contents get-all-block-avets get-all-tagged-pages get-single-block-contents
   get-all-templates get-block-and-children get-block-by-uuid get-block-children sort-by-left
   get-block-parent get-block-parents parents-collapsed? get-block-referenced-blocks get-all-referenced-blocks-uuid
   get-block-children-ids get-block-immediate-children get-block-page

src/main/frontend/db/model.cljs

@@ -1499,50 +1499,10 @@ independent of format as format specific heading characters are stripped"
          (map get-single-block-contents)
          (remove nil?))))
 
-(defn get-all-block-contents-non-blocking 
-  ([]
-   (when-let [db (conn/get-db)]
-     (let [ids (->> (d/datoms db :avet :block/uuid)
-                    (map :v)
-                    (remove nil?)
-                    (not-empty))]
-       (js/Promise. 
-         (fn [resolve-all _] 
-           (-> (fn [promise-acc id]
-                 (.then promise-acc 
-                    (fn [acc] 
-                      (let [contents (get-single-block-contents id)] 
-                        (js/Promise.
-                          (fn [resolve _]
-                            (js/setTimeout 
-                              #(if (nil? contents) 
-                                 (resolve acc) 
-                                 (resolve (conj acc contents))) 
-                              0)))))))
-               (reduce (js/Promise.resolve (list)) ids)
-               (.then #(resolve-all %)))))))))
-              ; (apply get-all-block-contents-non-blocking (list))))))
-          ; (map get-single-block-contents)
-          ; (remove nil?))))
-  ; ([acc id & more]
-  ;  (let [contents (get-single-block-contents id)
-  ;        acc (cond-> acc (seq contents) (conj contents))])
-
-  ;  (js/Promise. 
-  ;    (fn [resolve reject] 
-  ;      (let [callback (fn [acc] 
-  ;                       (let [contents (get-single-block-contents id)] 
-  ;                         (js/setTimeout
-  ;                           #(resolve
-  ;                              (if (nil? contents) 
-  ;                                acc 
-  ;                                (conj acc contents)))
-  ;                           0)))]
-  ;        (if (seq more) 
-  ;          (.then (apply get-all-block-contents-non-blocking more) callback)
-  ;          (callback (list))))))))
-           
-
+(defn get-all-block-avets
+  []
+  (when-let [db (conn/get-db)]
+    (->> (d/datoms db :avet :block/uuid))))
 
 ;; Deprecated?
 (defn delete-blocks

src/main/frontend/search/db.cljs

@@ -51,15 +51,6 @@
        (remove nil?)
        (bean/->js)))
 
-(defn build-blocks-indice-non-blocking
-  [repo]
-  (.then (db/get-all-block-contents-non-blocking)
-         (fn [block-contents] 
-           (->> block-contents
-                (map block->index)
-                (remove nil?)
-                (bean/->js)))))
-
 (defn build-pages-indice
   [repo]
   (->> (db/get-all-pages repo)
@@ -69,32 +60,39 @@
        (bean/->js)))
 
 (defn make-blocks-indice!
-  [repo]
-  (let [blocks (build-blocks-indice repo)
-        indice (fuse. blocks
-                      (clj->js {:keys ["uuid" "content" "page"]
-                                :shouldSort true
-                                :tokenize true
-                                :minMatchCharLength 1
-                                :distance 1000
-                                :threshold 0.35}))]
-    (swap! indices assoc-in [repo :blocks] indice)
-    indice))
+  ([repo] (make-blocks-indice! repo (build-blocks-indice repo)))
+  ([repo blocks]
+   (let [indice (fuse. blocks
+                       (clj->js {:keys ["uuid" "content" "page"]
+                                 :shouldSort true
+                                 :tokenize true
+                                 :minMatchCharLength 1
+                                 :distance 1000
+                                 :threshold 0.35}))]
+     (swap! indices assoc-in [repo :blocks] indice)
+     indice)))
+
+(defn process-block-avet [avet]
+  (some->> avet :v db/get-single-block-contents block->index bean/->js)) 
 
 (defn make-blocks-indice-non-blocking! 
-  [repo]
-  (.then (build-blocks-indice-non-blocking repo)
-    (fn [blocks]
-      (let [indice (fuse. blocks 
-                          (clj->js {:keys ["uuid" "content" "page"]
-                                    :shouldSort true
-                                    :tokenize true
-                                    :minMatchCharLength 1
-                                    :distance 1000
-                                    :threshold 0.35}))]
-        (swap! indices assoc-in [repo :blocks] indice)
-        indice))))
-  ; (let [blocks (build-blocks-indice-non-blocking repo)]))
+  ([repo] (make-blocks-indice-non-blocking! 1000))
+  ([repo chunk-size]
+   (let [avets (db/get-all-block-avets)
+         chunks (partition-all chunk-size avets)
+         acc (atom [])
+         process-chunks (fn process-recur [[curr & more]]
+                          (js/console.log "process-recur") 
+                          (when-not (empty? curr) 
+                            (->> (map process-block-avet curr)
+                                 (remove nil?)
+                                 (doall)
+                                 (swap! acc concat)))
+                          (if (empty? more)
+                            (make-blocks-indice! repo @acc)
+                            (js/setTimeout #(process-recur more) 0)))]
+     (js/console.log "process-make-blocks-indice-non-blocking!" (count avets))
+     (process-chunks chunks))))
 
 (defn original-page-name->index
   [p]