Tienson Qin пре 3 месеци
родитељ
комит
e1147c3264

+ 13 - 8
src/main/frontend/components/vector_search/sidebar.cljs

@@ -21,42 +21,46 @@
     (hooks/use-effect!
     (hooks/use-effect!
      (fn []
      (fn []
        (c.m/run-task
        (c.m/run-task
+         ::update-vec-search-state
          (m/reduce
          (m/reduce
-          (fn [_ v] (set-vec-search-state v))
+          (fn [_ v]
+            (set-vec-search-state v))
           (m/ap
           (m/ap
             (m/?> vector-search-flows/infer-worker-ready-flow)
             (m/?> vector-search-flows/infer-worker-ready-flow)
             (c.m/<? (state/<invoke-db-worker :thread-api/vec-search-update-index-info repo))
             (c.m/<? (state/<invoke-db-worker :thread-api/vec-search-update-index-info repo))
             (m/?> vector-search-flows/vector-search-state-flow)))
             (m/?> vector-search-flows/vector-search-state-flow)))
-         ::update-vec-search-state :succ (constantly nil)))
+         :succ (constantly nil)))
      [])
      [])
     (hooks/use-effect!
     (hooks/use-effect!
      (fn []
      (fn []
        (c.m/run-task
        (c.m/run-task
+         ::update-load-model-progress
          (m/reduce
          (m/reduce
           (fn [_ v] (set-load-model-progress v))
           (fn [_ v] (set-load-model-progress v))
           vector-search-flows/load-model-progress-flow)
           vector-search-flows/load-model-progress-flow)
-         ::update-load-model-progress :succ (constantly nil)))
+         :succ (constantly nil)))
      [])
      [])
     (hooks/use-effect!
     (hooks/use-effect!
      (fn []
      (fn []
        (c.m/run-task
        (c.m/run-task
+         ::fetch-model-info
          (m/reduce
          (m/reduce
           (constantly nil)
           (constantly nil)
           (m/ap
           (m/ap
             (m/?> vector-search-flows/infer-worker-ready-flow)
             (m/?> vector-search-flows/infer-worker-ready-flow)
-            (let [model-info (state/<invoke-db-worker :thread-api/vec-search-embedding-model-info repo)]
-              (prn :model-info model-info)
+            (let [model-info (c.m/<? (state/<invoke-db-worker :thread-api/vec-search-embedding-model-info repo))]
               (set-model-info model-info))))
               (set-model-info model-info))))
-         ::fetch-model-info :succ (constantly nil)))
+         :succ (constantly nil)))
      [])
      [])
     (hooks/use-effect!
     (hooks/use-effect!
      (fn []
      (fn []
        (c.m/run-task
        (c.m/run-task
+         :update-search-result
          (m/sp
          (m/sp
            (-> (c.m/<? (state/<invoke-db-worker :thread-api/vec-search-search repo query-string 10))
            (-> (c.m/<? (state/<invoke-db-worker :thread-api/vec-search-search repo query-string 10))
                ldb/read-transit-str
                ldb/read-transit-str
                set-result))
                set-result))
-         :update-search-result :succ (constantly nil)))
+         :succ (constantly nil)))
      [(hooks/use-debounced-value query-string 200)])
      [(hooks/use-debounced-value query-string 200)])
     [:div
     [:div
      [:b "State"]
      [:b "State"]
@@ -92,10 +96,11 @@
      (shui/select
      (shui/select
       {:on-value-change (fn [model-name]
       {:on-value-change (fn [model-name]
                           (c.m/run-task
                           (c.m/run-task
+                            ::load-model
                             (m/sp
                             (m/sp
                               (c.m/<?
                               (c.m/<?
                                (state/<invoke-db-worker :thread-api/vec-search-load-model repo model-name)))
                                (state/<invoke-db-worker :thread-api/vec-search-load-model repo model-name)))
-                            ::load-model :succ (constantly nil)))}
+                            :succ (constantly nil)))}
       (shui/select-trigger
       (shui/select-trigger
        (shui/select-value
        (shui/select-value
         {:placeholder "Select a model(need force-embedding-all-graph-blocks again)"}))
         {:placeholder "Select a model(need force-embedding-all-graph-blocks again)"}))

+ 1 - 6
src/main/frontend/inference_worker/inference_worker.cljs

@@ -28,11 +28,6 @@
    [_]
    [_]
    (clj->js (keys infer-worker.text-embedding/available-embedding-models)))
    (clj->js (keys infer-worker.text-embedding/available-embedding-models)))
 
 
-  (set-db-worker-proxy
-   [_this proxy]
-   (reset! infer-worker.state/*db-worker proxy)
-   (log/info :set-db-worker-proxy :done))
-
   (text-embedding
   (text-embedding
    [_this text-coll]
    [_this text-coll]
    (p/chain
    (p/chain
@@ -60,7 +55,7 @@
 
 
   (search
   (search
    [_this repo query-string nums-neighbors]
    [_this repo query-string nums-neighbors]
-    (infer-worker.text-embedding/<search-knn repo query-string nums-neighbors))
+   (infer-worker.text-embedding/<search-knn repo query-string nums-neighbors))
 
 
   (index-info
   (index-info
    [_this repo]
    [_this repo]

+ 0 - 2
src/main/frontend/inference_worker/state.cljs

@@ -1,8 +1,6 @@
 (ns frontend.inference-worker.state
 (ns frontend.inference-worker.state
   "State hub for inference-worker")
   "State hub for inference-worker")
 
 
-(defonce *db-worker (atom nil))
-
 (defonce *hnswlib (atom nil))
 (defonce *hnswlib (atom nil))
 
 
 ;;repo -> index
 ;;repo -> index

+ 3 - 5
src/main/frontend/persist_db/browser.cljs

@@ -94,7 +94,7 @@
                                                          (str (namespace qkw) "/" (name qkw))
                                                          (str (namespace qkw) "/" (name qkw))
                                                          direct-pass?
                                                          direct-pass?
                                                          (if direct-pass?
                                                          (if direct-pass?
-                                                           (into-array args)
+                                                           args
                                                            (ldb/write-transit-str args)))]
                                                            (ldb/write-transit-str args)))]
                              (if direct-pass?
                              (if direct-pass?
                                result
                                result
@@ -144,10 +144,8 @@
 
 
 (defn <connect-db-worker-and-infer-worker!
 (defn <connect-db-worker-and-infer-worker!
   []
   []
-  (assert (and @state/*infer-worker @state/*db-worker))
-  (p/do!
-   (.set-db-worker-proxy ^js @state/*infer-worker (Comlink/proxy @state/*db-worker))
-   (.set-infer-worker-proxy ^js @state/*db-worker (Comlink/proxy @state/*infer-worker))))
+  (state/<invoke-db-worker-direct-pass :thread-api/set-infer-worker-proxy
+                                       (Comlink/proxy @state/*infer-worker)))
 
 
 (defn <export-db!
 (defn <export-db!
   [repo data]
   [repo data]

+ 4 - 0
src/main/frontend/worker/db_worker.cljs

@@ -713,6 +713,10 @@
       (gc-sqlite-dbs! db client-ops conn {:full-gc? true})
       (gc-sqlite-dbs! db client-ops conn {:full-gc? true})
       nil)))
       nil)))
 
 
+(def-thread-api :thread-api/set-infer-worker-proxy
+  [infer-worker-proxy]
+  (reset! worker-state/*infer-worker infer-worker-proxy))
+
 (def-thread-api :thread-api/vec-search-embedding-model-info
 (def-thread-api :thread-api/vec-search-embedding-model-info
   [repo]
   [repo]
   (embedding/task--embedding-model-info repo))
   (embedding/task--embedding-model-info repo))

+ 7 - 5
src/main/frontend/worker/embedding.cljs

@@ -178,16 +178,18 @@
   [repo]
   [repo]
   (when-not (indexing? repo)
   (when-not (indexing? repo)
     (let [canceler (c.m/run-task
     (let [canceler (c.m/run-task
-                    (task--embedding-stale-blocks! repo)
-                    :embedding-stale-blocks! :succ (constantly nil))]
+                     :embedding-stale-blocks!
+                     (task--embedding-stale-blocks! repo)
+                     :succ (constantly nil))]
       (reset-*vector-search-state! repo :canceler canceler))))
       (reset-*vector-search-state! repo :canceler canceler))))
 
 
 (defn re-embedding-graph-data!
 (defn re-embedding-graph-data!
   [repo]
   [repo]
   (when-not (indexing? repo)
   (when-not (indexing? repo)
     (let [canceler (c.m/run-task
     (let [canceler (c.m/run-task
-                    (task--re-embedding-graph-data! repo)
-                    :re-embedding-graph-data! :succ (constantly nil))]
+                     :re-embedding-graph-data!
+                     (task--re-embedding-graph-data! repo)
+                     :succ (constantly nil))]
       (reset-*vector-search-state! repo :canceler canceler))))
       (reset-*vector-search-state! repo :canceler canceler))))
 
 
 (defn task--embedding-model-info
 (defn task--embedding-model-info
@@ -236,7 +238,7 @@
         (when-let [conn (worker-state/get-datascript-conn repo)]
         (when-let [conn (worker-state/get-datascript-conn repo)]
           (let [{:keys [distances neighbors] :as r}
           (let [{:keys [distances neighbors] :as r}
                 (worker-util/profile (str "search: '" query-string "'")
                 (worker-util/profile (str "search: '" query-string "'")
-                  (js->clj (c.m/<? (.search infer-worker repo query-string nums-neighbors)) :keywordize-keys true))
+                                     (js->clj (c.m/<? (.search infer-worker repo query-string nums-neighbors)) :keywordize-keys true))
                 labels (->> (map vector distances neighbors)
                 labels (->> (map vector distances neighbors)
                             (keep (fn [[distance label]] (when-not (js/isNaN distance) label))))
                             (keep (fn [[distance label]] (when-not (js/isNaN distance) label))))
                 datoms (map (fn [label]
                 datoms (map (fn [label]