|
|
@@ -1,10 +1,11 @@
|
|
|
(ns frontend.search.semantic
|
|
|
"Browser implementation of search protocol"
|
|
|
(:require ["@logseq/logmind" :refer [taskQueue]]
|
|
|
+ [cljs-bean.core :as bean]
|
|
|
+ [promesa.core :as p]
|
|
|
[frontend.search.protocol :as protocol]
|
|
|
[frontend.ai.vector-store :as vector-store]
|
|
|
[frontend.ai.text-encoder :as text-encoder]
|
|
|
- [promesa.core :as p]
|
|
|
[frontend.state :as state]
|
|
|
[logseq.graph-parser.util :as gp-util]))
|
|
|
|
|
|
@@ -27,12 +28,10 @@
|
|
|
nil)
|
|
|
|
|
|
(transact-blocks! [_this {:keys [blocks-to-remove-set
|
|
|
- blocks-to-add]
|
|
|
- :as data}]
|
|
|
- ;; Step 1: encoding all sentences
|
|
|
- ;; Step 2: inference vec length
|
|
|
- ;; Step 3: create vector store (optional)
|
|
|
- ;; Setp 4: add to vec store
|
|
|
+ blocks-to-add]}]
|
|
|
+ ;; Step 1: create vector store handler
|
|
|
+ ;; Step 2: deal with blocks-to-remove-set
|
|
|
+ ;; Step 3: deal with blocks-to-add
|
|
|
;; {:blocks-to-remove-set #{16634}, :blocks-to-add ({:id 16634, :uuid "647dcfc7-2aba-4015-8b71-cdf73c552761", :page 12, :content "adding me 2"})}
|
|
|
;; Handling blocks to add
|
|
|
(let [encoder (state/get-semsearch-encoder)
|
|
|
@@ -41,28 +40,44 @@
|
|
|
store-conn (if encoder-dim
|
|
|
(vector-store/create (idstr-template-string repo) encoder-dim)
|
|
|
(throw (js/Error. (str "record modelDim is not found in options of registrated encoder " encoder-name))))
|
|
|
- addtask-fn (fn [block] (.addTask taskQueue (:uuid block)
|
|
|
- (fn [] ;; Promise factory
|
|
|
- ;; TODO Junyi: Block Chunker
|
|
|
- (p/let [data {:snippet (gp-util/safe-subs (:content block) 0 20)
|
|
|
- :page (:page block)
|
|
|
- :id (:id block)}
|
|
|
- embed (text-encoder/text-encode (:content block) encoder-name)]
|
|
|
- (vector-store/add store-conn embed (:uuid block) data)))))]
|
|
|
- (mapv addtask-fn blocks-to-add)))
|
|
|
+ eid-del->vs (fn [eid]
|
|
|
+ ;; Would replace existing promise in queue (if any)
|
|
|
+ ;; If the promise is already in pending state,
|
|
|
+ ;; there's a race condition that the promise executed
|
|
|
+ ;; before the pending promise is resolved
|
|
|
+ (let [del->vs (fn [] ;; Promise factory
|
|
|
+ (vector-store/rm store-conn (str eid)))]
|
|
|
+ (.addTask taskQueue (str eid) del->vs)))
|
|
|
+ block-add->vs (fn [block]
|
|
|
+ ;; Would replace the task if there is already a task with the same id in the queue
|
|
|
+ ;; Here we use stringified id as key to keep consistency with the logMind type annotation
|
|
|
+ (let [add->vs (fn []
|
|
|
+ (p/let [metadata {:snippet (gp-util/safe-subs (:content block) 0 20)
|
|
|
+ :page (:page block)
|
|
|
+ :id (:id block)
|
|
|
+ :uuid (:uuid block)}
|
|
|
+ embeds (text-encoder/text-encode (:content block) encoder-name)
|
|
|
+ _ (vector-store/rm store-conn (str (:id block)))
|
|
|
+ emb-add->vs (fn [embed]
|
|
|
+ (vector-store/add store-conn embed (str (:id block)) (bean/->js metadata)))]
|
|
|
+ (p/all (mapv emb-add->vs embeds))))]
|
|
|
+ (.addTask taskQueue (str (:id block)) add->vs)))]
|
|
|
+ ;; Delete first, then add
|
|
|
+ (mapv eid-del->vs blocks-to-remove-set)
|
|
|
+ (mapv block-add->vs blocks-to-add)))
|
|
|
|
|
|
- (transact-pages! [_this data]
|
|
|
-
|
|
|
- (vector-store/create "test" 128)
|
|
|
+ (transact-pages! [_this data]
|
|
|
(prn "semantic: transact-pages!") ;; TODO Junyi
|
|
|
(prn data))
|
|
|
|
|
|
(truncate-blocks! [_this]
|
|
|
- (-> repo
|
|
|
- (idstr-template-string)
|
|
|
- (vector-store/reset)))
|
|
|
+ (-> repo
|
|
|
+ (idstr-template-string)
|
|
|
+ (vector-store/reset))
|
|
|
+ (.clean taskQueue))
|
|
|
|
|
|
(remove-db! [_this]
|
|
|
- (-> repo
|
|
|
- (idstr-template-string)
|
|
|
- (vector-store/reset))))
|
|
|
+ (-> repo
|
|
|
+ (idstr-template-string)
|
|
|
+ (vector-store/reset))
|
|
|
+ (.clean taskQueue)))
|
Junyi Du