|
|
@@ -1,11 +1,12 @@
|
|
|
(ns frontend.search.semantic
|
|
|
"Browser implementation of search protocol"
|
|
|
- (:require [frontend.search.protocol :as protocol]
|
|
|
+ (:require ["@logseq/logmind" :refer [taskQueue]]
|
|
|
+ [frontend.search.protocol :as protocol]
|
|
|
[frontend.ai.vector-store :as vector-store]
|
|
|
[frontend.ai.text-encoder :as text-encoder]
|
|
|
[promesa.core :as p]
|
|
|
-
|
|
|
- [frontend.state :as state]))
|
|
|
+ [frontend.state :as state]
|
|
|
+ [logseq.graph-parser.util :as gp-util]))
|
|
|
|
|
|
(defn idstr-template-string
|
|
|
"Accepts repo url and returns a string for the vector store comment"
|
|
|
@@ -40,18 +41,15 @@
|
|
|
store-conn (if encoder-dim
|
|
|
(vector-store/create (idstr-template-string repo) encoder-dim)
|
|
|
(throw (js/Error. (str "record modelDim is not found in options of registrated encoder " encoder-name))))
|
|
|
- block->promise (fn [block]
|
|
|
- ;; TODO Junyi: Chunker
|
|
|
- (p/let [embed (text-encoder/text-encode (:content block) encoder-name)
|
|
|
- _ (vector-store/add store-conn embed (:uuid block))]))
|
|
|
- embed-promises (map block->promise blocks-to-add)])
|
|
|
- (p/let [uuids (map blocks-to-add)
|
|
|
- store-handler (vector-store/try-create )]
|
|
|
- (vector-store/add store-handler embed uuid))
|
|
|
- (prn "sematic: transact-blocks!") ;; TODO Junyi
|
|
|
- (prn data)
|
|
|
- (prn blocks-to-remove-set)
|
|
|
- (prn blocks-to-add))
|
|
|
+ addtask-fn (fn [block] (.addTask taskQueue (:uuid block)
|
|
|
+ (fn [] ;; Promise factory
|
|
|
+ ;; TODO Junyi: Block Chunker
|
|
|
+ (p/let [data {:snippet (gp-util/safe-subs (:content block) 0 20)
|
|
|
+ :page (:page block)
|
|
|
+ :id (:id block)}
|
|
|
+ embed (text-encoder/text-encode (:content block) encoder-name)]
|
|
|
+ (vector-store/add store-conn embed (:uuid block) data)))))]
|
|
|
+ (mapv addtask-fn blocks-to-add)))
|
|
|
|
|
|
(transact-pages! [_this data]
|
|
|
|
Junyi Du