Merge branch 'master' into enhance/mobile-silk

.cljfmt.edn

@@ -1,4 +1,5 @@
 {:extra-indents {missionary.core/sp [[:block 0]]
                  missionary.core/ap [[:block 0]]
+                 frontend.util/profile [[:inner 0]]
                  frontend.common.missionary/run-task [[:inner 0]]}
  :sort-ns-references? true}

.github/workflows/clj-e2e.yml

@@ -78,7 +78,7 @@ jobs:
       # NOTE: require the app to be build with DEV-RELEASE flag
       - name: Prepare E2E test build
         run: |
-          yarn gulp:build && clojure -M:cljs release app --config-merge "{:closure-defines {frontend.config/DEV-RELEASE true}}" --debug
+          yarn gulp:build && clojure -M:cljs release app workers --config-merge "{:closure-defines {frontend.config/DEV-RELEASE true}}" --debug && yarn webpack-app-build
 
       - name: Run e2e tests
         run: cd clj-e2e && timeout 30m bb dev

.github/workflows/clj-rtc-e2e.yml

@@ -79,7 +79,7 @@ jobs:
       # NOTE: require the app to be build with DEV-RELEASE flag
       - name: Prepare E2E test build
         run: |
-          yarn gulp:build && clojure -M:cljs release app --config-merge "{:closure-defines {frontend.config/DEV-RELEASE true}}" --debug
+          yarn gulp:build && clojure -M:cljs release app workers --config-merge "{:closure-defines {frontend.config/DEV-RELEASE true}}" --debug && yarn webpack-app-build
           rsync -avz --exclude node_modules --exclude android --exclude ios ./static/ ./public/
           ls -lR ./public
 

.github/workflows/deploy-db-test-pages.yml

@@ -42,8 +42,8 @@ jobs:
 
       - name: Build Released-Web
         run: |
-          yarn gulp:build && clojure -M:cljs release app  --config-merge '{:compiler-options {:source-map true :source-map-include-sources-content true :source-map-detail-level :symbols}}'
-          rsync -avz --exclude node_modules --exclude android --exclude ios ./static/ ./public/
+          yarn gulp:build && clojure -M:cljs release app workers  --config-merge '{:compiler-options {:source-map true :source-map-include-sources-content true :source-map-detail-level :symbols}}' && yarn webpack-app-build
+          rsync -avz --exclude node_modules --exclude android --exclude ios --exclude mobile ./static/ ./public/
           ls -lR ./public && mkdir r2 && mv ./public/js/main.js.map ./r2/db-test.main.js.map
           sed -i 's/=main.js.map/=https:\/\/assets.logseq.io\/db-test.main.js.map/g' ./public/js/main.js
         env:

clj-e2e/dev/user.clj

@@ -4,6 +4,7 @@
             [logseq.e2e.block :as b]
             [logseq.e2e.commands-basic-test]
             [logseq.e2e.config :as config]
+            [logseq.e2e.editor-basic-test]
             [logseq.e2e.fixtures :as fixtures]
             [logseq.e2e.graph :as graph]
             [logseq.e2e.keyboard :as k]
@@ -73,6 +74,11 @@
   (->> (future (run-tests 'logseq.e2e.rtc-extra-test))
        (swap! *futures assoc :rtc-extra-test)))
 
+(defn run-editor-basic-test
+  []
+  (->> (future (run-tests 'logseq.e2e.editor-basic-test))
+       (swap! *futures assoc :editor-basic-test)))
+
 (defn run-tag-basic-test
   []
   (->> (future (run-tests 'logseq.e2e.tag-basic-test))
@@ -80,7 +86,8 @@
 
 (defn run-all-basic-test
   []
-  (run-tests 'logseq.e2e.commands-basic-test
+  (run-tests 'logseq.e2e.editor-basic-test
+             'logseq.e2e.commands-basic-test
              'logseq.e2e.multi-tabs-basic-test
              'logseq.e2e.outliner-basic-test
              'logseq.e2e.rtc-basic-test

clj-e2e/src/logseq/e2e/assert.clj

@@ -18,9 +18,10 @@
 
 (defn assert-non-editor-mode
   []
-  (assert-is-hidden (loc/or "[data-testid='block editor']"
-                            ;; TODO: remove this when this prop-name fixed on dom
-                            "[datatestid='block editor']")))
+  (w/wait-for-not-visible
+   (loc/or "[data-testid='block editor']"
+           ;; TODO: remove this when this prop-name fixed on dom
+           "[datatestid='block editor']")))
 
 (defn assert-in-normal-mode?
   "- not editing mode

clj-e2e/src/logseq/e2e/block.clj

@@ -97,3 +97,20 @@
 (defn outdent
   []
   (indent-outdent false))
+
+(defn toggle-property
+  [property-title property-value]
+  (k/press (if util/mac? "ControlOrMeta+p" "Control+Alt+p"))
+  (w/fill ".ls-property-dialog .ls-property-input input" property-title)
+  (w/wait-for (format "#ac-0.menu-link:has-text('%s')" property-title))
+  (k/enter)
+  (util/wait-timeout 100)
+  (w/click (w/-query ".ls-property-dialog .ls-property-input input"))
+  (util/wait-timeout 100)
+  (util/input property-value)
+  (w/wait-for (format "#ac-0.menu-link:has-text('%s')" property-value))
+  (k/enter))
+
+(defn select-blocks
+  [n]
+  (util/repeat-keyboard n "Shift+ArrowUp"))

clj-e2e/src/logseq/e2e/graph.clj

@@ -2,9 +2,11 @@
   (:require [clojure.edn :as edn]
             [clojure.string :as string]
             [logseq.e2e.assert :as assert]
+            [logseq.e2e.keyboard :as k]
             [logseq.e2e.locator :as loc]
             [logseq.e2e.util :as util]
-            [wally.main :as w]))
+            [wally.main :as w]
+            [wally.repl :as repl]))
 
 (defn- refresh-all-remote-graphs
   []
@@ -55,9 +57,11 @@
 
 (defn validate-graph
   []
+  (k/esc)
+  (k/esc)
   (util/search-and-click "(Dev) Validate current graph")
-  (assert/assert-is-visible (loc/and ".notifications div" (w/get-by-text "Your graph is valid")))
-  (let [content (.textContent (loc/and ".notifications div" (w/get-by-text "Your graph is valid")))
+  (assert/assert-is-visible (loc/and ".notifications div.notification-success div" (w/get-by-text "Your graph is valid")))
+  (let [content (.textContent (loc/and ".notifications div.notification-success div" (w/get-by-text "Your graph is valid")))
         summary (edn/read-string (subs content (string/index-of content "{")))]
-    (w/click ".notifications .ls-icon-x")
+    (w/click ".notifications div.notification-success .ls-icon-x")
     summary))

clj-e2e/src/logseq/e2e/page.clj

@@ -1,9 +1,9 @@
 (ns logseq.e2e.page
-  (:require [logseq.e2e.keyboard :as k]
+  (:require [logseq.e2e.block :as b]
+            [logseq.e2e.keyboard :as k]
             [logseq.e2e.util :as util]
             [wally.main :as w]
-            [wally.selectors :as ws]
-            [logseq.e2e.block :as b])
+            [wally.selectors :as ws])
   (:import (com.microsoft.playwright TimeoutError)))
 
 (defn goto-page

clj-e2e/src/logseq/e2e/util.clj

@@ -63,19 +63,31 @@
     (.pressSequentially input-node text
                         (.setDelay (Locator$PressSequentiallyOptions.) delay))))
 
+(defn exit-edit
+  []
+  (when (get-editor)
+    (k/esc))
+  (assert/assert-non-editor-mode))
+
 (defn double-esc
   "Exits editing mode and ensure there's no action bar"
   []
-  (k/esc)
-  (k/esc))
+  (when (w/visible? "div[data-radix-popper-content-wrapper]")
+    (k/esc))
+  (exit-edit)
+  (when (w/visible? "div[data-radix-popper-content-wrapper]")
+    (k/esc)))
 
 (defn search
   [text]
-  (double-esc)
-  (assert/assert-in-normal-mode?)
-  (w/click :#search-button)
-  (w/wait-for ".cp__cmdk-search-input")
-  (w/fill ".cp__cmdk-search-input" text))
+  (if (w/visible? ".cp__cmdk-search-input")
+    (w/fill ".cp__cmdk-search-input" text)
+    (do
+      (double-esc)
+      (assert/assert-in-normal-mode?)
+      (w/click :#search-button)
+      (w/wait-for ".cp__cmdk-search-input")
+      (w/fill ".cp__cmdk-search-input" text))))
 
 (defn search-and-click
   [search-text]
@@ -105,11 +117,6 @@
   []
   (count-elements ".ls-page-blocks .page-blocks-inner .ls-block"))
 
-(defn exit-edit
-  []
-  (k/esc)
-  (assert/assert-non-editor-mode))
-
 (defn get-text
   [locator]
   (if (string? locator)
@@ -129,7 +136,7 @@
 (defn repeat-keyboard
   [n shortcut]
   (dotimes [_i n]
-    (k/press shortcut)))
+    (k/press shortcut {:delay 20})))
 
 (defn get-page-blocks-contents
   []
@@ -161,11 +168,13 @@
 
 (defn move-cursor-to-end
   []
-  (k/press "ControlOrMeta+a" "ArrowRight"))
+  (k/press ["ControlOrMeta+a" "ArrowRight"]
+           {:delay 20}))
 
 (defn move-cursor-to-start
   []
-  (k/press "ControlOrMeta+a" "ArrowLeft"))
+  (k/press ["ControlOrMeta+a" "ArrowLeft"]
+           {:delay 20}))
 
 (defn input-command
   [command]
@@ -179,11 +188,13 @@
   (w/click "a.menu-link.chosen"))
 
 (defn set-tag
-  [tag]
+  "`hidden?`: some tags may be hidden from the UI, e.g. Page"
+  [tag & {:keys [hidden?]
+          :or {hidden? false}}]
   (press-seq " #" {:delay 20})
   (press-seq tag)
   (w/click (first (w/query (format "a.menu-link:has-text(\"%s\")" tag))))
-  (when (not= (string/lower-case tag) "task")
+  (when (and (not= (string/lower-case tag) "task") (not hidden?))
     ;; wait tag added on ui
     (assert/assert-is-visible
      (-> ".ls-block:not(.block-add-button)"

clj-e2e/test/logseq/e2e/editor_basic_test.clj

@@ -0,0 +1,96 @@
+(ns logseq.e2e.editor-basic-test
+  (:require
+   [clojure.set :as set]
+   [clojure.test :refer [deftest testing is use-fixtures]]
+   [logseq.e2e.assert :as assert]
+   [logseq.e2e.block :as b]
+   [logseq.e2e.fixtures :as fixtures]
+   [logseq.e2e.keyboard :as k]
+   [logseq.e2e.page :as p]
+   [logseq.e2e.util :as util]
+   [wally.main :as w]
+   [wally.repl :as repl]))
+
+(use-fixtures :once fixtures/open-page)
+
+(use-fixtures :each
+  fixtures/new-logseq-page
+  fixtures/validate-graph)
+
+(deftest toggle-between-page-and-block
+  (testing "Convert block to page and back"
+    (b/new-block "b1")
+    (util/set-tag "Page" {:hidden? true})
+    (assert/assert-is-visible ".ls-page-blocks .ls-block .ls-icon-file")
+    (b/toggle-property "Tags" "Page")
+    (assert/assert-is-hidden ".ls-page-blocks .ls-block .ls-icon-file")))
+
+(deftest toggle-between-page-and-block-for-selected-blocks
+  (testing "Convert selected blocks to pages and back"
+    (b/new-blocks ["b1" "b2" "b3"])
+    (b/select-blocks 3)
+    (b/toggle-property "Tags" "Page")
+    (assert/assert-is-visible ".ls-page-blocks .ls-block .ls-icon-file")
+    (w/wait-for (format "#ac-0.menu-link:has-text('%s')" "Page"))
+    (k/enter)
+    (w/wait-for-not-visible ".ls-page-blocks .ls-block .ls-icon-file")))
+
+(deftest disallow-adding-page-tag-to-normal-pages
+  (testing "Disallow adding #Page to normal pages"
+    (k/arrow-up)
+    (util/move-cursor-to-end)
+    (util/press-seq " #" {:delay 20})
+    (util/press-seq "Page")
+    (assert/assert-is-hidden (format "#ac-0.menu-link:has-text('%s')" "Page"))))
+
+(deftest move-blocks-mod+shift+m
+  (testing "move blocks using `mod+shift+m`"
+    (p/new-page "Target page")
+    (p/new-page "Source page")
+    (b/new-blocks ["b1" "b2" "b3"])
+    (b/select-blocks 3)
+    (k/press "ControlOrMeta+Shift+m")
+    (w/fill "input[placeholder=\"Move blocks to\"]" "Target page")
+    (w/wait-for (w/get-by-test-id "Target page"))
+    (.focus (w/-query ".cp__cmdk-search-input"))
+    (k/enter)
+    (assert/assert-have-count ".ls-page-blocks .page-blocks-inner .ls-block" 0)))
+
+(deftest move-blocks-cmdk
+  (testing "move blocks using cmdk"
+    (p/new-page "Target page 2")
+    (p/new-page "Source page 2")
+    (b/new-blocks ["b1" "b2" "b3"])
+    (b/select-blocks 3)
+    (util/search-and-click "Move blocks to")
+    (w/fill "input[placeholder=\"Move blocks to\"]" "Target page 2")
+    (w/wait-for (w/get-by-test-id "Target page 2"))
+    (.focus (w/-query ".cp__cmdk-search-input"))
+    (k/enter)
+    (assert/assert-have-count ".ls-page-blocks .page-blocks-inner .ls-block" 0)))
+
+(deftest move-pages-to-library
+  (testing "move pages using `mod+shift+m`"
+    (p/goto-page "Library")
+    (p/new-page "test page")
+    (b/new-blocks ["block1" "block2" "block3"])
+    (b/select-blocks 3)
+    (k/press "ControlOrMeta+Shift+m")
+    (w/fill "input[placeholder=\"Move blocks to\"]" "Library")
+    (w/wait-for (w/get-by-test-id "Library"))
+    (.focus (w/-query ".cp__cmdk-search-input"))
+    (k/enter)
+    (p/goto-page "Library")
+    (let [contents (set (util/get-page-blocks-contents))]
+      (is (set/subset? (set ["block1" "block2" "block3"]) contents)))
+    (p/goto-page "test page")
+    (b/new-blocks ["block4" "block5"])
+    (b/select-blocks 2)
+    (k/press "ControlOrMeta+Shift+m")
+    (w/fill "input[placeholder=\"Move blocks to\"]" "Library")
+    (w/wait-for (w/get-by-test-id "Library"))
+    (.focus (w/-query ".cp__cmdk-search-input"))
+    (k/enter)
+    (p/goto-page "Library")
+    (let [contents (set (util/get-page-blocks-contents))]
+      (is (set/subset? (set ["block1" "block2" "block3" "block4" "block5"]) contents)))))

clj-e2e/test/logseq/e2e/outliner_basic_test.clj

@@ -100,6 +100,7 @@
     (b/indent)
     (k/arrow-up)
     (b/delete-blocks)
+    (util/wait-editor-visible)
     (is (= "b1" (util/get-edit-content)))
     (is (= 1 (util/page-blocks-count)))))
 

clj-e2e/test/logseq/e2e/plugins_basic_test.clj

@@ -25,14 +25,14 @@
   (when (string? s)
     (-> s
       ;; Normalize input: replace hyphens/spaces with underscores, collapse multiple underscores
-      (clojure.string/replace #"[-\s]+" "_")
+        (clojure.string/replace #"[-\s]+" "_")
       ;; Split on uppercase letters (except at start) and join with underscore
-      (clojure.string/replace #"(?<!^)([A-Z])" "_$1")
+        (clojure.string/replace #"(?<!^)([A-Z])" "_$1")
       ;; Remove redundant underscores and trim
-      (clojure.string/replace #"_+" "_")
-      (clojure.string/trim)
+        (clojure.string/replace #"_+" "_")
+        (clojure.string/trim)
       ;; Convert to lowercase
-      (clojure.string/lower-case))))
+        (clojure.string/lower-case))))
 
 (defn- ls-api-call!
   [tag & args]
@@ -63,7 +63,7 @@
     (let [ret (ls-api-call! :editor.appendBlockInPage "test-block-apis" "append-block-in-page-0")
           uuid' (assert-api-ls-block! ret)]
       (-> (ls-api-call! :editor.insertBlock uuid' "insert-0")
-        (assert-api-ls-block!))
+          (assert-api-ls-block!))
       (ls-api-call! :editor.updateBlock uuid' "append-but-updated-0")
       (k/esc)
       (w/wait-for ".block-title-wrap:text('append-but-updated-0')")

deps.edn

@@ -53,16 +53,14 @@
                                           :git/sha "d61ce7e29186de021a2a453a8cee68efb5a88440"}}
 
  :aliases {:cljs {:extra-paths ["src/dev-cljs/" "src/test/" "src/electron/"]
-                  :extra-deps  {org.clojure/clojurescript        {:mvn/version "1.11.132"}
-                                org.clojure/tools.namespace      {:mvn/version "0.2.11"}
+                  :extra-deps  {org.clojure/tools.namespace      {:mvn/version "0.2.11"}
                                 cider/cider-nrepl                {:mvn/version "0.55.1"}
                                 org.clojars.knubie/cljs-run-test {:mvn/version "1.0.1"}
                                 tortue/spy                       {:mvn/version "2.14.0"}}
                   :main-opts   ["-m" "shadow.cljs.devtools.cli"]}
 
            :test {:extra-paths ["src/test/"]
-                  :extra-deps  {org.clojure/clojurescript        {:mvn/version "1.11.132"}
-                                org.clojure/test.check           {:mvn/version "1.1.1"}
+                  :extra-deps  {org.clojure/test.check           {:mvn/version "1.1.1"}
                                 pjstadig/humane-test-output      {:mvn/version "0.11.0"}
                                 org.clojars.knubie/cljs-run-test {:mvn/version "1.0.1"}
                                 tortue/spy                       {:mvn/version "2.14.0"}
@@ -70,8 +68,7 @@
                   :main-opts   ["-m" "shadow.cljs.devtools.cli"]}
 
            :rtc-e2e-test {:extra-paths ["src/rtc_e2e_test"]
-                          :extra-deps {org.clojure/clojurescript        {:mvn/version "1.11.132"}
-                                       cider/cider-nrepl                {:mvn/version "0.50.2"}}
+                          :extra-deps {cider/cider-nrepl                {:mvn/version "0.50.2"}}
                           :main-opts ["-m" "shadow.cljs.devtools.cli"]}
 
            :bench {:extra-paths ["src/bench/"]

deps/common/src/logseq/common/path.cljs

@@ -148,7 +148,7 @@
         scheme (.-protocol url)
         path (.-pathname url)
         domain (or (not-empty (.-host url))
-                 (if (string/starts-with? path "/") "" "/"))
+                   (if (string/starts-with? path "/") "" "/"))
         encoded-new-path (apply uri-path-join-internal path segments)]
     (str scheme "//" domain encoded-new-path)))
 

deps/db/src/logseq/db.cljs

@@ -7,6 +7,7 @@
             [clojure.walk :as walk]
             [datascript.core :as d]
             [datascript.impl.entity :as de]
+            [logseq.common.config :as common-config]
             [logseq.common.util :as common-util]
             [logseq.common.uuid :as common-uuid]
             [logseq.db.common.delete-blocks :as delete-blocks] ;; Load entity extensions
@@ -272,6 +273,9 @@
 (def get-built-in-page db-db/get-built-in-page)
 
 (def library? db-db/library?)
+(defn get-library-page
+  [db]
+  (get-built-in-page db common-config/library-page-name))
 
 (defn get-case-page
   "Case sensitive version of get-page. For use with DB graphs"
@@ -575,3 +579,17 @@
   (if (sqlite-util/db-based-graph? repo)
     db-schema/schema
     file-schema/schema))
+
+(defn page-in-library?
+  "Check whether a `page` exists on the Library page"
+  [db page]
+  (when (page? page)
+    (when-let [library-page (get-built-in-page db common-config/library-page-name)]
+      (loop [parent (:block/parent page)]
+        (cond
+          (nil? parent)
+          false
+          (= (:db/id parent) (:db/id library-page))
+          true
+          :else
+          (recur (:block/parent parent)))))))

deps/db/src/logseq/db/common/delete_blocks.cljs

@@ -5,8 +5,8 @@
             [logseq.common.util :as common-util]
             [logseq.common.util.block-ref :as block-ref]
             [logseq.common.util.page-ref :as page-ref]
-            [logseq.db.common.entity-util :as common-entity-util]
             [logseq.db.common.entity-plus :as entity-plus]
+            [logseq.db.common.entity-util :as common-entity-util]
             [logseq.db.frontend.entity-util :as entity-util]))
 
 (defn- replace-ref-with-deleted-block-title

deps/db/src/logseq/db/common/entity_plus.cljc

@@ -95,8 +95,8 @@
        (get (.-kv e) k)
        (if db-based?
          (let [result (lookup-entity e k default-value)
-             ;; Replace title for pages only, otherwise it'll recursively
-             ;; replace block id refs if there're cycle references of blocks
+               ;; Replace title for pages only, otherwise it'll recursively
+               ;; replace block id refs if there're cycle references of blocks
                refs (:block/refs e)
                result' (if (and (string? result) refs)
                          (db-content/id-ref->title-ref result refs

deps/db/src/logseq/db/common/initial_data.cljs

@@ -344,6 +344,7 @@
                         :logseq.kv/graph-uuid
                         :logseq.kv/latest-code-lang
                         :logseq.kv/graph-backup-folder
+                        :logseq.kv/graph-text-embedding-model-name
                         :logseq.property/empty-placeholder])
         favorites (when db-graph? (get-favorites db))
         views (when db-graph? (get-views-data db))

deps/db/src/logseq/db/file_based/schema.cljs

@@ -32,8 +32,8 @@
    :block/path-refs {:db/valueType   :db.type/ref
                      :db/cardinality :db.cardinality/many}
 
-  :block/tags {:db/valueType :db.type/ref
-               :db/cardinality :db.cardinality/many}
+   :block/tags {:db/valueType :db.type/ref
+                :db/cardinality :db.cardinality/many}
 
    ;; which block this block links to, used for tag, embeds
    :block/link {:db/valueType :db.type/ref

deps/db/src/logseq/db/frontend/db_ident.cljc

@@ -68,24 +68,24 @@
   ([user-namespace name-string random-suffix?]
    {:pre [(or (keyword? user-namespace) (string? user-namespace)) (string? name-string) (boolean? random-suffix?)]}
    (assert (not (re-find #"^(logseq|block)(\.|$)" (name user-namespace)))
-     "New ident is not allowed to use an internal namespace")
+           "New ident is not allowed to use an internal namespace")
    (if #?(:org.babashka/nbb true
           :cljs             (or (false? random-suffix?)
-                              (and (exists? js/process)
-                                (or js/process.env.REPEATABLE_IDENTS js/process.env.DB_GRAPH)))
+                                (and (exists? js/process)
+                                     (or js/process.env.REPEATABLE_IDENTS js/process.env.DB_GRAPH)))
           :default          false)
      ;; Used for contexts where we want repeatable idents e.g. tests and CLIs
      (keyword user-namespace
-       (->> (string/replace-first name-string #"^(\d)" "NUM-$1")
+              (->> (string/replace-first name-string #"^(\d)" "NUM-$1")
          ;; '-' must go last in char class
-         (filter #(re-find #"[0-9a-zA-Z*+!_'?<>=-]{1}" %))
-         (apply str)))
+                   (filter #(re-find #"[0-9a-zA-Z*+!_'?<>=-]{1}" %))
+                   (apply str)))
      (keyword user-namespace
-       (str
-         (->> (string/replace-first name-string #"^(\d)" "NUM-$1")
+              (str
+               (->> (string/replace-first name-string #"^(\d)" "NUM-$1")
            ;; '-' must go last in char class
-           (filter #(re-find #"[0-9a-zA-Z*+!_'?<>=-]{1}" %))
-           (apply str))
-         "-"
-         (rand-nth non-int-char-range)
-         (nano-id 7))))))
+                    (filter #(re-find #"[0-9a-zA-Z*+!_'?<>=-]{1}" %))
+                    (apply str))
+               "-"
+               (rand-nth non-int-char-range)
+               (nano-id 7))))))

deps/db/src/logseq/db/frontend/kv_entity.cljs

@@ -26,6 +26,11 @@ RTC won't start when major-schema-versions don't match"
                                            :rtc {:rtc/ignore-entity-when-init-upload true
                                                  :rtc/ignore-entity-when-init-download true}}
   :logseq.kv/graph-initial-schema-version {:doc "Graph's schema version when created"}
+
   :logseq.kv/graph-last-gc-at             {:doc "Last time graph gc at"
                                            :rtc {:rtc/ignore-entity-when-init-upload true
-                                                 :rtc/ignore-entity-when-init-download true}})
+                                                 :rtc/ignore-entity-when-init-download true}}
+
+  :logseq.kv/graph-text-embedding-model-name   {:doc "Graph's text-embedding model name"
+                                                :rtc {:rtc/ignore-entity-when-init-upload true
+                                                      :rtc/ignore-entity-when-init-download true}})

deps/db/src/logseq/db/frontend/property.cljs

@@ -559,7 +559,14 @@
                                            :schema {:type :class
                                                     :cardinality :many
                                                     :public? true}
-                                           :queryable? true})))
+                                           :queryable? true}
+     :logseq.property.embedding/hnsw-label-updated-at {:title "HNSW label updated-at"
+                                                       :schema {:type :datetime
+                                                                :public? false
+                                                                :hide? true}
+                                                       :rtc {:rtc/ignore-attr-when-init-upload true
+                                                             :rtc/ignore-attr-when-init-download true
+                                                             :rtc/ignore-attr-when-syncing true}})))
 
 (def db-attribute-properties
   "Internal properties that are also db schema attributes"
@@ -611,7 +618,7 @@
     "logseq.property.linked-references" "logseq.property.asset" "logseq.property.table" "logseq.property.node"
     "logseq.property.code" "logseq.property.repeat"
     "logseq.property.journal" "logseq.property.class" "logseq.property.view"
-    "logseq.property.user" "logseq.property.history"})
+    "logseq.property.user" "logseq.property.history" "logseq.property.embedding"})
 
 (defn logseq-property?
   "Determines if keyword is a logseq property"

deps/db/src/logseq/db/frontend/schema.cljs

@@ -37,7 +37,7 @@
          (map (juxt :major :minor)
               [(parse-schema-version x) (parse-schema-version y)])))
 
-(def version (parse-schema-version "65.8"))
+(def version (parse-schema-version "65.9"))
 
 (defn major-version
   "Return a number.

deps/db/test/logseq/db/common/initial_data_test.cljs

@@ -6,8 +6,8 @@
             [cljs.test :refer [deftest async use-fixtures is testing]]
             [datascript.core :as d]
             [logseq.db.common.initial-data :as common-initial-data]
-            [logseq.db.sqlite.build :as sqlite-build]
             [logseq.db.common.sqlite-cli :as sqlite-cli]
+            [logseq.db.sqlite.build :as sqlite-build]
             [logseq.db.sqlite.create-graph :as sqlite-create-graph]
             [logseq.db.test.helper :as db-test]))
 

deps/db/test/logseq/db/sqlite/build_test.cljs

@@ -3,10 +3,10 @@
             [datascript.core :as d]
             [logseq.common.util.page-ref :as page-ref]
             [logseq.db :as ldb]
+            [logseq.db.frontend.entity-util :as entity-util]
             [logseq.db.frontend.property :as db-property]
             [logseq.db.sqlite.build :as sqlite-build]
-            [logseq.db.test.helper :as db-test]
-            [logseq.db.frontend.entity-util :as entity-util]))
+            [logseq.db.test.helper :as db-test]))
 
 (deftest build-tags
   (let [conn (db-test/create-conn)

deps/outliner/src/logseq/outliner/core.cljs

@@ -251,7 +251,7 @@
           collapse-or-expand? (= outliner-op :collapse-expand-blocks)
           m* (cond->
               (-> data'
-                  (dissoc :block/children :block/meta :block/unordered
+                  (dissoc :block/children :block/meta :block/unordered :block/path-refs
                           :block.temp/ast-title :block.temp/ast-body :block/level :block.temp/load-status
                           :block.temp/has-children?)
                   common-util/remove-nils
@@ -522,22 +522,22 @@
      block)))
 
 (defn- get-target-block-page
-  [target-block]
+  [target-block sibling?]
   (or
-   (when (ldb/page? target-block)
-     (:db/id target-block))
    (:db/id (:block/page target-block))
    ;; target parent is a page
-   (when-let [parent (:block/parent target-block)]
-     (when (ldb/page? parent)
-       (:db/id parent)))
+   (when sibling?
+     (when-let [parent (:block/parent target-block)]
+       (when (ldb/page? parent)
+         (:db/id parent))))
+
    ;; target-block is a page itself
    (:db/id target-block)))
 
 (defn- build-insert-blocks-tx
   [db target-block blocks uuids get-new-id {:keys [sibling? outliner-op replace-empty-target? insert-template? keep-block-order?]}]
   (let [block-ids (set (map :block/uuid blocks))
-        target-page (get-target-block-page target-block)
+        target-page (get-target-block-page target-block sibling?)
         orders (get-block-orders blocks target-block sibling? keep-block-order?)]
     (map-indexed (fn [idx {:block/keys [parent] :as block}]
                    (when-let [uuid' (get uuids (:block/uuid block))]
@@ -551,28 +551,31 @@
                                                       (let [ref-ids (set (map :block/uuid (:block/refs block)))]
                                                         (->> (set/intersection block-ids ref-ids)
                                                              (remove #{(:block/uuid block)})))))
-                           m (cond->
-                              {:db/id (:db/id block)
-                               :block/uuid uuid'
-                               :block/parent parent
-                               :block/order order}
-                               (not (ldb/page? block))
-                               (assoc :block/page target-page))
-                           result (->
-                                   (if (de/entity? block)
-                                     (assoc m :block/level (:block/level block))
-                                     (merge block m))
-                                   (update :block/title (fn [value]
-                                                          (if (seq template-ref-block-ids)
-                                                            (reduce
-                                                             (fn [value id]
-                                                               (string/replace value
-                                                                               (page-ref/->page-ref id)
-                                                                               (page-ref/->page-ref (uuids id))))
-                                                             value
-                                                             template-ref-block-ids)
-                                                            value)))
-                                   (dissoc :db/id))]
+                           m {:db/id (:db/id block)
+                              :block/uuid uuid'
+                              :block/parent parent
+                              :block/order order}
+                           result* (->
+                                    (if (de/entity? block)
+                                      (assoc m :block/level (:block/level block))
+                                      (merge block m))
+                                    (update :block/title (fn [value]
+                                                           (if (seq template-ref-block-ids)
+                                                             (reduce
+                                                              (fn [value id]
+                                                                (string/replace value
+                                                                                (page-ref/->page-ref id)
+                                                                                (page-ref/->page-ref (uuids id))))
+                                                              value
+                                                              template-ref-block-ids)
+                                                             value)))
+                                    (dissoc :db/id))
+                           page? (or (ldb/page? block) (:block/name block))
+                           result (cond-> result*
+                                    (not page?)
+                                    (assoc :block/page target-page)
+                                    page?
+                                    (dissoc :block/page))]
                        (update-property-ref-when-paste result uuids))))
                  blocks)))
 
@@ -608,7 +611,7 @@
      :id->new-uuid id->new-uuid}))
 
 (defn- get-target-block
-  [db blocks target-block {:keys [outliner-op indent? sibling? up?]}]
+  [db blocks target-block {:keys [outliner-op bottom? top? indent? sibling? up?]}]
   (when-let [block (if (:db/id target-block)
                      (d/entity db (:db/id target-block))
                      (when (:block/uuid target-block)
@@ -636,7 +639,16 @@
                              [block sibling?]
 
                              (contains? #{:insert-blocks :move-blocks} outliner-op)
-                             [block sibling?]
+                             (cond
+                               top?
+                               [block false]
+
+                               bottom?
+                               (if-let [last-child (last (ldb/sort-by-order (:block/_parent block)))]
+                                 [last-child true]
+                                 [block false])
+                               :else
+                               [block sibling?])
 
                              linked
                              (get-last-child-or-self db linked)
@@ -683,6 +695,8 @@
     `target-block`: where `blocks` will be inserted.
     Options:
       `sibling?`: as siblings (true) or children (false).
+      `bottom?`: inserts block to the bottom.
+      `top?`: inserts block to the top.
       `keep-uuid?`: whether to replace `:block/uuid` from the parameter `blocks`.
                     For example, if `blocks` are from internal copy, the uuids
                     need to be changed, but there's no need for internal cut or drag & drop.
@@ -854,7 +868,7 @@
   (let [target-block (d/entity db (:db/id target-block))
         block (d/entity db (:db/id block))
         first-block-page (:db/id (:block/page block))
-        target-page (get-target-block-page target-block)
+        target-page (get-target-block-page target-block sibling?)
         not-same-page? (not= first-block-page target-page)
         block-order (if sibling?
                       (db-order/gen-key (:block/order target-block)
@@ -890,7 +904,7 @@
 
 (defn- move-blocks
   "Move `blocks` to `target-block` as siblings or children."
-  [_repo conn blocks target-block {:keys [_sibling? _up? outliner-op _indent?]
+  [_repo conn blocks target-block {:keys [_sibling? _top? _bottom? _up? outliner-op _indent?]
                                    :as opts}]
   {:pre [(seq blocks)
          (m/validate block-map-or-entity target-block)]}
@@ -1065,9 +1079,10 @@
     (op-transact! f repo conn blocks opts)))
 
 (defn move-blocks!
-  [repo conn blocks target-block sibling?]
-  (op-transact! move-blocks repo conn blocks target-block {:sibling? sibling?
-                                                           :outliner-op :move-blocks}))
+  [repo conn blocks target-block opts]
+  (op-transact! move-blocks repo conn blocks target-block
+                (assoc opts :outliner-op :move-blocks)))
+
 (defn move-blocks-up-down!
   [repo conn blocks up?]
   (op-transact! move-blocks-up-down repo conn blocks up?))

deps/outliner/src/logseq/outliner/op.cljs

@@ -26,7 +26,7 @@
    [:move-blocks
     [:catn
      [:op :keyword]
-     [:args [:tuple ::ids ::id :boolean]]]]
+     [:args [:tuple ::ids ::id ::option]]]]
    [:move-blocks-up-down
     [:catn
      [:op :keyword]
@@ -53,6 +53,10 @@
     [:catn
      [:op :keyword]
      [:args [:tuple ::block-id ::property-id ::value]]]]
+   [:batch-delete-property-value
+    [:catn
+     [:op :keyword]
+     [:args [:tuple ::block-ids ::property-id ::value]]]]
    [:create-property-text-block
     [:catn
      [:op :keyword]
@@ -171,11 +175,11 @@
            (outliner-core/delete-blocks! repo conn date-formatter blocks (merge opts opts')))
 
          :move-blocks
-         (let [[block-ids target-block-id sibling?] args
+         (let [[block-ids target-block-id opts] args
                blocks (keep #(d/entity @conn %) block-ids)
                target-block (d/entity @conn target-block-id)]
            (when (and target-block (seq blocks))
-             (outliner-core/move-blocks! repo conn blocks target-block sibling?)))
+             (outliner-core/move-blocks! repo conn blocks target-block opts)))
 
          :move-blocks-up-down
          (let [[block-ids up?] args
@@ -211,6 +215,9 @@
          :batch-remove-property
          (apply outliner-property/batch-remove-property! conn args)
 
+         :batch-delete-property-value
+         (apply outliner-property/batch-delete-property-value! conn args)
+
          :class-add-property
          (apply outliner-property/class-add-property! conn args)
 

deps/outliner/src/logseq/outliner/pipeline.cljs

@@ -50,7 +50,7 @@
                                    (recur (into parent-refs (:block-ref-ids parent))
                                           (:parent-id parent))
                                    ;; exits when top-level parent is reached
-                                   parent-refs)))})
+                                   (remove nil? parent-refs))))})
                            children-maps)]
     children-refs))
 
@@ -87,10 +87,13 @@
                        old-refs (if db-before
                                   (set (map :db/id (:block/path-refs (d/entity db-before (:db/id block)))))
                                   #{})
-                       new-refs (set (concat
-                                      (some-> (:db/id (:block/page block)) vector)
-                                      (map :db/id (:block/refs block))
-                                      parents-refs))
+                       new-refs (->>
+                                 (concat
+                                  (some-> (:db/id (:block/page block)) vector)
+                                  (map :db/id (:block/refs block))
+                                  parents-refs)
+                                 (remove nil?)
+                                 set)
                        refs-changed? (not= old-refs new-refs)
                        children (when refs-changed?
                                   (when-not page?

deps/outliner/src/logseq/outliner/property.cljs

@@ -520,22 +520,31 @@
                          {:outliner-op :upsert-property}))
         (d/entity @conn db-ident')))))
 
+(defn batch-delete-property-value!
+  "batch delete value when a property has multiple values"
+  [conn block-eids property-id property-value]
+  (when-let [property (d/entity @conn property-id)]
+    (when (and (db-property/many? property)
+               (not (some #(= property-id (:db/ident (d/entity @conn %))) block-eids)))
+      (when (= property-id :block/tags)
+        (outliner-validate/validate-tags-property-deletion @conn block-eids property-value))
+      (if (= property-id :block/tags)
+        (let [tx-data (map (fn [id] [:db/retract id property-id property-value]) block-eids)]
+          (ldb/transact! conn tx-data {:outliner-op :save-block}))
+        (doseq [block-eid block-eids]
+          (when-let [block (d/entity @conn block-eid)]
+            (let [current-val (get block property-id)
+                  fv (first current-val)]
+              (if (and (= 1 (count current-val)) (or (= property-value fv) (= property-value (:db/id fv))))
+                (remove-block-property! conn (:db/id block) property-id)
+                (ldb/transact! conn
+                               [[:db/retract (:db/id block) property-id property-value]]
+                               {:outliner-op :save-block})))))))))
+
 (defn delete-property-value!
   "Delete value if a property has multiple values"
   [conn block-eid property-id property-value]
-  (when-let [property (d/entity @conn property-id)]
-    (let [block (d/entity @conn block-eid)]
-      (when (and block (not= property-id (:db/ident block)) (db-property/many? property))
-        (when (= property-id :block/tags)
-          (outliner-validate/validate-tags-property-deletion @conn [block-eid] property-value))
-
-        (let [current-val (get block property-id)
-              fv (first current-val)]
-          (if (and (= 1 (count current-val)) (or (= property-value fv) (= property-value (:db/id fv))))
-            (remove-block-property! conn (:db/id block) property-id)
-            (ldb/transact! conn
-                           [[:db/retract (:db/id block) property-id property-value]]
-                           {:outliner-op :save-block})))))))
+  (batch-delete-property-value! conn [block-eid] property-id property-value))
 
 (defn ^:api get-classes-parents
   [tags]

deps/outliner/src/logseq/outliner/validate.cljs

@@ -204,11 +204,13 @@
 
 (defn- disallow-node-cant-tag-with-private-tags
   [db block-eids v & {:keys [delete?]}]
-  (when (and (ldb/private-tags (:db/ident (d/entity db v)))
-             ;; Allow assets to be tagged
-             (not (and
-                   (every? (fn [id] (ldb/asset? (d/entity db id))) block-eids)
-                   (= :logseq.class/Asset (:db/ident (d/entity db v))))))
+  ;; Skip #Page as it is validated by later fns
+  (when (and (contains? (disj ldb/private-tags :logseq.class/Page) (:db/ident (d/entity db v)))
+             (not
+               ;; Allow assets to be tagged
+              (and
+               (every? (fn [id] (ldb/asset? (d/entity db id))) block-eids)
+               (= :logseq.class/Asset (:db/ident (d/entity db v))))))
     (throw (ex-info (str (if delete? "Can't remove tag" "Can't set tag")
                          " with built-in #" (:block/title (d/entity db v)))
                     {:type :notification
@@ -229,15 +231,84 @@
                                              " on built-in " (pr-str (:block/title built-in-ent)))
                                :type :error}}))))
 
+(defn- disallow-removing-page-tag
+  "Disallow page->block when
+  1. this page doesn't have :block/parent
+  2. its parent is Library
+  3. it has page child"
+  [db eids v]
+  (when (= (:db/ident (d/entity db v)) :logseq.class/Page)
+    (let [library-page (ldb/get-library-page db)]
+      (doseq [eid eids]
+        (let [entity (d/entity db eid)]
+          (when (ldb/internal-page? entity)
+            (cond
+              (not (:block/parent entity))
+              (throw (ex-info "This page cannot be converted to a block"
+                              {:type :notification
+                               :payload
+                               {:message (str "Page " (pr-str (:block/title entity)) " cannot be converted to a block")
+                                :type :error
+                                :entity (into {} entity)
+                                :property :block/tags}}))
+              (= (:db/id library-page) (:db/id (:block/parent entity)))
+              (throw (ex-info "This page cannot be converted to a block"
+                              {:type :notification
+                               :payload
+                               {:message (str "Page " (pr-str (:block/title entity)) " cannot be converted to a block, please move it to another page first")
+                                :type :error
+                                :entity (into {} entity)
+                                :property :block/tags}}))
+              (some entity-util/page? (:block/_parent entity))
+              (throw (ex-info "This page cannot be converted to a block"
+                              {:type :notification
+                               :payload
+                               {:message (str "Page " (pr-str (:block/title entity)) " cannot be converted to a block because it has page children")
+                                :type :error
+                                :entity (into {} entity)
+                                :property :block/tags}})))))))))
+
+(defn- validate-block-can-tag-with-page-tag
+  "Validates block can convert to page by adding #Page for allowed scenarios"
+  [db eids v]
+  (when (= (:db/ident (d/entity db v)) :logseq.class/Page)
+    (doseq [eid eids]
+      (let [block (d/entity db eid)]
+        (when (:block/parent block)
+          (validate-page-title (:block/title block) {:node block})
+          (validate-page-title-characters (:block/title block) {:node block})
+
+          ;; Only allow block to be page when its parent is a page to guard against invalid pages
+          ;; in property values or pages being created with blocks as namespace parents
+          (when (or (not (entity-util/page? (:block/parent block)))
+                    (:logseq.property/created-from-property block))
+            (let [message (if (:logseq.property/created-from-property block)
+                            "Can't convert property value to page."
+                            "Can't convert this block to page since its parent is not a page.")]
+              (throw (ex-info message
+                              {:type :notification
+                               :payload {:message message
+                                         :type :error
+                                         :block (into {} block)}}))))
+          ;; Guard against classes and properties becoming namespace parents
+          (when (or (entity-util/class? (:block/page block)) (entity-util/property? (:block/page block)))
+            (throw (ex-info "Can't convert this block to page when block is in a property or tag."
+                            {:type :notification
+                             :payload {:message "Can't convert this block to page when block is in a property or tag."
+                                       :type :error
+                                       :block (into {} block)}}))))))))
+
 (defn validate-tags-property
   "Validates adding a property value to :block/tags for given blocks"
   [db block-eids v]
   (disallow-tagging-a-built-in-entity db block-eids)
   (disallow-node-cant-tag-with-private-tags db block-eids v)
+  (validate-block-can-tag-with-page-tag db block-eids v)
   (disallow-node-cant-tag-with-built-in-non-tags db block-eids v))
 
 (defn validate-tags-property-deletion
   "Validates deleting a property value from :block/tags for given blocks"
   [db block-eids v]
   (disallow-tagging-a-built-in-entity db block-eids {:delete? true})
-  (disallow-node-cant-tag-with-private-tags db block-eids v {:delete? true}))
+  (disallow-node-cant-tag-with-private-tags db block-eids v {:delete? true})
+  (disallow-removing-page-tag db block-eids v))

deps/outliner/test/logseq/outliner/validate_test.cljs

@@ -158,12 +158,21 @@
                                                         [(entity-plus/entity-memoized @conn :logseq.class/Cards)]))))))
 
 (deftest validate-tags-property
-  (let [conn (db-test/create-conn-with-blocks
-              {:classes {:SomeTag {}}
+  (let [class-uuid (random-uuid)
+        conn (db-test/create-conn-with-blocks
+              {:classes {:SomeTag {:block/uuid class-uuid :build/keep-uuid? true}}
                :pages-and-blocks
                [{:page {:block/title "page1"}
-                 :blocks [{:block/title "block"}]}]})
-        block (db-test/find-block-by-content @conn "block")]
+                 :blocks [{:block/title "block"
+                           :build/children [{:block/title "block - invalid location"}]}
+                          {:block/title "block / invalid title"}]}
+                {:page {:block/uuid class-uuid}
+                 :blocks [{:block/title "class block"}]}]
+               :build-existing-tx? true})
+        block (db-test/find-block-by-content @conn "block")
+        block-invalid-title (db-test/find-block-by-content @conn #"invalid title")
+        block-invalid-location (db-test/find-block-by-content @conn #"invalid location")
+        block-invalid-in-class (db-test/find-block-by-content @conn "class block")]
 
     (is (thrown-with-msg?
          js/Error
@@ -185,15 +194,66 @@
 
     (is (thrown-with-msg?
          js/Error
-         #"Can't set tag.*Page"
-         (outliner-validate/validate-tags-property @conn [(:db/id block)] :logseq.class/Page))
+         #"Can't set tag.*Tag"
+         (outliner-validate/validate-tags-property @conn [(:db/id block)] :logseq.class/Tag))
         "Nodes can't be tagged with built-in private tags")
 
     (is (thrown-with-msg?
          js/Error
          #"Can't set tag.*Priority"
          (outliner-validate/validate-tags-property @conn [(:db/id block)] :logseq.property/priority))
-        "Nodes can't be tagged with built-in non tags")))
+        "Nodes can't be tagged with built-in non tags")
+
+    (is (nil? (outliner-validate/validate-tags-property @conn [(:db/id block)] :logseq.class/Page))
+        "Blocks can be tagged with #Page")
+
+    (is (thrown-with-msg?
+         js/Error
+         #"Page name can't.*/"
+         (outliner-validate/validate-tags-property @conn [(:db/id block-invalid-title)] :logseq.class/Page))
+        "Block with invalid title can't be tagged with #Page")
+
+    (is (thrown-with-msg?
+         js/Error
+         #"Can't convert this block to page"
+         (outliner-validate/validate-tags-property @conn [(:db/id block-invalid-location)] :logseq.class/Page))
+        "Block with invalid location can't be tagged with #Page")
+
+    (is (thrown-with-msg?
+         js/Error
+         #"Can't convert this block to page"
+         (outliner-validate/validate-tags-property @conn [(:db/id block-invalid-in-class)] :logseq.class/Page))
+        "Block in class or property can't be tagged with #Page")))
+
+(deftest validate-tags-property-deletion
+  (let [conn (db-test/create-conn-with-blocks
+              {:classes {:SomeTag {}}
+               :pages-and-blocks
+               [{:page {:block/title "page1"}
+                 :blocks [{:block/title "block" :build/tags [:logseq.class/Page]}]}]})
+        page (db-test/find-page-by-title @conn "page1")
+        page-with-parent (db-test/find-block-by-content @conn "block")]
+
+    (is (thrown-with-msg?
+         js/Error
+         #"Can't remove tag.*Task"
+         (outliner-validate/validate-tags-property-deletion @conn [(:db/id (d/entity @conn :logseq.class/Task))] :logseq.class/Tag))
+        "built-in class must not have tag deleted by the user")
+
+    (is (thrown-with-msg?
+         js/Error
+         #"Can't remove tag.*Tag"
+         (outliner-validate/validate-tags-property-deletion @conn [(:db/id (d/entity @conn :user.class/SomeTag))] :logseq.class/Tag))
+        "Node can't have private tag deleted by user")
+
+    (is (nil? (outliner-validate/validate-tags-property-deletion @conn [(:db/id page-with-parent)] :logseq.class/Page))
+        "Page with parent can remove #Page")
+
+    (is (thrown-with-msg?
+         js/Error
+         #"This page cannot be converted"
+         (outliner-validate/validate-tags-property-deletion @conn [(:db/id page)] :logseq.class/Page))
+        "Page without parent can't remove #Page")))
 
 ;; Try as many of the validations against a new graph to confirm
 ;; that validations make sense and are valid for a new graph

deps/publishing/src/logseq/publishing/export.cljs

@@ -1,14 +1,14 @@
 (ns ^:node-only logseq.publishing.export
   "This electron only ns (for the main process) exports files from multiple
   locations to provide a complete publishing app"
-  (:require ["fs-extra" :as fse]
+  (:require ["fs" :as fs]
+            ["fs-extra" :as fse]
             ["path" :as node-path]
-            ["fs" :as fs]
             [promesa.core :as p]))
 
 (def ^:api js-files
   "js files from publishing release build"
-  (->> ["shared.js" "main.js" "code-editor.js" "excalidraw.js" "tldraw.js" "db-worker.js"]
+  (->> ["main.js" "code-editor.js" "excalidraw.js" "tldraw.js"]
        ;; Add source maps for all js files as it doesn't affect initial load time
        (mapcat #(vector % (str % ".map")))
        vec))

deps/publishing/src/logseq/publishing/html.cljs

@@ -124,7 +124,7 @@ necessary db filtering"
             [:script {:src "static/js/react.production.min.js"}]
             [:script {:src "static/js/react-dom.production.min.js"}]
             [:script {:src "static/js/ui.js"}]
-            [:script {:src "static/js/shared.js"}]
+            [:script {:src "static/js/main-bundle.js"}]
             [:script {:src "static/js/main.js"}]
             ;; Deferring scripts above results in errors
             [:script {:defer true :src "static/js/interact.min.js"}]

deps/shui/src/logseq/shui/util.cljs

@@ -1,13 +1,13 @@
 (ns logseq.shui.util
   (:require
-   [clojure.string :as s]
-   [rum.core :refer [use-state use-effect!] :as rum]
-   [logseq.shui.rum :as shui-rum]
-   [goog.object :refer [getValueByKeys] :as gobj]
+   [cljs-bean.core :as bean]
    [clojure.set :refer [rename-keys]]
+   [clojure.string :as s]
    [clojure.walk :as w]
-   [cljs-bean.core :as bean]
-   [goog.dom :as gdom]))
+   [goog.dom :as gdom]
+   [goog.object :refer [getValueByKeys] :as gobj]
+   [logseq.shui.rum :as shui-rum]
+   [rum.core :refer [use-state use-effect!] :as rum]))
 
 (goog-define NODETEST false)
 
@@ -16,7 +16,7 @@
   [input]
   (let [words (s/split input #"-")
         capitalize (->> (rest words)
-                     (map #(apply str (s/upper-case (first %)) (rest %))))]
+                        (map #(apply str (s/upper-case (first %)) (rest %))))]
     (apply str (first words) capitalize)))
 
 (defn map-keys->camel-case
@@ -28,7 +28,7 @@
   (let [convert-to-camel (fn [[key value]]
                            (let [k (name key)]
                              [(if-not (or (s/starts-with? k "data-")
-                                        (s/starts-with? k "aria-"))
+                                          (s/starts-with? k "aria-"))
                                 (kebab-case->camel-case k) k) value]))]
     (w/postwalk (fn [x]
                   (if (map? x)
@@ -37,7 +37,7 @@
                                     x)]
                       (into {} (map convert-to-camel new-map)))
                     x))
-      data)))
+                data)))
 
 (defn $LSUtils [] (aget js/window "LSUtils"))
 (def dev? (some-> ($LSUtils) (aget "isDev")))
@@ -45,11 +45,11 @@
 (defn uuid-color
   [uuid-str]
   (some-> ($LSUtils) (aget "uniqolor")
-    (apply [uuid-str
-            #js {:saturation #js [55, 70],
-                 :lightness 70,
-                 :differencePoint 60}])
-    (aget "color")))
+          (apply [uuid-str
+                  #js {:saturation #js [55, 70],
+                       :lightness 70,
+                       :differencePoint 60}])
+          (aget "color")))
 
 (defn get-path
   "Returns the component path."
@@ -67,10 +67,10 @@
         ;; we have to make sure to check if the children is sequential
         ;; as a list can be returned, eg: from a (for)
         new-children (if (and (not (nil? children#))
-                           (not (empty? children))
-                           (or (not (array? children#))
+                              (not (empty? children))
+                              (or (not (array? children#))
                              ;; maybe list children
-                             (not (vector? type#))))
+                                  (not (vector? type#))))
                        [children#] children#)
 
         ;; convert any options key value to a React element, if
@@ -83,22 +83,22 @@
         react-class (if dev? (react-class) react-class)]
     (apply js/React.createElement react-class
       ;; sablono html-to-dom-attrs does not work for nested hash-maps
-      (bean/->js (map-keys->camel-case new-options :html-props true))
-      new-children)))
+           (bean/->js (map-keys->camel-case new-options :html-props true))
+           new-children)))
 
 (defn use-atom-fn
   [a getter-fn setter-fn]
   (let [[val set-val] (use-state (getter-fn @a))]
     (use-effect!
-      (fn []
-        (let [id (str (random-uuid))]
-          (add-watch a id (fn [_ _ prev-state next-state]
-                            (let [prev-value (getter-fn prev-state)
-                                  next-value (getter-fn next-state)]
-                              (when-not (= prev-value next-value)
-                                (set-val next-value)))))
-          #(remove-watch a id)))
-      [])
+     (fn []
+       (let [id (str (random-uuid))]
+         (add-watch a id (fn [_ _ prev-state next-state]
+                           (let [prev-value (getter-fn prev-state)
+                                 next-value (getter-fn next-state)]
+                             (when-not (= prev-value next-value)
+                               (set-val next-value)))))
+         #(remove-watch a id)))
+     [])
     [val #(swap! a setter-fn %)]))
 
 (defn use-atom
@@ -110,10 +110,10 @@
   []
   (let [*mounted (rum/use-ref false)]
     (use-effect!
-      (fn []
-        (rum/set-ref! *mounted true)
-        #(rum/set-ref! *mounted false))
-      [])
+     (fn []
+       (rum/set-ref! *mounted true)
+       #(rum/set-ref! *mounted false))
+     [])
     #(rum/deref *mounted)))
 
 (defn react->rum [c static?]

externs/app.txt

@@ -0,0 +1,86 @@
+require
+DEBUG
+convertFileSrc
+getPlatform
+handleError
+isLoggingEnabled
+isNative
+isNativePlatform
+isPluginAvailable
+platform
+pluginMethodNoop
+registerPlugin
+registerHelper
+Pass
+showHint
+createRef
+call
+setState
+forceUpdate
+memo
+createElement
+render
+unmount
+createContext
+createPortal
+createRef
+createElement
+ref
+path
+current
+useState
+useReducer
+useEffect
+useLayoutEffect
+useCallback
+useMemo
+useRef
+renderToString
+idb-keyval
+newStore
+isError
+value
+CodeMirror
+over
+forEachNode
+on
+getTextArea
+setUser
+nativeEvent
+which
+multipliedBy
+minus
+plus
+isInteger
+isPositive
+isLessThan
+toNumber
+modulo
+exponentiatedBy
+dividedBy
+multipliedBy
+negated
+isZero
+isGreaterThanOrEqualTo
+isLessThan
+shiftedBy
+s
+dividedToIntegerBy
+abs
+toFraction
+e
+getCursor
+getLineTokens
+getTokenAt
+getValue
+completionActive
+state
+mime
+getWrapperElement
+doc
+lastLine
+getLine
+setOption
+string
+dispose
+path

externs/mobile.txt

@@ -0,0 +1,87 @@
+require
+DEBUG
+convertFileSrc
+getPlatform
+handleError
+isLoggingEnabled
+isNative
+isNativePlatform
+isPluginAvailable
+platform
+pluginMethodNoop
+registerPlugin
+registerHelper
+Pass
+showHint
+createRef
+call
+setState
+forceUpdate
+memo
+createElement
+render
+unmount
+createContext
+createPortal
+createRef
+createElement
+ref
+path
+current
+useState
+useReducer
+useEffect
+useLayoutEffect
+useCallback
+useMemo
+useRef
+renderToString
+idb-keyval
+newStore
+isError
+value
+CodeMirror
+over
+forEachNode
+on
+getTextArea
+setUser
+nativeEvent
+which
+multipliedBy
+minus
+plus
+isInteger
+isPositive
+isLessThan
+toNumber
+modulo
+exponentiatedBy
+dividedBy
+multipliedBy
+negated
+isZero
+isGreaterThanOrEqualTo
+isLessThan
+shiftedBy
+s
+dividedToIntegerBy
+abs
+toFraction
+e
+getCursor
+getLineTokens
+getTokenAt
+getValue
+completionActive
+state
+mime
+getWrapperElement
+doc
+lastLine
+getLine
+setOption
+string
+dispose
+path
+base64String

gulpfile.js

@@ -78,7 +78,7 @@ const common = {
         'node_modules/@isomorphic-git/lightning-fs/dist/lightning-fs.min.js',
         'packages/amplify/dist/amplify.js',
         'packages/ui/dist/ui/ui.js',
-        'node_modules/@logseq/sqlite-wasm/sqlite-wasm/jswasm/sqlite3.wasm',
+        'node_modules/@sqlite.org/sqlite-wasm/sqlite-wasm/jswasm/sqlite3.wasm',
         'node_modules/react/umd/react.production.min.js',
         'node_modules/react/umd/react.development.js',
         'node_modules/react-dom/umd/react-dom.production.min.js',
@@ -129,7 +129,7 @@ const common = {
         'node_modules/photoswipe/dist/umd/*.js',
         'packages/amplify/dist/amplify.js',
         'packages/ui/dist/ui/ui.js',
-        'node_modules/@logseq/sqlite-wasm/sqlite-wasm/jswasm/sqlite3.wasm',
+        'node_modules/@sqlite.org/sqlite-wasm/sqlite-wasm/jswasm/sqlite3.wasm',
       ]).pipe(gulp.dest(path.join(outputPath, 'mobile', 'js'))),
       () => gulp.src([
         'packages/ui/dist/silkhq/*.css*',

package.json

@@ -36,27 +36,36 @@
         "postcss-import": "15.0.0",
         "postcss-import-ext-glob": "2.0.1",
         "postcss-nested": "6.0.0",
+        "process": "^0.11.10",
         "purgecss": "4.0.2",
         "semver": "7.5.2",
-        "shadow-cljs": "2.26.0",
+        "shadow-cljs": "2.28.23",
         "stylelint": "^13.8.0",
         "stylelint-config-standard": "^20.0.0",
         "tailwindcss": "3.3.5",
         "tailwindcss-animate": "^1.0.7",
-        "typescript": "^4.4.3"
+        "typescript": "^4.4.3",
+        "webpack": "^5.98.0",
+        "webpack-cli": "^6.0.1"
     },
     "scripts": {
-        "watch": "run-p gulp:watch cljs:watch",
-        "electron-watch": "run-p gulp:watch cljs:electron-watch",
-        "app-watch": "run-p gulp:watch cljs:app-watch",
-        "dev": "run-p gulp:watch gulp:mobile-watch cljs:dev-watch",
-        "release": "run-s gulp:build cljs:release",
-        "release-app": "run-s gulp:build cljs:release-app",
-        "release-mobile": "run-s gulp:buildMobile cljs:release-mobile",
-        "dev-release-app": "run-s gulp:build cljs:dev-release-app",
+        "watch": "run-p gulp:watch cljs:watch webpack-app-watch",
+        "electron-watch": "run-p gulp:watch cljs:electron-watch webpack-app-watch",
+        "app-watch": "run-p gulp:watch cljs:app-watch webpack-app-watch",
+        "mobile-watch": "run-p gulp:mobile-watch cljs:mobile-watch webpack-mobile-watch",
+        "dev": "run-p gulp:watch gulp:mobile-watch cljs:dev-watch webpack",
+        "release": "run-s gulp:build cljs:release webpack-app-build",
+        "release-app": "run-s gulp:build cljs:release-app webpack-app-build",
+        "release-mobile": "run-s gulp:buildMobile cljs:release-mobile webpack-mobile-build",
+        "dev-release-app": "run-s gulp:build cljs:dev-release-app webpack-app-build",
         "dev-electron-app": "gulp electron",
-        "release-electron": "run-s gulp:build && gulp electronMaker",
+        "release-electron": "run-s gulp:build && gulp electronMaker webpack-app-build",
         "debug-electron": "cd static/ && yarn electron:debug",
+        "webpack": "webpack --watch",
+        "webpack-app-watch": "npx webpack --watch --config-name app",
+        "webpack-mobile-watch": "npx webpack --watch --config-name mobile",
+        "webpack-app-build": "npx webpack build --config-name app",
+        "webpack-mobile-build": "npx webpack build --config-name mobile",
         "sync-android-release": "yarn clean && yarn release-mobile && rm -rf ./static/mobile/**/*.map && npx cap sync android",
         "sync-ios-release": "yarn clean && yarn release-mobile && rm -rf ./static/mobile/**/*.map && npx cap sync ios",
         "clean": "gulp clean",
@@ -68,27 +77,27 @@
         "gulp:buildMobile": "cross-env NODE_ENV=production gulp buildMobile",
         "css:build": "postcss tailwind.all.css -o static/css/style.css --verbose --env production",
         "css:watch": "cross-env TAILWIND_MODE=watch postcss tailwind.all.css -o static/css/style.css --verbose --watch",
-        "cljs:watch": "clojure -M:cljs watch app electron",
+        "cljs:watch": "clojure -M:cljs watch app workers electron",
         "cljs:storybook-watch": "clojure -M:cljs watch stories-dev",
         "gulp:mobile-watch": "gulp watchMobile",
-        "css:mobile-build": "postcss tailwind.mobile.css -o static/mobile/css/style.css --verbose --env production",
-        "css:mobile-watch": "cross-env TAILWIND_MODE=watch postcss tailwind.mobile.css -o static/mobile/css/style.css --verbose --watch",
-        "cljs:mobile-watch": "clojure -M:cljs watch mobile",
-        "cljs:release-mobile": "clojure -M:cljs release mobile",
-        "cljs:dev-watch": "clojure -M:cljs watch app electron mobile",
-        "cljs:app-watch": "clojure -M:cljs watch app",
-        "cljs:electron-watch": "clojure -M:cljs watch app electron --config-merge \"{:asset-path \\\"./js\\\"}\"",
-        "cljs:release": "clojure -M:cljs release app publishing electron",
-        "cljs:release-electron": "clojure -M:cljs release app electron --debug && clojure -M:cljs release publishing",
-        "cljs:release-app": "clojure -M:cljs release app",
-        "cljs:release-publishing": "clojure -M:cljs release publishing",
+        "css:mobile-build": "postcss tailwind.mobile.css -o static/mobile/style.css --verbose --env production",
+        "css:mobile-watch": "cross-env TAILWIND_MODE=watch postcss tailwind.mobile.css -o static/mobile/style.css --verbose --watch",
+        "cljs:mobile-watch": "clojure -M:cljs watch mobile workers --config-merge \"{:output-dir \\\"./static/mobile/js\\\" :asset-path \\\"/static/mobile/js\\\" :release {:asset-path \\\"http://localhost\\\"}}\"",
+        "cljs:release-mobile": "clojure -M:cljs release mobile workers --config-merge \"{:output-dir \\\"./static/mobile/js\\\" :asset-path \\\"/static/mobile/js\\\" :release {:asset-path \\\"http://localhost\\\"}}\"",
+        "cljs:dev-watch": "clojure -M:cljs watch app workers electron mobile",
+        "cljs:app-watch": "clojure -M:cljs watch app workers",
+        "cljs:electron-watch": "clojure -M:cljs watch app workers electron --config-merge \"{:asset-path \\\"./js\\\"}\"",
+        "cljs:release": "clojure -M:cljs release app workers publishing electron",
+        "cljs:release-electron": "clojure -M:cljs release app workers electron --debug && clojure -M:cljs release publishing",
+        "cljs:release-app": "clojure -M:cljs release app workers",
+        "cljs:release-publishing": "clojure -M:cljs release app publishing",
         "cljs:test": "clojure -M:test compile test",
         "cljs:run-test": "node static/tests.js",
-        "cljs:dev-release-app": "clojure -M:cljs release app --config-merge \"{:closure-defines {frontend.config/DEV-RELEASE true}}\"",
-        "cljs:dev-release-electron": "clojure -M:cljs release app electron --debug --config-merge \"{:closure-defines {frontend.config/DEV-RELEASE true}}\" && clojure -M:cljs release publishing",
-        "cljs:debug": "clojure -M:cljs release app --debug",
-        "cljs:report": "clojure -M:cljs run shadow.cljs.build-report app report.html",
-        "cljs:build-electron": "clojure -A:cljs compile app electron",
+        "cljs:dev-release-app": "clojure -M:cljs release app workers --config-merge \"{:closure-defines {frontend.config/DEV-RELEASE true}}\"",
+        "cljs:dev-release-electron": "clojure -M:cljs release app workers electron --debug --config-merge \"{:closure-defines {frontend.config/DEV-RELEASE true}}\" && clojure -M:cljs release publishing",
+        "cljs:debug": "clojure -M:cljs release app workers --debug",
+        "cljs:report": "clojure -M:cljs run shadow.cljs.build-report app workers report.html",
+        "cljs:build-electron": "clojure -A:cljs compile app workers electron",
         "cljs:lint": "clojure -M:clj-kondo --parallel --lint src --cache false",
         "ios:dev": "cross-env PLATFORM=ios gulp cap",
         "android:dev": "cross-env PLATFORM=android gulp cap",
@@ -122,16 +131,17 @@
         "@excalidraw/excalidraw": "0.16.1",
         "@glidejs/glide": "^3.6.0",
         "@highlightjs/cdn-assets": "10.4.1",
+        "@huggingface/transformers": "^3.6.3",
         "@isomorphic-git/lightning-fs": "^4.6.0",
         "@js-joda/core": "3.2.0",
         "@js-joda/locale_en-us": "3.1.1",
         "@js-joda/timezone": "2.5.0",
         "@logseq/diff-merge": "^0.2.2",
         "@logseq/react-tweet-embed": "1.3.1-1",
-        "@logseq/sqlite-wasm": "=0.1.0",
         "@radix-ui/colors": "^0.1.8",
         "@sentry/react": "^6.18.2",
         "@sentry/tracing": "^6.18.2",
+        "@sqlite.org/sqlite-wasm": "^3.50.3-build1",
         "@tabler/icons-react": "^2.47.0",
         "@tabler/icons-webfont": "^2.47.0",
         "@tippyjs/react": "4.2.5",
@@ -153,6 +163,7 @@
         "fuse.js": "6.4.6",
         "grapheme-splitter": "1.0.4",
         "graphology": "0.20.0",
+        "hnswlib-wasm": "^0.8.2",
         "html2canvas": "^1.4.1",
         "ignore": "5.1.8",
         "inter-ui": "^3.19.3",
@@ -173,7 +184,6 @@
         "react-dom": "18.3.1",
         "react-grid-layout": "0.16.6",
         "react-intersection-observer": "^9.3.5",
-        "react-resize-context": "3.0.0",
         "react-textarea-autosize": "8.3.3",
         "react-transition-group": "4.3.0",
         "react-virtuoso": "4.12.5",
@@ -184,6 +194,7 @@
         "tailwind-capitalize-first-letter": "^1.0.4",
         "threads": "1.6.5",
         "url": "^0.11.0",
+        "util": "^0.12.5",
         "yargs-parser": "20.2.4"
     },
     "resolutions": {

public/index.html

@@ -0,0 +1,69 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <meta charset="utf-8">
+  <!-- Handle iOS indexedDB preloading. Ref: https://developer.apple.com/forums/thread/681201 -->
+  <script type="text/javascript">
+    window.indexedDB;
+  </script>
+  <meta content="minimum-scale=1, initial-scale=1, maximum-scale=1, width=device-width, shrink-to-fit=no" name="viewport">
+  <link href="/static/css/style.css" rel="stylesheet" type="text/css">
+  <link href="/static/img/logo.png" rel="shortcut icon" type="image/png">
+  <link href="/static/img/logo.png" rel="shortcut icon" sizes="192x192">
+  <link href="/static/img/logo.png" rel="apple-touch-icon">
+  <meta content="Logseq" name="apple-mobile-web-app-title">
+  <meta content="yes" name="apple-mobile-web-app-capable">
+  <meta content="yes" name="apple-touch-fullscreen">
+  <meta content="black-translucent" name="apple-mobile-web-app-status-bar-style">
+  <meta content="yes" name="mobile-web-app-capable">
+  <meta content="summary" name="twitter:card">
+  <meta content="A privacy-first, open-source platform for knowledge management and collaboration." name="twitter:description">
+  <meta content="@logseq" name="twitter:site">
+  <meta content="A privacy-first, open-source platform for knowledge management and collaboration." name="twitter:title">
+  <meta content="https://asset.logseq.com/static/img/logo.png" name="twitter:image:src">
+  <meta content="A privacy-first, open-source platform for knowledge management and collaboration." name="twitter:image:alt">
+  <meta content="A privacy-first, open-source platform for knowledge management and collaboration." property="og:title">
+  <meta content="site" property="og:type">
+  <meta content="https://logseq.com" property="og:url">
+  <meta content="https://asset.logseq.com/static/img/logo.png" property="og:image">
+  <meta content="A privacy-first, open-source platform for knowledge management and collaboration." property="og:description">
+  <title>Logseq: "A privacy-first platform for knowledge management and collaboration."</title>
+  <meta content="logseq" property="og:site_name">
+  <meta content="A privacy-first, open-source platform for knowledge management and collaboration." name="description">
+</head>
+<body>
+<div id="root"></div>
+<script>window.user = null</script>
+<script>window.__LSP__HOST__ = true</script>
+<script src="/static/js/magic_portal.js"></script>
+<script>let worker = new Worker('/static/js/worker.js')
+  const portal = new MagicPortal(worker);
+  (async () => {
+    const fs = await portal.get('fs')
+    window.fs = fs
+    const pfs = await portal.get('pfs')
+    window.pfs = pfs
+    const workerThread = await portal.get('workerThread')
+    window.workerThread = workerThread
+  })()
+</script>
+<script defer src="/static/js/highlight.min.js"></script>
+<script defer src="/static/js/interact.min.js"></script>
+<script defer src="/static/js/marked.min.js"></script>
+<script defer src="/static/js/eventemitter3.umd.min.js"></script>
+<script defer src="/static/js/html2canvas.min.js"></script>
+<script defer src="/static/js/lsplugin.core.js"></script>
+<script defer src="/static/js/react.production.min.js"></script>
+<script defer src="/static/js/react-dom.production.min.js"></script>
+<script defer src="/static/js/main-bundle.js"></script>
+<script defer src="/static/js/ui.js"></script>
+<script defer src="/static/js/main.js"></script>
+<script defer src="/static/js/amplify.js"></script>
+<script defer src="/static/js/prop-types.min.js"></script>
+<script defer src="/static/js/tabler-icons-react.min.js"></script>
+<script defer src="/static/js/tabler.ext.js"></script>
+<script defer src="/static/js/code-editor.js"></script>
+<script defer src="/static/js/tldraw.js"></script>
+<script defer src="/static/js/excalidraw.js"></script>
+</body>
+</html>

resources/index.html

@@ -58,8 +58,8 @@ const portal = new MagicPortal(worker);
 <script defer src="./js/lsplugin.core.js"></script>
 <script defer src="./js/react.production.min.js"></script>
 <script defer src="./js/react-dom.production.min.js"></script>
+<script defer src="./js/main-bundle.js"></script>
 <script defer src="./js/ui.js"></script>
-<script defer src="./js/shared.js"></script>
 <script defer src="./js/main.js"></script>
 <script defer src="./js/amplify.js"></script>
 <script defer src="./js/prop-types.min.js"></script>

resources/mobile/index.html

@@ -36,7 +36,7 @@ const portal = new MagicPortal(worker);
 <script defer src="./js/ui.js"></script>
 <script defer src="./js/amplify.js"></script>
 <script defer src="./js/silkhq.js"></script>
-<script defer src="./js/shared.js"></script>
+<script defer src="./js/main-bundle.js"></script>
 <script defer src="./js/main.js"></script>
 <script defer src="./js/code-editor.js"></script>
 <script>

scripts/src/logseq/tasks/db_graph/create_graph_with_schema_org.cljs

@@ -195,11 +195,11 @@
                              (map #(vector % (str % "_Class")))
                              (into {}))]
     (when (seq renamed-classes)
-     (if verbose
-       (println "Renaming the following classes because they have names that conflict with Logseq's case insensitive :block/name:"
-                (keys renamed-classes) "\n")
-       (when (pos? (count renamed-classes))
-         (println "Renaming" (count renamed-classes) "classes due to page name conflicts"))))
+      (if verbose
+        (println "Renaming the following classes because they have names that conflict with Logseq's case insensitive :block/name:"
+                 (keys renamed-classes) "\n")
+        (when (pos? (count renamed-classes))
+          (println "Renaming" (count renamed-classes) "classes due to page name conflicts"))))
     renamed-classes))
 
 (defn- get-all-properties [schema-data {:keys [verbose]}]

scripts/src/logseq/tasks/dev.clj

@@ -1,17 +1,17 @@
 (ns logseq.tasks.dev
   "Tasks for general development. For desktop or mobile development see their
   namespaces"
-  (:require [babashka.process :refer [shell]]
+  (:require [babashka.cli :as cli]
             [babashka.fs :as fs]
-            [babashka.cli :as cli]
-            [logseq.tasks.util :as task-util]
-            [logseq.tasks.dev.lint :as dev-lint]
+            [babashka.process :refer [shell]]
+            [clojure.core.async :as async]
+            [clojure.data :as data]
+            [clojure.edn :as edn]
             [clojure.java.io :as io]
             [clojure.pprint :as pp]
-            [clojure.edn :as edn]
-            [clojure.data :as data]
-            [clojure.core.async :as async]
-            [clojure.string :as string]))
+            [clojure.string :as string]
+            [logseq.tasks.dev.lint :as dev-lint]
+            [logseq.tasks.util :as task-util]))
 
 (defn test
   "Run tests. Pass args through to cmd 'yarn cljs:run-test'"
@@ -75,7 +75,7 @@
                                                     (fs/glob "." "{src/main,deps/graph-parser/src}/**")))))]
     (do
       (println "Building publishing js asset...")
-      (shell "clojure -M:cljs release publishing"))
+      (shell "clojure -M:cljs release publishing workers"))
     (println "Publishing js asset is up to date")))
 
 (defn publishing-backend
@@ -87,7 +87,7 @@
 
 (defn watch-publishing-frontend
   [& _args]
-  (shell "clojure -M:cljs watch publishing"))
+  (shell "npx shadow-cljs watch publishing"))
 
 (defn watch-publishing-backend
   "Builds publishing backend once watch-publishing-frontend has built initial frontend"
@@ -110,4 +110,4 @@
       (let [db-graph (fs/path parent-graph-dir (fs/file-name file-graph))]
         (println "Importing" (str db-graph) "...")
         (apply shell "bb" "dev:db-import" file-graph db-graph import-options)
-        (shell "bb" "dev:validate-db" db-graph "-gHc")))))
+        (shell "bb" "dev:validate-db" db-graph "-gHc")))))

scripts/src/logseq/tasks/dev/db_and_file_graphs.clj

@@ -18,7 +18,8 @@
          "electron.db"
          "frontend.handler.db-based."
          "frontend.worker.handler.page.db-based"
-         "frontend.components.property" "frontend.components.class" "frontend.components.quick-add"
+         "frontend.inference-worker"
+         "frontend.components.property" "frontend.components.class" "frontend.components.quick-add" "frontend.components.vector-search"
          "frontend.components.db-based" "frontend.components.objects" "frontend.components.query.view"
          "mobile.core" "mobile.events" "mobile.externals" "mobile.init" "mobile.state"
          "mobile.components"]))
@@ -58,8 +59,10 @@
          "src/main/frontend/components/property"
          "src/main/frontend/components/objects.cljs"
          "src/main/frontend/components/quick_add.cljs"
+         "src/main/frontend/components/vector_search"
          "src/main/frontend/components/db_based"
          "src/main/frontend/components/query/view.cljs"
+         "src/main/frontend/inference_worker"
          "src/electron/electron/db.cljs"
          "src/main/mobile"]))
 

scripts/src/logseq/tasks/dev/lint.clj

@@ -43,13 +43,14 @@
           (when (pos? (:exit res)) (System/exit (:exit res)))))
       (println "No clj* files have changed to lint."))))
 
-(defn- validate-frontend-not-in-worker
+(defn- validate-frontend-not-in-workers
   []
   (let [res (shell {:out :string}
-                   "git grep -h" "\\[frontend.*:as" "src/main/frontend/worker")
+                   "git grep -h" "\\[frontend.*:as"
+                   "src/main/frontend/worker" "src/main/frontend/worker_common" "src/main/frontend/inference_worker")
         req-lines (->> (:out res)
                        string/split-lines
-                       (remove #(re-find #"frontend\.worker|frontend\.common" %)))]
+                       (remove #(re-find #"frontend\.worker|frontend\.common|frontend\.inference-worker" %)))]
 
     (if (seq req-lines)
       (do
@@ -58,10 +59,10 @@
         (System/exit 1))
       (println "Valid worker namespaces!"))))
 
-(defn- validate-worker-not-in-frontend
+(defn- validate-workers-not-in-frontend
   []
   (let [res (shell {:out :string :continue true}
-                   "grep -r --exclude-dir=worker" "\\[frontend.worker.*:" "src/main/frontend")
+                   "grep -r --exclude-dir=worker --exclude-dir=inference_worker" "\\[frontend.worker.*:" "src/main/frontend")
         ;; allow reset-file b/c it's only affects tests
         allowed-exceptions #{"src/main/frontend/handler/file_based/file.cljs:            [frontend.worker.file.reset :as file-reset]"}
         invalid-lines (when (= 0 (:exit res))
@@ -75,8 +76,7 @@
       (println "Valid frontend namespaces!"))))
 
 (defn worker-and-frontend-separate
-  "Ensures worker is independent of frontend"
+  "Ensures workers are independent of frontend"
   []
-  (validate-frontend-not-in-worker)
-  (validate-worker-not-in-frontend))
-
+  (validate-frontend-not-in-workers)
+  (validate-workers-not-in-frontend))

shadow-cljs.edn

@@ -14,16 +14,18 @@
  :builds
  {:app {:target        :browser
         :module-loader true
-        :js-options    {:ignore-asset-requires true
+        :js-options    {:js-provider    :external
+                        :external-index "target/main.js"
+                        :external-index-format :esm
+                        :entry-keys ["module" "browser" "main"]
+                        :export-conditions ["module" "import", "browser" "require" "default"]
+                        :ignore-asset-requires true
                         :resolve {"react" {:target :global
                                            :global "React"}
                                   "react-dom" {:target :global
-                                               :global "ReactDOM"}}} ;; handle `require(xxx.css)`
-        :modules       {:shared
-                        {:entries []}
-                        :main
-                        {:init-fn frontend.core/init
-                         :depends-on #{:shared}}
+                                               :global "ReactDOM"}}}
+        :modules       {:main
+                        {:init-fn frontend.core/init}
                         :code-editor
                         {:entries [frontend.extensions.code]
                          :depends-on #{:main}}
@@ -32,11 +34,7 @@
                          :depends-on #{:main}}
                         :tldraw
                         {:entries [frontend.extensions.tldraw]
-                         :depends-on #{:main}}
-                        :db-worker
-                        {:init-fn frontend.worker.db-worker/init
-                         :depends-on #{:shared}
-                         :web-worker true}}
+                         :depends-on #{:main}}}
 
         :output-dir       "./static/js"
         :asset-path       "/static/js"
@@ -46,8 +44,7 @@
                            :externs ["datascript/externs.js"
                                      "externs.js"]
                            :warnings {:fn-deprecated false
-                                      :redef false}
-                           :cross-chunk-method-motion false}
+                                      :redef false}}
         :build-hooks [(shadow.hooks/git-revision-hook "--long --always --dirty")]
         :closure-defines {lambdaisland.glogi.console/colorize "false"
                           goog.debug.LOGGING_ENABLED true
@@ -58,32 +55,68 @@
                           frontend.config/ENABLE-FILE-SYNC-PRODUCTION #shadow/env ["ENABLE_FILE_SYNC_PRODUCTION" :as :bool :default true]
                           frontend.config/ENABLE-RTC-SYNC-PRODUCTION #shadow/env ["ENABLE_RTC_SYNC_PRODUCTION" :as :bool :default true]
                           frontend.config/REVISION #shadow/env ["LOGSEQ_REVISION" :default "dev"]} ;; set by git-revision-hook
-
+        
         :devtools {:before-load frontend.core/stop          ;; before live-reloading any code call this function
                    :after-load frontend.core/start          ;; after live-reloading finishes call this function
                    :watch-path "/static"
                    :preloads [devtools.preload
-                              shadow.remote.runtime.cljs.browser]}}
-
+                              shadow.remote.runtime.cljs.browser]
+                   ;; :ignore-warnings true
+                   }}
+
+  :workers {:target        :browser
+            :module-loader true
+            :js-options    {:js-provider    :external
+                            :external-index "target/workers.js"
+                            :external-index-format :esm
+                            :entry-keys ["module" "browser" "main"]
+                            :export-conditions ["module" "import", "browser" "require" "default"]}
+            :modules       {:workers-shared {:entries []}
+                            :db-worker
+                            {:init-fn frontend.worker.db-worker/init
+                             :web-worker true
+                             :prepend "importScripts('workers-bundle.js');\n"
+                             :depends-on #{:workers-shared}}
+                            :inference-worker
+                            {:init-fn frontend.inference-worker.inference-worker/init
+                             :web-worker true
+                             :prepend "importScripts('workers-bundle.js');\n"
+                             :depends-on #{:workers-shared}}}
+
+            :output-dir       "./static/js"
+            :asset-path       "/static/js"
+            :release          {:asset-path "https://asset.logseq.com/static/js"}
+            :compiler-options {:infer-externs      :auto
+                               :output-feature-set :es-next-in
+                               :source-map true
+                               :externs ["datascript/externs.js"
+                                         "externs.js"]
+                               :warnings {:fn-deprecated false
+                                          :redef false}}
+            :closure-defines {goog.debug.LOGGING_ENABLED true}
+            :devtools {:before-load frontend.core/stop          ;; before live-reloading any code call this function
+                       :after-load frontend.core/start          ;; after live-reloading finishes call this function
+                       :watch-path "/static"
+                       :preloads [devtools.preload
+                                  shadow.remote.runtime.cljs.browser]}}
   :mobile {:target        :browser
            :module-loader true
-           :js-options    {:ignore-asset-requires true
+           :js-options    {:js-provider    :external
+                           :external-index "target/mobile.js"
+                           :external-index-format :esm
+                           :entry-keys ["module" "browser" "main"]
+                           :export-conditions ["module" "import", "browser" "require" "default"]
+                           :ignore-asset-requires true
                            :resolve {"react" {:target :global
                                               :global "React"}
                                      "react-dom" {:target :global
-                                                  :global "ReactDOM"}}} ;; handle `require(xxx.css)`
-           :modules       {:shared
-                           {:entries []}
-                           :main
-                           {:init-fn mobile.core/init
-                            :depends-on #{:shared}}
+                                                  :global "ReactDOM"}}}
+           ;; handle `require(xxx.css)`
+           :modules       {:main
+                           {:init-fn mobile.core/init}
                            :code-editor
                            {:entries [frontend.extensions.code]
-                            :depends-on #{:main}}
-                           :db-worker
-                           {:init-fn frontend.worker.db-worker/init
-                            :depends-on #{:shared}
-                            :web-worker true}}
+                            :depends-on #{:main}}}
 
            :output-dir       "./static/mobile/js"
            :asset-path       "./js"
@@ -94,24 +127,22 @@
                               :externs ["datascript/externs.js"
                                         "externs.js"]
                               :warnings {:fn-deprecated false
-                                         :redef false}
-                              :cross-chunk-method-motion false}
+                                         :redef false}}
            :build-hooks [(shadow.hooks/git-revision-hook "--long --always --dirty")]
            :closure-defines
            {;; Set to switch file sync server to dev, set this to false in `yarn watch`
             frontend.config/ENABLE-FILE-SYNC-PRODUCTION #shadow/env ["ENABLE_FILE_SYNC_PRODUCTION" :as :bool :default true]
             frontend.config/ENABLE-RTC-SYNC-PRODUCTION #shadow/env ["ENABLE_RTC_SYNC_PRODUCTION" :as :bool :default true]}
 
-;; NOTE: electron, browser/mobile-app use different asset-paths.
-                  ;;   For browser/mobile-app devs, assets are located in /static/js(via HTTP root).
-                  ;;   For electron devs, assets are located in ./js(via relative path).
-                  ;; :dev      {:asset-path "./js"}
+           ;; NOTE: electron, browser/mobile-app use different asset-paths.
+           ;;   For browser/mobile-app devs, assets are located in /static/js(via HTTP root).
+           ;;   For electron devs, assets are located in ./js(via relative path).
+           ;; :dev      {:asset-path "./js"}
            :devtools {:before-load mobile.core/stop!          ;; before live-reloading any code call this function
                       :after-load mobile.core/render!          ;; after live-reloading finishes call this function
                       :watch-path "/static/mobile"
                       :preloads [devtools.preload
                                  shadow.remote.runtime.cljs.browser]}}
-
   :electron {:target :node-script
              :output-to "static/electron.js"
              :main electron.core/main
@@ -153,17 +184,18 @@
 
   :publishing {:target :browser
                :module-loader true
-               :js-options {;; handle `require(xxx.css)`
+               :js-options {:js-provider    :external
+                            :external-index "target/main.js"
+                            :external-index-format :esm
+                            :entry-keys ["module" "browser" "main"]
+                            :export-conditions ["module" "import", "browser" "require" "default"]
                             :ignore-asset-requires true
                             :resolve {"react" {:target :global
                                                :global "React"}
                                       "react-dom" {:target :global
                                                    :global "ReactDOM"}}}
-               :modules {:shared
-                         {:entries []}
-                         :main
-                         {:init-fn frontend.publishing/init
-                          :depends-on #{:shared}}
+               :modules {:main
+                         {:init-fn frontend.publishing/init}
                          :code-editor
                          {:entries [frontend.extensions.code]
                           :depends-on #{:main}}
@@ -172,26 +204,19 @@
                           :depends-on #{:main}}
                          :tldraw
                          {:entries [frontend.extensions.tldraw]
-                          :depends-on #{:main}}
-                         :db-worker
-                         {:init-fn frontend.worker.db-worker/init
-                          :depends-on #{:shared}
-                          :web-worker true}}
+                          :depends-on #{:main}}}
 
                :output-dir "./static/js/publishing"
                :asset-path "static/js"
                :closure-defines {logseq.common.config/PUBLISHING true
                                  goog.debug.LOGGING_ENABLED true}
                :compiler-options {:infer-externs :auto
-                                  :output-feature-set :es-next
+                                  :output-feature-set :es-next-in
                                   :source-map true
                                   :externs ["datascript/externs.js"
                                             "externs.js"]
                                   :warnings {:fn-deprecated false
-                                             :redef false}
-                                  ;; https://github.com/thheller/shadow-cljs/issues/611#issuecomment-620845276
-                                  ;; fixes cljs.spec bug with code splitting
-                                  :cross-chunk-method-motion false}
+                                             :redef false} }
                :devtools {:before-load frontend.core/stop
                           :after-load frontend.core/start
                           :preloads [devtools.preload]}}

src/dev-cljs/shadow/hooks.clj

@@ -12,24 +12,6 @@
         (string/trim out))
       (println err))))
 
-(defn purge-css
-  {:shadow.build/stage :flush}
-  [state {:keys [css-source
-                 js-globs
-                 public-dir]}]
-  (case (:shadow.build/mode state)
-    :release
-    (exec "purgecss --css " css-source
-          (for [content (if (string? js-globs) [js-globs] js-globs)]
-            (str "--content " content))
-          "-o" public-dir)
-
-    :dev
-    (do
-      (exec "mkdir -p" public-dir)
-      (exec "cp" css-source (str public-dir "/" (last (string/split css-source #"/"))))))
-  state)
-
 (defn git-revision-hook
   {:shadow.build/stage :configure}
   [build-state & args]

src/dev-cljs/shadow/user.clj

@@ -19,7 +19,7 @@
 ;; Get the runtime id from http://localhost:9630/runtimes, pick the one which shows `browser-worker`
 (defn worker-repl
   ([]
-   (when-let [runtime-id (->> (api/repl-runtimes :app)
+   (when-let [runtime-id (->> (api/repl-runtimes :workers)
                               (filter (fn [runtime] (= :browser-worker (:host runtime))))
                               (map :client-id)
                               (apply max))]
@@ -29,8 +29,8 @@
    (if
     (number? runtime-id-or-which)
      (do (prn :worker-runtime-id runtime-id-or-which)
-         (api/repl :app {:runtime-id runtime-id-or-which}))
-     (let [runtime-ids (->> (api/repl-runtimes :app)
+         (api/repl :workers {:runtime-id runtime-id-or-which}))
+     (let [runtime-ids (->> (api/repl-runtimes :workers)
                             (filter (fn [runtime] (= :browser-worker (:host runtime))))
                             (map :client-id))
            runtime-id (apply (if (= :old runtime-id-or-which) min max) runtime-ids)]

src/electron/electron/fs_watcher.cljs

@@ -2,12 +2,12 @@
   "This ns is a wrapper around the chokidar file watcher,
   https://www.npmjs.com/package/chokidar. File watcher events are sent to the
   `file-watcher` ipc channel"
-  (:require [cljs-bean.core :as bean]
+  (:require ["chokidar" :as watcher]
+            ["electron" :refer [app]]
             ["fs" :as fs]
-            ["chokidar" :as watcher]
-            [electron.utils :as utils]
+            [cljs-bean.core :as bean]
             [electron.logger :as logger]
-            ["electron" :refer [app]]
+            [electron.utils :as utils]
             [electron.window :as window]
             [logseq.common.graph :as common-graph]))
 

src/electron/electron/utils.cljs

@@ -3,12 +3,12 @@
             ["electron" :refer [app BrowserWindow]]
             ["fs-extra" :as fs]
             ["path" :as node-path]
+            [cljs-bean.core :as bean]
             [clojure.string :as string]
             [electron.configs :as cfgs]
+            [electron.db :as db]
             [electron.logger :as logger]
             [logseq.db.sqlite.util :as sqlite-util]
-            [cljs-bean.core :as bean]
-            [electron.db :as db]
             [promesa.core :as p]))
 
 (defonce *win (atom nil)) ;; The main window

src/electron/electron/window.cljs

@@ -1,15 +1,15 @@
 (ns electron.window
-  (:require ["electron-window-state" :as windowStateKeeper]
-            [electron.utils :refer [mac? win32? linux? dev? open] :as utils]
+  (:require ["electron" :refer [BrowserWindow app session shell dialog] :as electron]
+            ["electron-window-state" :as windowStateKeeper]
+            ["path" :as node-path]
+            ["url" :as URL]
+            [cljs-bean.core :as bean]
+            [clojure.string :as string]
             [electron.configs :as cfgs]
             [electron.context-menu :as context-menu]
             [electron.logger :as logger]
-            ["electron" :refer [BrowserWindow app session shell dialog] :as electron]
-            ["path" :as node-path]
-            ["url" :as URL]
             [electron.state :as state]
-            [cljs-bean.core :as bean]
-            [clojure.string :as string]))
+            [electron.utils :refer [mac? win32? linux? dev? open] :as utils]))
 
 (defonce *quitting? (atom false))
 
@@ -28,27 +28,27 @@
          native-titlebar? (cfgs/get-item :window/native-titlebar?)
          url (if graph (str url "#/?graph=" graph) url)
          win-opts  (cond->
-                     {:backgroundColor      "#fff" ; SEE https://www.electronjs.org/docs/latest/faq#the-font-looks-blurry-what-is-this-and-what-can-i-do
-                      :width                (.-width win-state)
-                      :height               (.-height win-state)
-                      :frame                (or mac? native-titlebar?)
-                      :titleBarStyle        "hiddenInset"
-                      :trafficLightPosition {:x 16 :y 16}
-                      :autoHideMenuBar      (not mac?)
-                      :show                 false
-                      :webPreferences
-                      {:plugins                 true        ; pdf
-                       :nodeIntegration         false
-                       :nodeIntegrationInWorker false
-                       :nativeWindowOpen        true
-                       :sandbox                 false
-                       :webSecurity             (not dev?)
-                       :contextIsolation        true
-                       :spellcheck              ((fnil identity true) (cfgs/get-item :spell-check))
+                    {:backgroundColor      "#fff" ; SEE https://www.electronjs.org/docs/latest/faq#the-font-looks-blurry-what-is-this-and-what-can-i-do
+                     :width                (.-width win-state)
+                     :height               (.-height win-state)
+                     :frame                (or mac? native-titlebar?)
+                     :titleBarStyle        "hiddenInset"
+                     :trafficLightPosition {:x 16 :y 16}
+                     :autoHideMenuBar      (not mac?)
+                     :show                 false
+                     :webPreferences
+                     {:plugins                 true        ; pdf
+                      :nodeIntegration         false
+                      :nodeIntegrationInWorker false
+                      :nativeWindowOpen        true
+                      :sandbox                 false
+                      :webSecurity             (not dev?)
+                      :contextIsolation        true
+                      :spellcheck              ((fnil identity true) (cfgs/get-item :spell-check))
                        ;; Remove OverlayScrollbars and transition `.scrollbar-spacing`
                        ;; to use `scollbar-gutter` after the feature is implemented in browsers.
-                       :enableBlinkFeatures     'OverlayScrollbars'
-                       :preload                 (node-path/join js/__dirname "js/preload.js")}}
+                      :enableBlinkFeatures     'OverlayScrollbars'
+                      :preload                 (node-path/join js/__dirname "js/preload.js")}}
 
                      (seq opts)
                      (merge opts)
@@ -129,12 +129,12 @@
       (if (contains? #{"https:" "http:" "mailto:"} (.-protocol parsed-url))
         (.openExternal shell url)
         (when-let [^js res (and (fn? default-open)
-                             (.showMessageBoxSync dialog
-                               #js {:type "warning"
-                                    :message (str "Are you sure you want to open this link? \n\n" url)
-                                    :defaultId 1
-                                    :cancelId 0
-                                    :buttons #js ["Cancel" "OK"]}))]
+                                (.showMessageBoxSync dialog
+                                                     #js {:type "warning"
+                                                          :message (str "Are you sure you want to open this link? \n\n" url)
+                                                          :defaultId 1
+                                                          :cancelId 0
+                                                          :buttons #js ["Cancel" "OK"]}))]
           (when (= res 1)
             (default-open url)))))))
 
@@ -187,7 +187,7 @@
                               :webSecurity (not dev?)
                               :preload (node-path/join js/__dirname "js/preload.js")
                               :nativeWindowOpen true}}}
-                      features)
+                           features)
                     (do (open-external! url) {:action "deny"}))
                   (bean/->js))))]
 

src/main/frontend/common/file/util.cljs

@@ -76,13 +76,13 @@
 ;; A fast pprint alternative.
 (defn print-prefix-map* [prefix m print-one writer opts]
   (pr-sequential-writer
-    writer
-    (fn [e w opts]
-      (print-one (key e) w opts)
-      (-write w \space)
-      (print-one (val e) w opts))
-    (str prefix "\n{") \newline "}"
-    opts (seq m)))
+   writer
+   (fn [e w opts]
+     (print-one (key e) w opts)
+     (-write w \space)
+     (print-one (val e) w opts))
+   (str prefix "\n{") \newline "}"
+   opts (seq m)))
 
 (defn ugly-pr-str
   "Ugly printing fast, with newlines so that git diffs are smaller"
@@ -91,6 +91,6 @@
     (pr-str x)))
 
 (defn post-message
-  [type data]
-  (when (exists? js/self)
-    (.postMessage js/self (ldb/write-transit-str [type data]))))
+  [type data & {:keys [port]}]
+  (when-let [worker (or port js/self)]
+    (.postMessage worker (ldb/write-transit-str [type data]))))

src/main/frontend/common/search_fuzzy.cljs

@@ -1,8 +1,8 @@
 (ns frontend.common.search-fuzzy
   "fuzzy search. Used by frontend and worker namespaces"
-  (:require [clojure.string :as string]
+  (:require ["remove-accents" :as removeAccents]
             [cljs-bean.core :as bean]
-            ["remove-accents" :as removeAccents]))
+            [clojure.string :as string]))
 
 (def MAX-STRING-LENGTH 1000.0)
 
@@ -28,10 +28,19 @@
                (/ (- maxed mined)
                   maxed)))))
 
+(defn search-normalize
+  "Normalize string for searching (loose)"
+  [s remove-accents?]
+  (when s
+    (let [normalize-str (.normalize (string/lower-case s) "NFKC")]
+      (if remove-accents?
+        (removeAccents normalize-str)
+        normalize-str))))
+
 (defn score
   [oquery ostr]
-  (let [query (clean-str oquery)
-        original-s (clean-str ostr)]
+  (let [query (-> (clean-str oquery) (search-normalize true))
+        original-s (-> (clean-str ostr) (search-normalize true))]
     (loop [q (seq (char-array query))
            s (seq (char-array original-s))
            mult 1
@@ -68,24 +77,14 @@
                        (dec idx)
                        (- score' 0.1)))))))
 
-(defn search-normalize
-  "Normalize string for searching (loose)"
-  [s remove-accents?]
-  (when s
-    (let [normalize-str (.normalize (string/lower-case s) "NFKC")]
-      (if remove-accents?
-        (removeAccents normalize-str)
-        normalize-str))))
-
 (defn fuzzy-search
   [data query & {:keys [limit extract-fn]
                  :or {limit 20}}]
-  (let [query (search-normalize query true)]
-    (->> (take limit
-               (sort-by :score (comp - compare)
-                        (filter #(< 0 (:score %))
-                                (for [item data]
-                                  (let [s (str (if extract-fn (extract-fn item) item))]
-                                    {:data item
-                                     :score (score query (search-normalize s true))})))))
-         (map :data))))
+  (->> (take limit
+             (sort-by :score (comp - compare)
+                      (filter #(< 0 (:score %))
+                              (for [item data]
+                                (let [s (str (if extract-fn (extract-fn item) item))]
+                                  {:data item
+                                   :score (score query s)})))))
+       (map :data)))

src/main/frontend/common/thread_api.cljc

@@ -29,8 +29,10 @@
      (let [qkw (keyword qualified-kw-str)]
        (vswap! *profile update qkw inc)
        (if-let [f (@*thread-apis qkw)]
-         (let [result (apply f (cond-> args-transit-str-or-args-array
-                                 (not direct-pass?) ldb/read-transit-str))
+         (let [result (if (= qkw :thread-api/set-infer-worker-proxy)
+                        (f args-transit-str-or-args-array)
+                        (apply f (cond-> args-transit-str-or-args-array
+                                   (not direct-pass?) ldb/read-transit-str)))
                result-promise
                (if (fn? result) ;; missionary task is a fn
                  (js/Promise. result)

src/main/frontend/components/block/macros.cljs

@@ -1,15 +1,15 @@
 (ns frontend.components.block.macros
   "Logseq macros that render and evaluate in blocks"
   (:require [clojure.walk :as walk]
+            [datascript.core :as d]
+            [frontend.config :as config]
+            [frontend.db.conn :as db-conn]
             [frontend.extensions.sci :as sci]
             [frontend.handler.common :as common-handler]
+            [frontend.state :as state]
             [goog.string :as gstring]
             [goog.string.format]
-            [frontend.state :as state]
-            [frontend.config :as config]
-            [datascript.core :as d]
-            [logseq.db.frontend.property :as db-property]
-            [frontend.db.conn :as db-conn]))
+            [logseq.db.frontend.property :as db-property]))
 
 (defn- properties-by-name
   "Given a block from a query result, returns a map of its properties indexed by

src/main/frontend/components/cmdk/core.cljs

@@ -40,6 +40,10 @@
             [promesa.core :as p]
             [rum.core :as rum]))
 
+(defn- get-action
+  []
+  (:action (:search/args @state/state)))
+
 (defn translate [t {:keys [id desc]}]
   (when id
     (let [desc-i18n (t (shortcut-utils/decorate-namespace id))]
@@ -71,7 +75,7 @@
    :commands       {:status :success :show :less :items nil}
    :favorites      {:status :success :show :less :items nil}
    :current-page   {:status :success :show :less :items nil}
-   :nodes         {:status :success :show :less :items nil}
+   :nodes          {:status :success :show :less :items nil}
    :files          {:status :success :show :less :items nil}
    :themes         {:status :success :show :less :items nil}
    :filters        {:status :success :show :less :items nil}})
@@ -197,8 +201,10 @@
                (first))))
 
 (defn state->action [state]
-  (let [highlighted-item (state->highlighted-item state)]
-    (cond (:source-page highlighted-item) :open
+  (let [highlighted-item (state->highlighted-item state)
+        action (get-action)]
+    (cond (and (:source-page highlighted-item) (= action :move-blocks)) :trigger
+          (:source-page highlighted-item) :open
           (:source-block highlighted-item) :open
           (:file-path highlighted-item) :open
           (:source-search highlighted-item) :search
@@ -327,7 +333,9 @@
         repo (state/get-current-repo)
         current-page (when-let [id (page-util/get-current-page-id)]
                        (db/entity id))
-        opts {:limit 100 :dev? config/dev? :built-in? true}]
+        opts (cond-> {:limit 100 :dev? config/dev? :built-in? true}
+               (contains? #{:move-blocks} (get-action))
+               (assoc :page-only? true))]
     (swap! !results assoc-in [group :status] :loading)
     (swap! !results assoc-in [:current-page :status] :loading)
     (p/let [blocks (search/block-search repo @!input opts)
@@ -548,12 +556,21 @@
       (reset! (::input state) search-query))))
 
 (defmethod handle-action :trigger [_ state _event]
-  (let [command (some-> state state->highlighted-item :source-command)
-        dont-close-commands #{:graph/open :graph/remove :dev/replace-graph-with-db-file :misc/import-edn-data}]
-    (when-let [action (:action command)]
+  (let [highlighted-item (some-> state state->highlighted-item)
+        command (:source-command highlighted-item)
+        dont-close-commands #{:graph/open :graph/remove :dev/replace-graph-with-db-file :misc/import-edn-data :editor/move-blocks}
+        search-args (:search/args @state/state)
+        action (or (:action command)
+                   (when-let [trigger (:trigger search-args)]
+                     #(trigger highlighted-item)))
+        input-ref @(::input-ref state)]
+    (when action
+      (when input-ref
+        (set! (.-value input-ref) "")
+        (.focus input-ref))
+      (action)
       (when-not (contains? dont-close-commands (:id command))
-        (shui/dialog-close! :ls-dialog-cmdk))
-      (util/schedule #(action) 32))))
+        (shui/dialog-close! :ls-dialog-cmdk)))))
 
 (defmethod handle-action :create [_ state _event]
   (let [item (state->highlighted-item state)
@@ -630,7 +647,6 @@
   [state' state title group visible-items first-item sidebar?]
   (let [{:keys [show items]} (some-> state ::results deref group)
         highlighted-item (or @(::highlighted-item state) first-item)
-        highlighted-group @(::highlighted-group state)
         *mouse-active? (::mouse-active? state')
         filter' @(::filter state)
         can-show-less? (< (get-group-limit group) (count visible-items))
@@ -660,8 +676,7 @@
 
          [:div {:class "flex-1"}]
 
-         (when (and (= group highlighted-group)
-                    (or can-show-more? can-show-less?)
+         (when (and (or can-show-more? can-show-less?)
                     (empty? filter')
                     (not sidebar?))
            [:a.text-link.select-node.opacity-50.hover:opacity-90
@@ -694,11 +709,13 @@
                              ;; for some reason, the highlight effect does not always trigger on a
                              ;; boolean value change so manually pass in the dep
                             :on-highlight-dep highlighted-item
-                            :on-click (fn [e]
-                                        (reset! (::highlighted-item state) item)
-                                        (handle-action :default state item)
-                                        (when-let [on-click (:on-click item)]
-                                          (on-click e)))
+                            :on-click
+                            (fn [e]
+                              (util/stop-propagation e)
+                              (reset! (::highlighted-item state) item)
+                              (handle-action :default state item)
+                              (when-let [on-click (:on-click item)]
+                                (on-click e)))
                              ;; :on-mouse-enter (fn [e]
                              ;;                   (when (not highlighted?)
                              ;;                     (reset! (::highlighted-item state) (assoc item :mouse-enter-triggered-highlight true))))
@@ -807,12 +824,20 @@
       (and enter? (not composing?)) (do
                                       (handle-action :default state e)
                                       (util/stop-propagation e))
-      esc? (let [filter' @(::filter state)]
-             (if filter'
+      esc? (let [filter' @(::filter state)
+                 action (get-action)
+                 move-blocks? (= :move-blocks action)]
+             (cond
+               (and move-blocks? (string/blank? input))
+               (state/close-modal!)
+
+               (and filter' (not move-blocks?))
                (do
                  (util/stop e)
                  (reset! (::filter state) nil)
                  (load-results :default state))
+
+               :else
                (when-not (string/blank? input)
                  (util/stop e)
                  (handle-input-change state nil ""))))
@@ -833,12 +858,15 @@
 (defn- input-placeholder
   [sidebar?]
   (let [search-mode (:search/mode @state/state)
-        search-args (:search/args @state/state)]
+        action (get-action)]
     (cond
+      (= action :move-blocks)
+      "Move blocks to"
+
       (and (= search-mode :graph) (not sidebar?))
       "Add graph filter"
 
-      (= search-args :new-page)
+      (= action :new-page)
       "Type a page name to create"
 
       :else
@@ -1002,8 +1030,10 @@
        (shortcut/listen-all!))
      state)}
   {:init (fn [state]
-           (let [search-mode (:search/mode @state/state)
+           (let [search-mode (or (:search/mode @state/state) :global)
                  opts (last (:rum/args state))]
+             (when (nil? search-mode)
+               (state/set-state! :search/mode :global))
              (assoc state
                     ::ref (atom nil)
                     ::filter (if (and search-mode
@@ -1034,8 +1064,10 @@
   (rum/local false ::input-changed?)
   [state {:keys [sidebar?] :as opts}]
   (let [*input (::input state)
-        search-mode (:search/mode @state/state)
-        group-filter (:group (rum/react (::filter state)))
+        search-mode (state/sub :search/mode)
+        group-filter (or (when (and (not (contains? #{:global :graph} search-mode)) (not (:sidebar? opts)))
+                           search-mode)
+                         (:group (rum/react (::filter state))))
         results-ordered (state->results-ordered state search-mode)
         all-items (mapcat last results-ordered)
         first-item (first all-items)]
@@ -1074,7 +1106,8 @@
      (when-not sidebar? (hints state))]))
 
 (rum/defc cmdk-modal [props]
-  [:div {:class "cp__cmdk__modal rounded-lg w-[90dvw] max-w-4xl relative"}
+  [:div {:class "cp__cmdk__modal rounded-lg w-[90dvw] max-w-4xl relative"
+         :data-keep-selection true}
    (cmdk props)])
 
 (rum/defc cmdk-block [props]

src/main/frontend/components/container.cljs

@@ -1019,7 +1019,8 @@
        :on-pointer-up (fn []
                         (when-let [container (gdom/getElement "app-container-wrapper")]
                           (d/remove-class! container "blocks-selection-mode")
-                          (when (> (count (state/get-selection-blocks)) 1)
+                          (when (and (> (count (state/get-selection-blocks)) 1)
+                                     (not (util/input? js/document.activeElement)))
                             (util/clear-selection!))))}
 
       [:button#skip-to-main

src/main/frontend/components/content.cljs

@@ -19,10 +19,10 @@
             [frontend.handler.property :as property-handler]
             [frontend.handler.property.util :as pu]
             [frontend.modules.shortcut.core :as shortcut]
-            [frontend.util.ref :as ref]
             [frontend.state :as state]
             [frontend.ui :as ui]
             [frontend.util :as util]
+            [frontend.util.ref :as ref]
             [frontend.util.url :as url-util]
             [goog.dom :as gdom]
             [goog.object :as gobj]

src/main/frontend/components/file.cljs

@@ -69,8 +69,7 @@
   [:div.flex-1.overflow-hidden
    [:h1.title
     (t :all-files)]
-   (files-all)
-   ])
+   (files-all)])
 
 ;; FIXME: misuse of rpath and fpath
 (rum/defcs file-inner < rum/reactive

src/main/frontend/components/file_based/block.cljs

@@ -82,8 +82,8 @@
   [{:block/keys [pre-block? priority] :as block}]
   (when (and (not pre-block?) priority)
     (ui/tooltip
-      (priority-text priority)
-      (set-priority block priority))))
+     (priority-text priority)
+     (set-priority block priority))))
 
 (defn clock-summary-cp
   [block body]
@@ -96,19 +96,19 @@
                  (not (string/blank? summary)))
         [:div {:style {:max-width 100}}
          (ui/tooltip
-           [:div.text-sm.time-spent.ml-1 {:style {:padding-top 3}}
-            [:a.fade-link
-             summary]]
+          [:div.text-sm.time-spent.ml-1 {:style {:padding-top 3}}
+           [:a.fade-link
+            summary]]
 
-           (when-let [logbook (drawer/get-logbook body)]
-             (let [clocks (->> (last logbook)
-                            (filter #(string/starts-with? % "CLOCK:"))
-                            (remove string/blank?))]
-               [:div.p-4
-                [:div.font-bold.mb-2 "LOGBOOK:"]
-                [:ul
-                 (for [clock (take 10 (reverse clocks))]
-                   [:li clock])]])))]))))
+          (when-let [logbook (drawer/get-logbook body)]
+            (let [clocks (->> (last logbook)
+                              (filter #(string/starts-with? % "CLOCK:"))
+                              (remove string/blank?))]
+              [:div.p-4
+               [:div.font-bold.mb-2 "LOGBOOK:"]
+               [:ul
+                (for [clock (take 10 (reverse clocks))]
+                  [:li clock])]])))]))))
 
 (rum/defc timestamp-editor
   [ast *show-datapicker?]

src/main/frontend/components/file_based/query.cljs

@@ -11,16 +11,16 @@
 (rum/defc query-refresh-button
   [query-time {:keys [on-pointer-down full-text-search?]}]
   (ui/tooltip
-    [:a.fade-link.flex
-     {:on-pointer-down on-pointer-down}
-     (ui/icon "refresh" {:style {:font-size 20}})]
-    [:div
-     [:p
-      (if full-text-search?
-        [:span "Full-text search results will not be refreshed automatically."]
-        [:span (str "This query takes " (int query-time) "ms to finish, it's a bit slow so that auto refresh is disabled.")])]
-     [:p
-      "Click the refresh button instead if you want to see the latest result."]]))
+   [:a.fade-link.flex
+    {:on-pointer-down on-pointer-down}
+    (ui/icon "refresh" {:style {:font-size 20}})]
+   [:div
+    [:p
+     (if full-text-search?
+       [:span "Full-text search results will not be refreshed automatically."]
+       [:span (str "This query takes " (int query-time) "ms to finish, it's a bit slow so that auto refresh is disabled.")])]
+    [:p
+     "Click the refresh button instead if you want to see the latest result."]]))
 
 ;; Custom query header only used by file graphs
 (rum/defc custom-query-header

src/main/frontend/components/header.cljs

@@ -20,6 +20,7 @@
             [frontend.db :as db]
             [frontend.handler :as handler]
             [frontend.handler.db-based.rtc-flows :as rtc-flows]
+            [frontend.handler.db-based.vector-search-flows :as vector-search-flows]
             [frontend.handler.page :as page-handler]
             [frontend.handler.plugin :as plugin-handler]
             [frontend.handler.route :as route-handler]
@@ -342,6 +343,30 @@
                                     (:block/uuid page)
                                     {:header? true})]])))
 
+(rum/defc semantic-search-progressing
+  [repo]
+  (let [[vec-search-state set-vec-search-state] (hooks/use-state nil)
+        {:keys [indexing?]} (get-in vec-search-state [:repo->index-info repo])]
+    (hooks/use-effect!
+     (fn []
+       (c.m/run-task
+         ::update-vec-search-state
+         (m/reduce
+          (fn [_ v]
+            (set-vec-search-state v))
+          (m/ap
+            (m/?> vector-search-flows/infer-worker-ready-flow)
+            (c.m/<? (state/<invoke-db-worker :thread-api/vec-search-update-index-info repo))
+            (m/?> vector-search-flows/vector-search-state-flow)))
+         :succ (constantly nil)))
+     [])
+    (when indexing?
+      (shui/button
+       {:class   "opacity-50"
+        :variant :ghost
+        :size    :sm}
+       "Embedding..."))))
+
 (rum/defc ^:large-vars/cleanup-todo header-aux < rum/reactive
   [{:keys [current-repo default-home new-block-mode]}]
   (let [electron-mac? (and util/mac? (util/electron?))
@@ -349,7 +374,8 @@
                                                  (state/set-left-sidebar-open!
                                                   (not (:ui/left-sidebar-open? @state/state))))})
         custom-home-page? (and (state/custom-home-page?)
-                               (= (state/sub-default-home-page) (state/get-current-page)))]
+                               (= (state/sub-default-home-page) (state/get-current-page)))
+        db-based? (config/db-based-graph? current-repo)]
     [:div.cp__header.drag-region#head
      {:class           (util/classnames [{:electron-mac   electron-mac?
                                           :native-ios     (mobile-util/native-ios?)
@@ -377,7 +403,8 @@
          (when current-repo
            (ui/with-shortcut :go/search "right"
              [:button.button.icon#search-button
-              {:title (t :header/search)
+              {:data-keep-selection true
+               :title (t :header/search)
                :on-click #(do (when (or (mobile-util/native-android?)
                                         (mobile-util/native-iphone?))
                                 (state/set-left-sidebar-open! false))
@@ -387,11 +414,11 @@
      [:div.r.flex.drag-region.justify-between.items-center.gap-2.overflow-x-hidden.w-full
       [:div.flex.flex-1
        (block-breadcrumb (state/get-current-page))]
-      [:div.flex
+      [:div.flex.items-center
        (when (and current-repo
                   (ldb/get-graph-rtc-uuid (db/get-db))
                   (user-handler/logged-in?)
-                  (config/db-based-graph? current-repo)
+                  db-based?
                   (user-handler/rtc-group?))
          [:<>
           (recent-slider)
@@ -404,9 +431,12 @@
        (when (user-handler/logged-in?)
          (rtc-indicator/uploading-detail))
 
+       (when db-based?
+         (semantic-search-progressing current-repo))
+
        (when (and current-repo
                   (not (config/demo-graph? current-repo))
-                  (not (config/db-based-graph? current-repo))
+                  (not db-based?)
                   (user-handler/alpha-or-beta-user?))
          (fs-sync/indicator))
 

src/main/frontend/components/library.cljs

@@ -7,7 +7,6 @@
             [frontend.search :as search]
             [frontend.state :as state]
             [frontend.ui :as ui]
-            [logseq.db :as ldb]
             [logseq.shui.hooks :as hooks]
             [logseq.shui.ui :as shui]
             [promesa.core :as p]
@@ -39,12 +38,8 @@
       :selected-choices selected-choices
       :on-chosen (fn [chosen selected?]
                    (if selected?
-                     (let [last-child (->> (:block/_parent (db/entity (:db/id library-page)))
-                                           ldb/sort-by-order
-                                           last)
-                           target (or last-child library-page)
-                           chosen-block (db/entity chosen)]
-                       (editor-handler/move-blocks! [chosen-block] target (if last-child true false))
+                     (let [chosen-block (db/entity chosen)]
+                       (editor-handler/move-blocks! [chosen-block] library-page {:bottom? true})
                        (set-selected-choices! (conj selected-choices chosen)))
                      (do
                        (db/transact! (state/get-current-repo)

src/main/frontend/components/property.cljs

@@ -490,7 +490,10 @@
 
          (let [property-desc (when-not (= (:db/ident property) :logseq.property/description)
                                (:logseq.property/description property))]
-           [:div.ls-block.property-value-container.flex.flex-row.gap-1.items-start
+           [:div.ls-block.property-value-container.flex.flex-row.gap-1
+            {:class (if (contains? #{:checkbox :date :datetime} type)
+                      "items-center"
+                      "items-start")}
 
             (when-not (or block? (and property-desc (:class-schema? opts)))
               [:div.flex.items-center {:style {:height 28}}

src/main/frontend/components/property/dialog.cljs

@@ -14,6 +14,7 @@
                  k (:property-key opts)]
              (when-let [view-selected-blocks (:selected-blocks opts)]
                (state/set-state! :view/selected-blocks view-selected-blocks))
+             (state/set-state! :ui/show-property-dialog? true)
              (assoc state
                     ::property-key (atom k)
                     ::property (atom (when k (db/get-case-page k))))))
@@ -21,6 +22,7 @@
                    (when-let [close-fn (:on-dialog-close (last (:rum/args state)))]
                      (close-fn))
                    (state/set-state! :view/selected-blocks nil)
+                   (state/set-state! :ui/show-property-dialog? false)
                    state)}
   [state blocks opts]
   (when (seq blocks)

src/main/frontend/components/property/value.cljs

@@ -249,8 +249,7 @@
        (p/do!
         (ui-outliner-tx/transact!
          {:outliner-op :save-block}
-         (doseq [block blocks]
-           (db-property-handler/delete-property-value! (:db/id block) (:db/ident property) value)))
+         (db-property-handler/batch-delete-property-value! (map :db/id blocks) (:db/ident property) value))
         (when (or (not many?)
                   ;; values will be cleared
                   (and many? (<= (count (get block (:db/ident property))) 1)))
@@ -698,10 +697,13 @@
                   excluded-options)
 
                 (contains? #{:class :property} property-type)
-                (let [classes (model/get-all-classes
-                               repo
-                               {:except-root-class? true
-                                :except-private-tags? (not (contains? #{:logseq.property/template-applied-to} (:db/ident property)))})]
+                (let [classes (cond->
+                               (model/get-all-classes
+                                repo
+                                {:except-root-class? true
+                                 :except-private-tags? (not (contains? #{:logseq.property/template-applied-to} (:db/ident property)))})
+                                (not (or (and (entity-util/page? block) (not (ldb/internal-page? block))) (:logseq.property/created-from-property block)))
+                                (conj (db/entity :logseq.class/Page)))]
                   (if (= property-type :class)
                     classes
                     (property-handler/get-class-property-choices)))
@@ -771,7 +773,8 @@
                                 :label label
                                 :value id
                                 :disabled? (and tags? (contains?
-                                                       (set/union #{:logseq.class/Journal :logseq.class/Whiteboard} ldb/internal-tags)
+                                                       (set/union #{:logseq.class/Journal :logseq.class/Whiteboard}
+                                                                  (set/difference ldb/internal-tags #{:logseq.class/Page}))
                                                        (:db/ident node)))))) nodes)
         classes' (remove (fn [class] (= :logseq.class/Root (:db/ident class))) classes)
         opts' (cond->
@@ -788,13 +791,12 @@
                                               "Set alias"
                                               :else
                                               (str "Set " (:block/title property)))
-                 :show-new-when-not-exact-match? (if (or extends-property?
-                                                         ;; Don't allow creating private tags
-                                                         (and (= :block/tags (:db/ident property))
-                                                              (seq (set/intersection (set (map :db/ident classes'))
-                                                                                     ldb/private-tags))))
-                                                   false
-                                                   true)
+                 :show-new-when-not-exact-match? (not
+                                                  (or (and extends-property? (contains? (set children-pages) (:db/id block)))
+                                                      ;; Don't allow creating private tags
+                                                      (and (= :block/tags (:db/ident property))
+                                                           (seq (set/intersection (set (map :db/ident classes'))
+                                                                                  ldb/private-tags)))))
                  :extract-chosen-fn :value
                  :extract-fn (fn [x] (or (:label-value x) (:label x)))
                  :input-opts input-opts
@@ -825,6 +827,10 @@
                                   (when-not add-tag-property?
                                     (log/error :msg "No :db/id found or created for chosen" :chosen chosen)))))})
 
+                (= :block/tags (:db/ident property))
+                (assoc :exact-match-exclude-items
+                       (set (map (fn [ident] (:block/title (db/entity ident))) ldb/private-tags)))
+
                 (and (seq classes') (not tags-or-alias?))
                 (assoc
                   ;; Provides additional completion for inline classes on new pages or objects

src/main/frontend/components/right_sidebar.cljs

@@ -8,6 +8,7 @@
             [frontend.components.page :as page]
             [frontend.components.profiler :as profiler]
             [frontend.components.shortcut-help :as shortcut-help]
+            [frontend.components.vector-search.sidebar :as vector-search]
             [frontend.config :as config]
             [frontend.context.i18n :refer [t]]
             [frontend.date :as date]
@@ -150,6 +151,10 @@
         [[:.flex.items-center (ui/icon "cloud" {:class "text-md mr-2"}) "(Dev) Profiler"]
          (profiler/profiler)]
 
+        :vector-search
+        [[:.flex.items-center (ui/icon "file-search" {:class "text-md mr-2"}) "(Dev) VectorSearch"]
+         (vector-search/vector-search-sidebar)]
+
         ["" [:span]])))
    (p/catch (fn [error]
               (js/console.error error)))))
@@ -443,6 +448,12 @@
            [:button.button.cp__right-sidebar-settings-btn {:on-click (fn [_e]
                                                                        (state/sidebar-add-block! repo "rtc" :rtc))}
             "(Dev) RTC"]])
+        (when (state/sub [:ui/developer-mode?])
+          [:div.text-sm
+           [:button.button.cp__right-sidebar-settings-btn
+            {:on-click (fn [_e]
+                         (state/sidebar-add-block! repo "vector-search" :vector-search))}
+            "(Dev) vector-search"]])
         (when (state/sub [:ui/developer-mode?])
           [:div.text-sm
            [:button.button.cp__right-sidebar-settings-btn {:on-click (fn [_e]

src/main/frontend/components/select.cljs

@@ -69,7 +69,7 @@
               :on-change (fn [e]
                            (let [v (util/evalue e)]
                              (set-input! v)))}
-        input-opts)]]))
+             input-opts)]]))
 
 ;; TODO: rewrite using hooks
 (rum/defcs ^:large-vars/cleanup-todo select
@@ -172,6 +172,7 @@
                                                                      (render-item result chosen? multiple-choices? *selected-choices)))
                                     :class             "cp__select-results"
                                     :on-chosen         (fn [raw-chosen e]
+                                                         (util/stop-propagation e)
                                                          (when clear-input-on-chosen?
                                                            (reset! *input ""))
                                                          (let [chosen (extract-chosen-fn raw-chosen)]

src/main/frontend/components/selection.cljs

@@ -9,79 +9,81 @@
             [logseq.shui.ui :as shui]
             [rum.core :as rum]))
 
-(rum/defc action-bar
+(rum/defc action-bar < rum/reactive
   [& {:keys [on-cut on-copy selected-blocks hide-dots? button-border?]
       :or {on-cut #(editor-handler/cut-selection-blocks true)}}]
-  (let [selected-blocks (map (fn [block] (if (number? block) (db/entity block) block)) selected-blocks)
-        on-copy (if (and selected-blocks (nil? on-copy))
-                  #(editor-handler/copy-selection-blocks true {:selected-blocks selected-blocks})
-                  (or on-copy #(editor-handler/copy-selection-blocks true)))
-        button-opts {:variant :outline
-                     :size :sm
-                     :class (str "p-2 text-xs h-8"
-                                 (when-not button-border?
-                                   " !border-b-0"))}
-        db-graph? (config/db-based-graph?)]
-    [:div.selection-action-bar
-     (shui/button-group
-      ;; set tag
-      (when db-graph?
+  (when-not (or (state/sub :search/mode)
+                (state/sub :ui/show-property-dialog?))
+    (let [selected-blocks (map (fn [block] (if (number? block) (db/entity block) block)) selected-blocks)
+          on-copy (if (and selected-blocks (nil? on-copy))
+                    #(editor-handler/copy-selection-blocks true {:selected-blocks selected-blocks})
+                    (or on-copy #(editor-handler/copy-selection-blocks true)))
+          button-opts {:variant :outline
+                       :size :sm
+                       :class (str "p-2 text-xs h-8"
+                                   (when-not button-border?
+                                     " !border-b-0"))}
+          db-graph? (config/db-based-graph?)]
+      [:div.selection-action-bar
+       (shui/button-group
+        ;; set tag
+        (when db-graph?
+          (shui/button
+           (assoc button-opts
+                  :on-pointer-down (fn [e]
+                                     (util/stop e)
+                                     (state/pub-event! [:editor/new-property {:target (.-target e)
+                                                                              :selected-blocks selected-blocks
+                                                                              :property-key "Tags"
+                                                                              :on-dialog-close #(state/pub-event! [:editor/hide-action-bar])}])))
+           (ui/tooltip (ui/icon "hash" {:size 13}) "Set tag"
+                       {:trigger-props {:class "flex"}})))
         (shui/button
          (assoc button-opts
                 :on-pointer-down (fn [e]
                                    (util/stop e)
-                                   (state/pub-event! [:editor/new-property {:target (.-target e)
-                                                                            :selected-blocks selected-blocks
-                                                                            :property-key "Tags"
-                                                                            :on-dialog-close #(state/pub-event! [:editor/hide-action-bar])}])))
-         (ui/tooltip (ui/icon "hash" {:size 13}) "Set tag"
-                     {:trigger-props {:class "flex"}})))
-      (shui/button
-       (assoc button-opts
-              :on-pointer-down (fn [e]
-                                 (util/stop e)
-                                 (on-copy)
-                                 (state/clear-selection!)
-                                 (state/pub-event! [:editor/hide-action-bar])))
-       "Copy")
-      (when db-graph?
+                                   (on-copy)
+                                   (state/clear-selection!)
+                                   (state/pub-event! [:editor/hide-action-bar])))
+         "Copy")
+        (when db-graph?
+          (shui/button
+           (assoc button-opts
+                  :on-pointer-down (fn [e]
+                                     (util/stop e)
+                                     (state/pub-event! [:editor/new-property {:target (.-target e)
+                                                                              :selected-blocks selected-blocks
+                                                                              :on-dialog-close #(state/pub-event! [:editor/hide-action-bar])}])))
+           "Set property"))
+        (when db-graph?
+          (shui/button
+           (assoc button-opts
+                  :on-pointer-down (fn [e]
+                                     (util/stop e)
+                                     (state/pub-event! [:editor/new-property {:target (.-target e)
+                                                                              :selected-blocks selected-blocks
+                                                                              :remove-property? true
+                                                                              :select-opts {:show-new-when-not-exact-match? false}
+                                                                              :on-dialog-close #(state/pub-event! [:editor/hide-action-bar])}])))
+           "Unset property"))
         (shui/button
          (assoc button-opts
                 :on-pointer-down (fn [e]
                                    (util/stop e)
-                                   (state/pub-event! [:editor/new-property {:target (.-target e)
-                                                                            :selected-blocks selected-blocks
-                                                                            :on-dialog-close #(state/pub-event! [:editor/hide-action-bar])}])))
-         "Set property"))
-      (when db-graph?
-        (shui/button
-         (assoc button-opts
-                :on-pointer-down (fn [e]
-                                   (util/stop e)
-                                   (state/pub-event! [:editor/new-property {:target (.-target e)
-                                                                            :selected-blocks selected-blocks
-                                                                            :remove-property? true
-                                                                            :select-opts {:show-new-when-not-exact-match? false}
-                                                                            :on-dialog-close #(state/pub-event! [:editor/hide-action-bar])}])))
-         "Unset property"))
-      (shui/button
-       (assoc button-opts
-              :on-pointer-down (fn [e]
-                                 (util/stop e)
-                                 (on-cut)
-                                 (state/pub-event! [:editor/hide-action-bar])))
-       (ui/icon "trash" {:size 13}))
-      (when-not hide-dots?
-        (shui/button
-         (assoc button-opts
-                :on-pointer-down (fn [e]
-                                   (util/stop e)
-                                   (shui/popup-hide!)
-                                   (shui/popup-show! e
-                                                     (fn [{:keys [id]}]
-                                                       [:div {:on-click #(shui/popup-hide! id)
-                                                              :data-keep-selection true}
-                                                        ((state/get-component :selection/context-menu))])
-                                                     {:content-props {:class "w-[280px] ls-context-menu-content"}
-                                                      :as-dropdown? true})))
-         (ui/icon "dots" {:size 13}))))]))
+                                   (on-cut)
+                                   (state/pub-event! [:editor/hide-action-bar])))
+         (ui/icon "trash" {:size 13}))
+        (when-not hide-dots?
+          (shui/button
+           (assoc button-opts
+                  :on-pointer-down (fn [e]
+                                     (util/stop e)
+                                     (shui/popup-hide!)
+                                     (shui/popup-show! e
+                                                       (fn [{:keys [id]}]
+                                                         [:div {:on-click #(shui/popup-hide! id)
+                                                                :data-keep-selection true}
+                                                          ((state/get-component :selection/context-menu))])
+                                                       {:content-props {:class "w-[280px] ls-context-menu-content"}
+                                                        :as-dropdown? true})))
+           (ui/icon "dots" {:size 13}))))])))

src/main/frontend/components/settings.cljs

@@ -1,7 +1,9 @@
 (ns frontend.components.settings
   (:require [clojure.string :as string]
+            [clojure.walk :as walk]
             [electron.ipc :as ipc]
             [frontend.colors :as colors]
+            [frontend.common.missionary :as c.m]
             [frontend.components.assets :as assets]
             [frontend.components.file-sync :as fs]
             [frontend.components.shortcut :as shortcut]
@@ -13,6 +15,7 @@
             [frontend.dicts :as dicts]
             [frontend.handler.config :as config-handler]
             [frontend.handler.db-based.rtc :as rtc-handler]
+            [frontend.handler.db-based.vector-search-flows :as vector-search-flows]
             [frontend.handler.file-sync :as file-sync-handler]
             [frontend.handler.global-config :as global-config-handler]
             [frontend.handler.notification :as notification]
@@ -35,6 +38,7 @@
             [logseq.db :as ldb]
             [logseq.shui.hooks :as hooks]
             [logseq.shui.ui :as shui]
+            [missionary.core :as m]
             [promesa.core :as p]
             [reitit.frontend.easy :as rfe]
             [rum.core :as rum]))
@@ -1189,6 +1193,87 @@
                            (rtc-handler/<rtc-invite-email graph-uuid user-email)))))}
         "Invite")]]]))
 
+(rum/defc settings-ai
+  []
+  (let [[model-info set-model-info] (hooks/use-state nil)
+        [load-model-progress set-load-model-progress] (hooks/use-state nil)
+        {:keys [status]} load-model-progress
+        repo (state/get-current-repo)
+        current-model (:graph-text-embedding-model-name model-info)]
+    (hooks/use-effect!
+     (fn []
+       (c.m/run-task
+         ::fetch-model-info
+         (m/reduce
+          (constantly nil)
+          (m/ap
+            (m/?> vector-search-flows/infer-worker-ready-flow)
+            (let [model-info (c.m/<? (state/<invoke-db-worker :thread-api/vec-search-embedding-model-info repo))]
+              (set-model-info model-info))))
+         :succ (constantly nil)))
+     [])
+    (hooks/use-effect!
+     (fn []
+       (c.m/run-task
+         ::update-load-model-progress
+         (m/reduce
+          (fn [_ v] (set-load-model-progress (walk/keywordize-keys v)))
+          vector-search-flows/load-model-progress-flow)
+         :succ (constantly nil)))
+     [])
+    [:div.panel-wrap
+     [:div.flex.flex-col.gap-2.mt-4
+      [:div.font-medium.text-muted-foreground.text-sm "Semantic search:"]
+
+      [:div.flex.flex-col.gap-2
+       [:div.it.sm:grid.sm:grid-cols-3.sm:gap-4.sm:items-start
+        [:label.block.text-sm.font-medium.leading-8.opacity-70
+         {:for "local-embedding-model"}
+         "Local embedding model"]
+        [:div.rounded-md.sm:max-w-tss.sm:col-span-2
+         [:div.flex.flex-col.gap-2
+          (shui/select
+           (cond->
+            {:on-value-change (fn [model-name]
+                                (c.m/run-task
+                                  ::load-model
+                                  (m/sp
+                                    (c.m/<?
+                                     (state/<invoke-db-worker :thread-api/vec-search-load-model repo model-name))
+                                    (set-model-info (assoc model-info :graph-text-embedding-model-name model-name))
+                                    (c.m/<?
+                                     (state/<invoke-db-worker :thread-api/vec-search-cancel-indexing repo))
+                                    (c.m/<?
+                                     (state/<invoke-db-worker :thread-api/vec-search-re-embedding-graph-data repo)))
+                                  :succ (constantly nil)))}
+             current-model
+             (assoc :value current-model))
+           (shui/select-trigger
+            {:class "h-8"}
+            (shui/select-value
+             {:placeholder "Select a model"}))
+
+           (shui/select-content
+            (shui/select-group
+             (for [model-name (:available-model-names model-info)]
+               (shui/select-item {:value model-name} model-name)))))
+
+          (when (and status current-model)
+            [:div.text-muted-foreground.text-sm
+             (let [{:keys [file progress loaded total]} load-model-progress]
+               (case status
+                 ("progress" "download" "initiate")
+                 (str "Downloading " file
+                      (when progress
+                        (util/format " %d/%dm"
+                                     (int (/ loaded 1024 1024))
+                                     (int (/ total 1024 1024)))))
+                 "done"
+                 (str "Downloaded " file)
+                 "ready"
+                 "Model is ready  🚀"
+                 nil))])]]]]]]))
+
 (rum/defcs ^:large-vars/cleanup-todo settings
   < (rum/local DEFAULT-ACTIVE-TAB-STATE ::active)
   {:will-mount
@@ -1212,7 +1297,8 @@
         _installed-plugins (state/sub :plugin/installed-plugins)
         plugins-of-settings (and config/lsp-enabled? (seq (plugin-handler/get-enabled-plugins-if-setting-schema)))
         *active (::active state)
-        logged-in? (user-handler/logged-in?)]
+        logged-in? (user-handler/logged-in?)
+        db-based? (config/db-based-graph?)]
 
     [:div#settings.cp__settings-main
      (settings-effect @*active)
@@ -1228,6 +1314,8 @@
                [:editor "editor" (t :settings-page/tab-editor) (ui/icon "writing")]
                [:keymap "keymap" (t :settings-page/tab-keymap) (ui/icon "keyboard")]
 
+               (when db-based?
+                 [:ai (t :settings-page/tab-ai) (t :settings-page/ai) (ui/icon "wand")])
                (when (util/electron?)
                  [:version-control "git" (t :settings-page/tab-version-control) (ui/icon "history")])
 
@@ -1293,4 +1381,7 @@
          :collaboration
          (settings-collaboration)
 
+         :ai
+         (settings-ai)
+
          nil)]]]))

src/main/frontend/components/settings.css

@@ -56,7 +56,7 @@
     }
 
     h1.cp__settings-category-title {
-      @apply text-xl lowercase;
+      @apply text-xl;
     }
 
     h1.cp__settings-modal-title:first-letter,

src/main/frontend/components/shortcut.cljs

@@ -52,8 +52,8 @@
        (let [key-handler (KeyHandler. js/document)]
           ;; setup
          (util/profile
-          "[shortcuts] unlisten*"
-          (shortcut/unlisten-all! true))
+           "[shortcuts] unlisten*"
+           (shortcut/unlisten-all! true))
          (events/listen key-handler "key"
                         (fn [^js e]
                           (.preventDefault e)
@@ -62,8 +62,8 @@
           ;; teardown
          #(do
             (util/profile
-             "[shortcuts] listen*"
-             (shortcut/listen-all!))
+              "[shortcuts] listen*"
+              (shortcut/listen-all!))
             (.dispose key-handler))))
      [])
 

src/main/frontend/components/vector_search/sidebar.cljs

@@ -0,0 +1,119 @@
+(ns frontend.components.vector-search.sidebar
+  (:require [clojure.string :as string]
+            [fipp.edn :as fipp]
+            [frontend.common.missionary :as c.m]
+            [frontend.handler.db-based.vector-search-flows :as vector-search-flows]
+            [frontend.state :as state]
+            [frontend.util :as util]
+            [logseq.shui.hooks :as hooks]
+            [logseq.shui.ui :as shui]
+            [missionary.core :as m]
+            [rum.core :as rum]))
+
+(rum/defc ^:large-vars/cleanup-todo vector-search-sidebar
+  []
+  (let [repo (state/get-current-repo)
+        [model-info set-model-info] (hooks/use-state nil)
+        [vec-search-state set-vec-search-state] (hooks/use-state nil)
+        [load-model-progress set-load-model-progress] (hooks/use-state nil)
+        [query-string set-query-string] (hooks/use-state nil)
+        [result set-result] (hooks/use-state nil)]
+    (hooks/use-effect!
+     (fn []
+       (c.m/run-task
+         ::update-vec-search-state
+         (m/reduce
+          (fn [_ v]
+            (set-vec-search-state v))
+          (m/ap
+            (m/?> vector-search-flows/infer-worker-ready-flow)
+            (c.m/<? (state/<invoke-db-worker :thread-api/vec-search-update-index-info repo))
+            (m/?> vector-search-flows/vector-search-state-flow)))
+         :succ (constantly nil)))
+     [])
+    (hooks/use-effect!
+     (fn []
+       (c.m/run-task
+         ::update-load-model-progress
+         (m/reduce
+          (fn [_ v] (set-load-model-progress v))
+          vector-search-flows/load-model-progress-flow)
+         :succ (constantly nil)))
+     [])
+    (hooks/use-effect!
+     (fn []
+       (c.m/run-task
+         ::fetch-model-info
+         (m/reduce
+          (constantly nil)
+          (m/ap
+            (m/?> vector-search-flows/infer-worker-ready-flow)
+            (let [model-info (c.m/<? (state/<invoke-db-worker :thread-api/vec-search-embedding-model-info repo))]
+              (set-model-info model-info))))
+         :succ (constantly nil)))
+     [])
+    (hooks/use-effect!
+     (fn []
+       (when-not (string/blank? query-string)
+         (c.m/run-task
+           :update-search-result
+           (m/sp
+             (-> (c.m/<? (state/<invoke-db-worker :thread-api/vec-search-search repo query-string 10))
+                 set-result))
+           :succ (constantly nil))))
+     [(hooks/use-debounced-value query-string 200)])
+    [:div
+     [:b "State"]
+     (let [state-map (assoc (get-in vec-search-state [:repo->index-info repo])
+                            :load-model-progress load-model-progress)]
+       [:pre.select-text
+        (with-out-str
+          (fipp/pprint state-map {:width 10}))])
+     [:hr]
+     [:b "Actions"]
+     [:div
+      (shui/button
+       {:size :sm
+        :class "mx-2"
+        :on-click (fn [_]
+                    (state/<invoke-db-worker :thread-api/vec-search-embedding-stale-blocks repo))}
+       "embedding-stale-blocks")
+      (shui/button
+       {:size :sm
+        :class "mx-2"
+        :on-click (fn [_]
+                    (state/<invoke-db-worker :thread-api/vec-search-re-embedding-graph-data repo))}
+       "force-embedding-all-graph-blocks")
+      (when (get-in vec-search-state [:repo->index-info repo :indexing?])
+        (shui/button
+         {:size :sm
+          :class "mx-2"
+          :on-click (fn [_]
+                      (state/<invoke-db-worker :thread-api/vec-search-cancel-indexing repo))}
+         "cancel-current-indexing"))]
+     [:hr]
+     [:b "Settings"]
+     (shui/select
+      {:on-value-change (fn [model-name]
+                          (c.m/run-task
+                            ::load-model
+                            (m/sp
+                              (c.m/<?
+                               (state/<invoke-db-worker :thread-api/vec-search-load-model repo model-name)))
+                            :succ (constantly nil)))}
+      (shui/select-trigger
+       (shui/select-value
+        {:placeholder "Select a model(need force-embedding-all-graph-blocks again)"}))
+      (shui/select-content
+       (shui/select-group
+        (let [graph-text-embedding-model-name (:graph-text-embedding-model-name model-info)]
+          (for [model-name (:available-model-names model-info)]
+            (shui/select-item {:value model-name :disabled? (= graph-text-embedding-model-name model-name)} model-name))))))
+     [:hr]
+     [:b "Search"]
+     [:input.form-input.my-2.py-1
+      {:on-change (fn [e] (set-query-string (util/evalue e)))}]
+     [:b "Search Result:"]
+     [:pre.select-text
+      (with-out-str
+        (fipp/pprint result {:width 10}))]]))

src/main/frontend/core.cljs

@@ -9,6 +9,7 @@
             [frontend.fs.sync :as sync]
             [frontend.handler :as handler]
             [frontend.handler.db-based.rtc-background-tasks]
+            [frontend.handler.db-based.vector-search-background-tasks]
             [frontend.handler.plugin :as plugin-handler]
             [frontend.handler.route :as route-handler]
             [frontend.log]

src/main/frontend/db/file_based/model.cljs

@@ -3,8 +3,8 @@
   (:require [clojure.string :as string]
             [datascript.core :as d]
             [frontend.common.file-based.db :as common-file-db]
-            [frontend.db.utils :as db-utils]
             [frontend.db.conn :as conn]
+            [frontend.db.utils :as db-utils]
             [frontend.state :as state]
             [frontend.util :as util]
             [logseq.db :as ldb]

src/main/frontend/db/react.cljs

@@ -172,34 +172,34 @@
   [repo-url affected-keys & {:keys [skip-kv-custom-keys?]
                              :or {skip-kv-custom-keys? false}}]
   (util/profile
-   "refresh!"
-   (let [db (conn/get-db repo-url)
-         affected-keys-set (set affected-keys)
-         state (->> (keep (fn [[k cache]]
-                            (let [k' (vec (rest k))]
-                              (when (and (= (first k) repo-url)
-                                         (or (contains? affected-keys-set k')
-                                             (contains? #{:custom :kv} (first k'))))
-                                [k' cache]))) @*query-state)
-                    (into {}))
-         all-keys (concat (distinct affected-keys)
-                          (when-not skip-kv-custom-keys?
-                            (filter #(contains? #{:custom :kv} (first %)) (keys state))))]
-     (doseq [k all-keys]
-       (when-let [cache (get state k)]
-         (let [{:keys [query query-fn]} cache
-               custom? (= :custom (first k))]
-           (when (or query query-fn)
-             (try
-               (let [f #(execute-query! repo-url db (vec (cons repo-url k)) cache)]
-                 (if custom?
+    "refresh!"
+    (let [db (conn/get-db repo-url)
+          affected-keys-set (set affected-keys)
+          state (->> (keep (fn [[k cache]]
+                             (let [k' (vec (rest k))]
+                               (when (and (= (first k) repo-url)
+                                          (or (contains? affected-keys-set k')
+                                              (contains? #{:custom :kv} (first k'))))
+                                 [k' cache]))) @*query-state)
+                     (into {}))
+          all-keys (concat (distinct affected-keys)
+                           (when-not skip-kv-custom-keys?
+                             (filter #(contains? #{:custom :kv} (first %)) (keys state))))]
+      (doseq [k all-keys]
+        (when-let [cache (get state k)]
+          (let [{:keys [query query-fn]} cache
+                custom? (= :custom (first k))]
+            (when (or query query-fn)
+              (try
+                (let [f #(execute-query! repo-url db (vec (cons repo-url k)) cache)]
+                  (if custom?
                    ;; perf: don't execute custom queries if they were collapsed
-                   (when-not (query-collapsed? k)
-                     (async/put! (state/get-reactive-custom-queries-chan) [f query]))
-                   (f)))
-               (catch :default e
-                 (js/console.error e)
-                 nil)))))))))
+                    (when-not (query-collapsed? k)
+                      (async/put! (state/get-reactive-custom-queries-chan) [f query]))
+                    (f)))
+                (catch :default e
+                  (js/console.error e)
+                  nil)))))))))
 
 (defn refresh!
   "Re-compute corresponding queries (from tx) and refresh the related react components."

src/main/frontend/db/utils.cljs

@@ -3,8 +3,8 @@
   (:require [datascript.core :as d]
             [frontend.db.conn :as conn]
             [frontend.state :as state]
-            [logseq.db.frontend.content :as db-content]
-            [logseq.db.common.entity-plus :as entity-plus]))
+            [logseq.db.common.entity-plus :as entity-plus]
+            [logseq.db.frontend.content :as db-content]))
 
 ;; transit serialization
 

src/main/frontend/extensions/latex.cljs

@@ -37,18 +37,18 @@
     (when-not @*loading?
       (reset! *loading? true)
       (loader/load "/js/katex.min.js"
-        (fn []
-          (loader/load "/js/mhchem.min.js"
-            (fn []
-              (-> (when-let [enhancers (and config/lsp-enabled?
-                                         (seq (hook-extensions-enhancers-by-key :katex)))]
-                    (for [{f :enhancer} enhancers]
-                      (when (fn? f) (f js/window.katex))))
-                (p/all)
-                (p/finally (fn []
-                             (reset! *loading? false)
-                             (render! state)))))))
-        state))))
+                   (fn []
+                     (loader/load "/js/mhchem.min.js"
+                                  (fn []
+                                    (-> (when-let [enhancers (and config/lsp-enabled?
+                                                                  (seq (hook-extensions-enhancers-by-key :katex)))]
+                                          (for [{f :enhancer} enhancers]
+                                            (when (fn? f) (f js/window.katex))))
+                                        (p/all)
+                                        (p/finally (fn []
+                                                     (reset! *loading? false)
+                                                     (render! state)))))))
+                   state))))
 
 (defn- state-&-load-and-render!
   [state]

src/main/frontend/fs/diff_merge.cljs

@@ -2,14 +2,13 @@
   "Implementation of text (file) based content diff & merge for conflict resolution"
   (:require ["@logseq/diff-merge" :refer [attach_uuids Differ Merger]]
             [cljs-bean.core :as bean]
+            [clojure.string :as string]
             [frontend.db.file-based.model :as file-model]
             [frontend.db.utils :as db-utils]
             [logseq.graph-parser.block :as gp-block]
             [logseq.graph-parser.mldoc :as gp-mldoc]
             [logseq.graph-parser.property :as gp-property]
-            [logseq.graph-parser.utf8 :as utf8]
-            [clojure.string :as string]))
-
+            [logseq.graph-parser.utf8 :as utf8]))
 
 (defn diff
   "2-ways diff
@@ -91,7 +90,6 @@
                    :uuid uuid}
                   (reverse headings))))))))
 
-
 (defn- get-sub-content-from-pos-meta
   "Replace gp-block/get-block-content, return bare content, without any trim"
   [raw-content pos-meta]

src/main/frontend/fs/watcher_handler.cljs

@@ -5,8 +5,8 @@
             [frontend.config :as config]
             [frontend.db :as db]
             [frontend.db.async :as db-async]
-            [frontend.db.model :as model]
             [frontend.db.file-based.model :as file-model]
+            [frontend.db.model :as model]
             [frontend.fs :as fs]
             [frontend.handler.file-based.file :as file-handler]
             [frontend.handler.file-based.property :as file-property-handler]

src/main/frontend/handler.cljs

@@ -16,6 +16,7 @@
             [frontend.db.restore :as db-restore]
             [frontend.error :as error]
             [frontend.handler.command-palette :as command-palette]
+            [frontend.handler.db-based.vector-search-flows :as vector-search-flows]
             [frontend.handler.events :as events]
             [frontend.handler.events.ui]
             [frontend.handler.file-based.events]
@@ -167,8 +168,15 @@
                repo (or (state/get-current-repo) (:url (first repos)))
                _ (if (empty? repos)
                    (repo-handler/new-db! config/demo-repo)
-                   (restore-and-setup! repo))]
+                   (restore-and-setup! repo))
+               webgpu-available? (db-browser/<check-webgpu-available?)]
+         (log/info :webgpu-available? webgpu-available?)
+         (when webgpu-available?
+           (p/do! (db-browser/start-inference-worker!)
+                  (db-browser/<connect-db-worker-and-infer-worker!)
+                  (reset! vector-search-flows/*infer-worker-ready true)))
          (set-network-watcher!)
+
          (when (util/electron?)
            (persist-db/run-export-periodically!))
          (when (mobile-util/native-platform?)

src/main/frontend/handler/db_based/property.cljs

@@ -37,6 +37,12 @@
    {:outliner-op :delete-property-value}
    (outliner-op/delete-property-value! block-id property-id property-value)))
 
+(defn batch-delete-property-value!
+  [block-ids property-id property-value]
+  (ui-outliner-tx/transact!
+   {:outliner-op :batch-delete-property-value}
+   (outliner-op/batch-delete-property-value! block-ids property-id property-value)))
+
 (defn create-property-text-block!
   [block-id property-id value opts]
   (ui-outliner-tx/transact!

src/main/frontend/handler/db_based/property/util.cljs

@@ -1,7 +1,7 @@
 (ns frontend.handler.db-based.property.util
   "DB-graph only utility fns for properties"
-  (:require [frontend.db.utils :as db-utils]
-            [frontend.db.conn :as conn]
+  (:require [frontend.db.conn :as conn]
+            [frontend.db.utils :as db-utils]
             [frontend.state :as state]
             [logseq.db.frontend.property :as db-property]))
 
@@ -20,22 +20,22 @@
   ([properties {:keys [original-key? key-fn]
                 :or {key-fn identity}}]
    (->> properties
-     (map (fn [[k v]]
-            (let [prop-ent (db-utils/entity k)]
-              [(key-fn (if original-key? k (-> prop-ent :block/title keyword)))
-               (cond
-                 (set? v)
-                 (set (map db-property/property-value-content v))
+        (map (fn [[k v]]
+               (let [prop-ent (db-utils/entity k)]
+                 [(key-fn (if original-key? k (-> prop-ent :block/title keyword)))
+                  (cond
+                    (set? v)
+                    (set (map db-property/property-value-content v))
 
-                 (sequential? v)
-                 (map #(get-property-value (or (:db/id %) %)) v)
+                    (sequential? v)
+                    (map #(get-property-value (or (:db/id %) %)) v)
 
-                 (:db/id v)
-                 (get-property-value (or (:db/id v) v))
+                    (:db/id v)
+                    (get-property-value (or (:db/id v) v))
 
-                 :else
-                 v)])))
-     (into {}))))
+                    :else
+                    v)])))
+        (into {}))))
 
 (defn get-closed-property-values
   [property-id]

src/main/frontend/handler/db_based/vector_search_background_tasks.cljs

@@ -0,0 +1,28 @@
+(ns frontend.handler.db-based.vector-search-background-tasks
+  "Background tasks for vector-search"
+  (:require [frontend.common.missionary :as c.m]
+            [frontend.config :as config]
+            [frontend.flows :as flows]
+            [frontend.handler.db-based.vector-search-flows :as vector-search-flows]
+            [frontend.state :as state]
+            [missionary.core :as m])
+  (:import [missionary Cancelled]))
+
+(defn- run-background-task-when-not-publishing
+  [key' task]
+  (when-not config/publishing?
+    (c.m/run-background-task key' task)))
+
+(run-background-task-when-not-publishing
+ ::init-load-model-when-switch-graph
+ (m/reduce
+  (constantly nil)
+  (m/ap
+    (m/?< vector-search-flows/infer-worker-ready-flow)
+    (when-let [repo (m/?< flows/current-repo-flow)]
+      (try
+        (c.m/<? (state/<invoke-db-worker :thread-api/vec-search-init-embedding-model repo))
+        (m/?< (c.m/clock (* 30 1000)))
+        (c.m/<? (state/<invoke-db-worker :thread-api/vec-search-embedding-graph repo))
+        (catch Cancelled _
+          (m/amb)))))))

src/main/frontend/handler/db_based/vector_search_flows.cljs

@@ -0,0 +1,22 @@
+(ns frontend.handler.db-based.vector-search-flows
+  "Flows for vector-search state"
+  (:require [frontend.state :as state]
+            [missionary.core :as m]))
+
+;; input atoms
+(def *infer-worker-ready (atom nil))
+
+(def infer-worker-ready-flow
+  (m/eduction
+   (filter some?)
+   (take 1)
+   (m/watch *infer-worker-ready)))
+
+(def vector-search-state-flow
+  (m/watch (:vector-search/state @state/state)))
+
+(def load-model-progress-flow
+  (m/watch (:vector-search/load-model-progress @state/state)))
+
+(comment
+  ((m/reduce (fn [_ x] (prn :xx x)) vector-search-state-flow) prn js/console.log))

src/main/frontend/handler/dnd.cljs

@@ -59,12 +59,13 @@
                     (:block/uuid (ldb/get-left-sibling target-block)))]
              (if first-child?
                (when-let [parent (:block/parent target-block)]
-                 (outliner-op/move-blocks! blocks' parent false))
+                 (outliner-op/move-blocks! blocks' parent {:sibling? false}))
                (if-let [before-node (ldb/get-left-sibling target-block)]
-                 (outliner-op/move-blocks! blocks' before-node true)
+                 (outliner-op/move-blocks! blocks' before-node {:sibling? true})
                  (when-let [parent (:block/parent target-block)]
-                   (outliner-op/move-blocks! blocks' parent false)))))
-           (outliner-op/move-blocks! blocks' target-block (not nested?)))))
+                   (outliner-op/move-blocks! blocks' parent {:sibling? false})))))
+           (outliner-op/move-blocks! blocks' target-block
+                                     {:sibling? (not nested?)}))))
 
       :else
       nil)))

src/main/frontend/handler/editor.cljs

@@ -860,7 +860,7 @@
                        (ui-outliner-tx/transact!
                         transact-opts
                         (when (seq children)
-                          (outliner-op/move-blocks! children prev-block false))
+                          (outliner-op/move-blocks! children prev-block {:sibling? false}))
                         (delete-block-aux! block)
                         (save-block! repo prev-block new-content {})))))
 
@@ -870,11 +870,29 @@
                    (delete-block-aux! block)))))))))))
 
 (defn move-blocks!
-  [blocks target sibling?]
+  [blocks target opts]
   (when (seq blocks)
     (ui-outliner-tx/transact!
      {:outliner-op :move-blocks}
-     (outliner-op/move-blocks! blocks target sibling?))))
+     (outliner-op/move-blocks! blocks target opts))))
+
+(defn move-selected-blocks
+  [e]
+  (util/stop e)
+  (let [block-ids (or (seq (state/get-selection-block-ids))
+                      (when-let [id (:block/uuid (state/get-edit-block))]
+                        [id]))]
+    (if (seq block-ids)
+      (let [blocks (->> (map (fn [id] (db/entity [:block/uuid id])) block-ids)
+                        block-handler/get-top-level-blocks)]
+        (route-handler/go-to-search! :nodes
+                                     {:action :move-blocks
+                                      :blocks blocks
+                                      :trigger (fn [chosen]
+                                                 (state/pub-event! [:editor/hide-action-bar])
+                                                 (state/clear-selection!)
+                                                 (move-blocks! blocks (:source-page chosen) {:bottom? true}))}))
+      (notification/show! "There's no block selected, please select blocks first." :warning))))
 
 (defn delete-block!
   [repo]
@@ -1284,6 +1302,10 @@
   (some->> (shui-popup/get-popups)
            (some #(some-> % (:id) (str) (string/includes? (str id))))))
 
+(defn dialog-exists?
+  [id]
+  (shui-dialog/get-modal id))
+
 (defn show-action-bar!
   [& {:keys [delay]
       :or {delay 200}}]
@@ -1644,7 +1666,12 @@
 (defn get-matched-classes
   "Return matched classes except the root tag"
   [q]
-  (let [classes (->> (db-model/get-all-classes (state/get-current-repo) {:except-root-class? true})
+  (let [editing-block (some-> (state/get-edit-block) :db/id db/entity)
+        non-page-block? (and editing-block (not (ldb/page? editing-block)))
+        all-classes (cond-> (db-model/get-all-classes (state/get-current-repo) {:except-root-class? true})
+                      non-page-block?
+                      (conj (db/entity :logseq.class/Page)))
+        classes (->> all-classes
                      (mapcat (fn [class]
                                (conj (:block/alias class) class)))
                      (common-util/distinct-by :db/id)
@@ -3340,18 +3367,19 @@
 
 (defn open-selected-block!
   [direction e]
-  (let [selected-blocks (state/get-selection-blocks)
-        f (case direction :left first :right last)
-        node (some-> selected-blocks f)]
-    (if (some-> node (dom/has-class? "block-add-button"))
-      (.click node)
-      (when-let [block-id (some-> node (dom/attr "blockid") uuid)]
-        (util/stop e)
-        (let [block {:block/uuid block-id}
-              left? (= direction :left)
-              opts {:container-id (some-> node (dom/attr "containerid") (parse-long))
-                    :event e}]
-          (edit-block! block (if left? 0 :max) opts))))))
+  (when-not (auto-complete?)
+    (let [selected-blocks (state/get-selection-blocks)
+          f (case direction :left first :right last)
+          node (some-> selected-blocks f)]
+      (if (some-> node (dom/has-class? "block-add-button"))
+        (.click node)
+        (when-let [block-id (some-> node (dom/attr "blockid") uuid)]
+          (util/stop e)
+          (let [block {:block/uuid block-id}
+                left? (= direction :left)
+                opts {:container-id (some-> node (dom/attr "containerid") (parse-long))
+                      :event e}]
+            (edit-block! block (if left? 0 :max) opts)))))))
 
 (defn shortcut-left-right [direction]
   (fn [e]
@@ -3960,8 +3988,8 @@
         (p/do!
          (when (seq children)
            (if-let [today-last-child (last (ldb/sort-by-order (:block/_parent today)))]
-             (move-blocks! children today-last-child true)
-             (move-blocks! children today false)))
+             (move-blocks! children today-last-child {:sibling? true})
+             (move-blocks! children today {:sibling? false})))
          (state/close-modal!)
          (mobile-state/set-popup! nil)
          (when (seq children)

src/main/frontend/handler/events.cljs

@@ -232,8 +232,8 @@
     (when (mobile-util/native-platform?)
       (reset! util/keyboard-height keyboard-height)
       (util/schedule
-        #(some-> (state/get-input)
-           (util/scroll-editor-cursor false))))))
+       #(some-> (state/get-input)
+                (util/scroll-editor-cursor false))))))
 
 (defmethod handle :mobile/keyboard-will-hide [[_]]
   (let [main-node (util/app-scroll-container-node)]
@@ -373,6 +373,9 @@
                       (db/entity [:block/uuid (:block/uuid block)])))]
        (js/setTimeout #(editor-handler/edit-block! block :max) 100)))))
 
+(defmethod handle :vector-search/sync-state [[_ state]]
+  (state/set-state! :vector-search/state state))
+
 (defmethod handle :rtc/sync-state [[_ state]]
   (state/update-state! :rtc/state (fn [old] (merge old state))))
 
@@ -435,6 +438,9 @@
                   (db-async/<get-block (state/get-current-repo) id
                                        {:skip-refresh? false})) ids))))
 
+(defmethod handle :vector-search/load-model-progress [[_ data]]
+  (state/set-state! :vector-search/load-model-progress data))
+
 (defn run!
   []
   (let [chan (state/get-events-chan)]

src/main/frontend/handler/events/ui.cljs

@@ -44,13 +44,14 @@
             [promesa.core :as p]))
 
 (defmethod events/handle :go/search [_]
-  (shui/dialog-open!
-   cmdk/cmdk-modal
-   {:id :ls-dialog-cmdk
-    :align :top
-    :content-props {:class "ls-dialog-cmdk"}
-    :close-btn? false
-    :onEscapeKeyDown (fn [e] (.preventDefault e))}))
+  (when-not (editor-handler/dialog-exists? :ls-dialog-cmdk)
+    (shui/dialog-open!
+     cmdk/cmdk-modal
+     {:id :ls-dialog-cmdk
+      :align :top
+      :content-props {:class "ls-dialog-cmdk"}
+      :close-btn? false
+      :onEscapeKeyDown (fn [e] (.preventDefault e))})))
 
 (defmethod events/handle :command/run [_]
   (when (util/electron?)
@@ -233,8 +234,7 @@
         (if target'
           (shui/popup-show! target'
                             #(property-dialog/dialog blocks opts')
-                            {:align "start"
-                             :auto-focus? true})
+                            {:align "start"})
           (shui/dialog-open! #(property-dialog/dialog blocks opts')
                              {:id :property-dialog
                               :align "start"}))))))

src/main/frontend/handler/export/opml.cljs

@@ -435,19 +435,19 @@
   {:pre [(or (coll? root-block-uuids-or-page-uuid)
              (uuid? root-block-uuids-or-page-uuid))]}
   (util/profile
-   :export-blocks-as-opml
-   (let [content
-         (if (uuid? root-block-uuids-or-page-uuid)
+    :export-blocks-as-opml
+    (let [content
+          (if (uuid? root-block-uuids-or-page-uuid)
            ;; page
-           (common/get-page-content root-block-uuids-or-page-uuid)
-           (common/root-block-uuids->content repo root-block-uuids-or-page-uuid))
-         title (if (uuid? root-block-uuids-or-page-uuid)
-                 (:block/title (db/entity [:block/uuid root-block-uuids-or-page-uuid]))
-                 "untitled")
-         first-block (and (coll? root-block-uuids-or-page-uuid)
-                          (db/entity [:block/uuid (first root-block-uuids-or-page-uuid)]))
-         format (get first-block :block/format :markdown)]
-     (export-helper content format options :title title))))
+            (common/get-page-content root-block-uuids-or-page-uuid)
+            (common/root-block-uuids->content repo root-block-uuids-or-page-uuid))
+          title (if (uuid? root-block-uuids-or-page-uuid)
+                  (:block/title (db/entity [:block/uuid root-block-uuids-or-page-uuid]))
+                  "untitled")
+          first-block (and (coll? root-block-uuids-or-page-uuid)
+                           (db/entity [:block/uuid (first root-block-uuids-or-page-uuid)]))
+          format (get first-block :block/format :markdown)]
+      (export-helper content format options :title title))))
 
 (defn- export-files-as-opml
   "options see also `export-blocks-as-opml`"
@@ -456,7 +456,7 @@
    (fn [{:keys [path content title format]}]
      (when (and title (not (string/blank? content)))
        (util/profile (print-str :export-files-as-opml path)
-                     [path (export-helper content format options :title title)])))
+         [path (export-helper content format options :title title)])))
    files))
 
 (defn export-repo-as-opml!

src/main/frontend/handler/export/text.cljs

@@ -512,25 +512,25 @@
   {:pre [(or (coll? root-block-uuids-or-page-uuid)
              (uuid? root-block-uuids-or-page-uuid))]}
   (util/profile
-   :export-blocks-as-markdown
-   (try
-     (let [content
-           (cond
+    :export-blocks-as-markdown
+    (try
+      (let [content
+            (cond
              ;; page
-             (and (= 1 (count root-block-uuids-or-page-uuid))
-                  (ldb/page? (db/entity [:block/uuid (first root-block-uuids-or-page-uuid)])))
-             (common/get-page-content (first root-block-uuids-or-page-uuid))
-             (and (coll? root-block-uuids-or-page-uuid) (every? #(ldb/page? (db/entity [:block/uuid %])) root-block-uuids-or-page-uuid))
-             (->> (mapv (fn [id] (:block/title (db/entity [:block/uuid id]))) root-block-uuids-or-page-uuid)
-                  (string/join "\n"))
-             :else
-             (common/root-block-uuids->content repo root-block-uuids-or-page-uuid))
-           first-block (and (coll? root-block-uuids-or-page-uuid)
-                            (db/entity [:block/uuid (first root-block-uuids-or-page-uuid)]))
-           format (get first-block :block/format :markdown)]
-       (export-helper content format options))
-     (catch :default e
-       (js/console.error e)))))
+              (and (= 1 (count root-block-uuids-or-page-uuid))
+                   (ldb/page? (db/entity [:block/uuid (first root-block-uuids-or-page-uuid)])))
+              (common/get-page-content (first root-block-uuids-or-page-uuid))
+              (and (coll? root-block-uuids-or-page-uuid) (every? #(ldb/page? (db/entity [:block/uuid %])) root-block-uuids-or-page-uuid))
+              (->> (mapv (fn [id] (:block/title (db/entity [:block/uuid id]))) root-block-uuids-or-page-uuid)
+                   (string/join "\n"))
+              :else
+              (common/root-block-uuids->content repo root-block-uuids-or-page-uuid))
+            first-block (and (coll? root-block-uuids-or-page-uuid)
+                             (db/entity [:block/uuid (first root-block-uuids-or-page-uuid)]))
+            format (get first-block :block/format :markdown)]
+        (export-helper content format options))
+      (catch :default e
+        (js/console.error e)))))
 
 (defn export-files-as-markdown
   "options see also `export-blocks-as-markdown`"
@@ -538,7 +538,7 @@
   (mapv
    (fn [{:keys [path title content]}]
      (util/profile (print-str :export-files-as-markdown title)
-                   [(or path title) (export-helper content :markdown options)]))
+       [(or path title) (export-helper content :markdown options)]))
    files))
 
 (defn export-repo-as-markdown!

src/main/frontend/handler/file_based/page_property.cljs

@@ -75,8 +75,8 @@
                    :block/parent page-id
                    :block/page page-id
                    :block/title (if org?
-                                    (str "#+" (string/upper-case (name key)) ": " value)
-                                    (str (name key) ":: " value))
+                                  (str "#+" (string/upper-case (name key)) ": " value)
+                                  (str (name key) ":: " value))
                    :block/format format
                    :block/properties {key value}
                    :block/pre-block? true}

src/main/frontend/handler/plugin.cljs

@@ -891,7 +891,7 @@
 
                   (.on "themes-changed" (fn [^js themes]
                                           (swap! state/state assoc :plugin/installed-themes
-                                            (vec (mapcat (fn [[pid vs]] (mapv #(assoc % :pid pid) (bean/->clj vs))) (bean/->clj themes))))))
+                                                 (vec (mapcat (fn [[pid vs]] (mapv #(assoc % :pid pid) (bean/->clj vs))) (bean/->clj themes))))))
 
                   (.on "theme-selected" (fn [^js theme]
                                           (let [theme (bean/->clj theme)

src/main/frontend/handler/worker.cljs

@@ -1,5 +1,5 @@
 (ns frontend.handler.worker
-  "Handle messages received from the db worker"
+  "Handle messages received from the webworkers"
   (:require [cljs-bean.core :as bean]
             [frontend.handler.file-based.file :as file-handler]
             [frontend.handler.notification :as notification]
@@ -37,6 +37,9 @@
   (let [state data]
     (state/pub-event! [:rtc/sync-state state])))
 
+(defmethod handle :vector-search-sync-state [_ _worker data]
+  (state/pub-event! [:vector-search/sync-state data]))
+
 (defmethod handle :sync-db-changes [_ _worker data]
   (state/pub-event! [:db/sync-changes data]))
 
@@ -49,6 +52,9 @@
 (defmethod handle :capture-error [_ _worker data]
   (state/pub-event! [:capture-error data]))
 
+(defmethod handle :vector-search/load-model-progress [_ _ data]
+  (state/pub-event! [:vector-search/load-model-progress data]))
+
 (defmethod handle :backup-file [_ _worker data]
   (state/pub-event! [:graph/backup-file data]))
 

src/main/frontend/inference_worker/inference_worker.cljs

@@ -0,0 +1,72 @@
+(ns frontend.inference-worker.inference-worker
+  "Worker used for text embedding and vector-db"
+  (:require ["comlink" :as Comlink]
+            [frontend.inference-worker.text-embedding :as infer-worker.text-embedding]
+            [lambdaisland.glogi.console :as glogi-console]
+            [promesa.core :as p]
+            [shadow.cljs.modern :refer [defclass]]))
+
+#_{:clj-kondo/ignore [:unresolved-symbol]}
+(defclass InferenceWorker
+  (extends js/Object)
+  (constructor
+   [this]
+   (super))
+
+  Object
+  (init
+   [_this model-name]
+   (infer-worker.text-embedding/<init model-name))
+
+  (load-model
+   [_this model-name]
+   (infer-worker.text-embedding/<load-model model-name))
+
+  (available-embedding-models
+   [_]
+   (clj->js (keys infer-worker.text-embedding/available-embedding-models)))
+
+  (text-embedding
+   [_this text-coll]
+   (p/chain
+    (infer-worker.text-embedding/<text-embedding text-coll)
+    clj->js))
+
+  (text-embedding+store!
+   ;; return labels(js array)
+   [_this repo text-array labels replace-deleted?]
+   (p/chain
+    (js/Promise. (infer-worker.text-embedding/task--text-embedding&store! repo text-array labels replace-deleted?))
+    clj->js))
+
+  (delete-labels
+   [_this repo labels]
+   (infer-worker.text-embedding/delete-items repo labels))
+
+  (force-reset-index!
+   [_this repo]
+   (js/Promise. (infer-worker.text-embedding/task--force-reset-index! repo)))
+
+  (write-index!
+   [_this repo]
+   (js/Promise. (infer-worker.text-embedding/task--write-index! repo)))
+
+  (search
+   [_this repo query-string nums-neighbors]
+   (infer-worker.text-embedding/<search-knn repo query-string nums-neighbors))
+
+  (index-info
+   [_this repo]
+   (clj->js (infer-worker.text-embedding/index-info repo))))
+
+(defn init
+  [])
+
+(.addEventListener js/self "connect"
+                   (fn [^js e]
+                     (glogi-console/install!)
+                     (let [port (first (.-ports e))
+                           ^js obj #_{:clj-kondo/ignore [:unresolved-symbol]} (InferenceWorker.)]
+                       (reset! infer-worker.text-embedding/*port port)
+                       (.start port)
+                       (Comlink/expose obj port))))

src/main/frontend/inference_worker/state.cljs

@@ -0,0 +1,10 @@
+(ns frontend.inference-worker.state
+  "State hub for inference-worker")
+
+(defonce *hnswlib (atom nil))
+
+;;repo -> index
+(defonce *hnsw-index (atom {}))
+
+(defonce *extractor (atom nil))
+(defonce *model-name+config (atom nil))

src/main/frontend/inference_worker/text_embedding.cljs

@@ -0,0 +1,237 @@
+(ns frontend.inference-worker.text-embedding
+  "text embedding fns"
+  (:require ["@huggingface/transformers" :refer [pipeline]]
+            ["hnswlib-wasm" :refer [loadHnswlib]]
+            [clojure.data :as data]
+            [frontend.common.missionary :as c.m]
+            [frontend.inference-worker.state :as infer-worker.state]
+            [frontend.worker-common.util :as worker-util]
+            [lambdaisland.glogi :as log]
+            [logseq.common.config :as common-config]
+            [missionary.core :as m]
+            [promesa.core :as p]))
+
+(add-watch infer-worker.state/*hnsw-index :delete-obj-when-dissoc
+           (fn [_ _ o n]
+             (let [[old-only] (data/diff o n)]
+               (doseq [[repo ^js hnsw-index] old-only]
+                 (when hnsw-index
+                   (log/info :delete-hnsw-index repo)
+                   (.delete hnsw-index))))))
+
+(defonce *port (atom nil))
+
+(def ^:private embedding-opts #js{"pooling" "mean" "normalize" true})
+
+(def ^:private init-max-elems 100)
+
+(defn- split-into-chunks
+  [js-array chunk-size]
+  (let [length (alength js-array)
+        result (array)]
+    (loop [i 0]
+      (when (< i length)
+        (.push result (.slice js-array i (+ i chunk-size)))
+        (recur (+ i chunk-size))))
+    result))
+
+(defn- init-index!
+  [^js hnsw]
+  (.initIndex hnsw init-max-elems 16 200 100)
+  (.setEfSearch hnsw 32))
+
+(defn- ^js get-hnsw-index
+  [repo]
+  (or (@infer-worker.state/*hnsw-index repo)
+      (let [hnsw-ctor (.-HierarchicalNSW ^js @infer-worker.state/*hnswlib)
+            hnsw (new hnsw-ctor "cosine" (or (:dims (:hnsw-config (second @infer-worker.state/*model-name+config))) 384) "")
+            file-exists? (.checkFileExists (.-EmscriptenFileSystemManager ^js @infer-worker.state/*hnswlib) repo)]
+        (when file-exists?
+          (.readIndex hnsw repo init-max-elems)
+          (swap! infer-worker.state/*hnsw-index assoc repo hnsw)
+          hnsw))))
+
+(defn- ^js new-hnsw-index!
+  [repo]
+  (when (get-hnsw-index repo)
+    (swap! infer-worker.state/*hnsw-index dissoc repo))
+  (let [hnsw-ctor (.-HierarchicalNSW ^js @infer-worker.state/*hnswlib)
+        hnsw (new hnsw-ctor "cosine" (or (:dims (:hnsw-config (second @infer-worker.state/*model-name+config))) 384) "")]
+    (init-index! hnsw)
+    (swap! infer-worker.state/*hnsw-index assoc repo hnsw)
+    hnsw))
+
+(defn- model-loaded?
+  []
+  (and @infer-worker.state/*extractor
+       @infer-worker.state/*model-name+config))
+
+(defn <text-embedding
+  [text-array]
+  (assert (and (array? text-array) (every? string? text-array)))
+  (p/let [^js r (._call ^js @infer-worker.state/*extractor text-array embedding-opts)]
+    {:data (.-data r)
+     :type (.-type r)
+     :dims (.-dims r)
+     :size (.-size r)}))
+
+(defn- add-items
+  [^js hnsw data-coll labels replace-deleted?]
+  (let [max-elems (.getMaxElements hnsw)
+        current-count (.getCurrentCount hnsw)
+        add-count (count data-coll)]
+    (when (>= (+ add-count current-count) max-elems)
+      (let [new-size (+ current-count (max (* 2 add-count) current-count))]
+        (log/info :hnsw-resize {:from current-count :to new-size})
+        (.resizeIndex hnsw new-size)))
+    ;; (.addItems hnsw data-coll labels replace-deleted?)
+    (dorun
+     (mapcat
+      (fn [embedding label]
+        (assert (and embedding label) {:embedding embedding
+                                       :label label})
+        (.addPoint hnsw embedding label replace-deleted?))
+      data-coll
+      labels))))
+
+(defn delete-items
+  [repo labels]
+  (when-let [hnsw (get-hnsw-index repo)]
+    (.markDeleteItems hnsw (into-array labels))))
+
+(defn task--text-embedding&store!
+  "return labels(js-array)"
+  [repo text-array labels replace-deleted?]
+  (m/sp
+    (when (model-loaded?)
+      (let [hnsw (or (get-hnsw-index repo) (new-hnsw-index! repo))
+            {:keys [data _type dims _size]} (worker-util/profile :<text-embedding
+                                                                 (c.m/<? (<text-embedding text-array)))
+            data-coll (split-into-chunks data (last dims))
+            _ (assert (= (count text-array) (count data-coll)))]
+        (worker-util/profile (keyword "add-items" (str (alength data-coll)))
+                             (add-items hnsw data-coll labels replace-deleted?))))))
+
+(def ^:private write-index-wait-delays-flow
+  (m/ap
+    (loop [[delay-ms & others]
+           ;; 50ms + 100 * 100ms = ~10s
+           (cons 50 (repeat 100 100))]
+      (if delay-ms
+        (m/amb
+         (m/? (m/sleep delay-ms))
+         (recur others))
+        (m/amb :timeout)))))
+
+(defn- task--write-index!*
+  "NOTE: writeIndex return nil, but it should be a promise.
+  so we loop check fs-synced here, until synced=true"
+  [repo ^js hnsw]
+  (m/sp
+    (.writeIndex hnsw repo)
+    (let [^js fs (.-EmscriptenFileSystemManager ^js @infer-worker.state/*hnswlib)]
+      (m/?
+       (m/reduce
+        (fn [_ x]
+          (if-not (keyword-identical? :timeout x)
+            (when (.isSynced fs)
+              (reduced true))
+            (reduced false)))
+        write-index-wait-delays-flow)))))
+
+(defn task--force-reset-index!
+  "Remove all data in hnsw-index.
+  Return synced? (bool)"
+  [repo]
+  (m/sp
+    (when-let [hnsw (new-hnsw-index! repo)]
+      (when-not (zero? (.getCurrentCount hnsw))
+        (init-index! hnsw)
+        (m/? (task--write-index!* repo hnsw))))))
+
+(defn task--write-index!
+  [repo]
+  (m/sp
+    (when-let [hnsw (get-hnsw-index repo)]
+      (m/? (task--write-index!* repo hnsw)))))
+
+(defn- search-knn
+  [repo query-point num-neighbors]
+  (when-let [hnsw (get-hnsw-index repo)]
+    (.searchKnn hnsw query-point num-neighbors nil)))
+
+(defn <search-knn
+  "return labels"
+  [repo query-string num-neighbors]
+  (when (model-loaded?)
+    (p/let [query-embedding (<text-embedding #js[query-string])
+            query-point (:data query-embedding)]
+      (search-knn repo query-point num-neighbors))))
+
+(defn index-info
+  [repo]
+  (when-let [hnsw (get-hnsw-index repo)]
+    {:current-count (.getCurrentCount hnsw)
+     :max-elements (.getMaxElements hnsw)
+     :ef-search (.getEfSearch hnsw)
+     :num-dims (.getNumDimensions hnsw)}))
+
+(def available-embedding-models
+  {"Xenova/all-MiniLM-L6-v2" {:tf-config {:dtype "fp32"}
+                              :hnsw-config {:dims 384}}
+   ;; "onnx-community/Qwen3-Embedding-0.6B-ONNX" {:tf-config {:dtype "fp16"}
+   ;;                                             :hnsw-config {:dims 1024}}
+   })
+
+(defonce ^:private *load-model-progress (atom nil))
+
+(defn <load-model
+  [model-name]
+  (if (= model-name (first @infer-worker.state/*model-name+config))
+    true
+    (when-let [config (get available-embedding-models model-name)]
+      (p/let [extractor (pipeline "feature-extraction" model-name
+                                  (clj->js
+                                   (-> (:tf-config config)
+                                       (assoc "device" "webgpu")
+                                       (assoc "progress_callback" #(reset! *load-model-progress %)))))]
+        (reset! infer-worker.state/*extractor extractor)
+        (reset! infer-worker.state/*model-name+config [model-name config])
+        true))))
+
+(defn <init
+  [model-name]
+  (p/let [hnswlib (loadHnswlib)]
+    (reset! infer-worker.state/*hnswlib hnswlib)
+    (.setDebugLogs (.-EmscriptenFileSystemManager ^js @infer-worker.state/*hnswlib) true)
+    (log/info :loaded :hnswlib)
+    (when model-name
+      (<load-model model-name))))
+
+(when-not common-config/PUBLISHING
+  (c.m/run-background-task
+   ::push-load-model-progress
+   (m/reduce
+    (fn [_ v]
+      (when-let [port @*port]
+        (worker-util/post-message :vector-search/load-model-progress v {:port port})))
+    (c.m/throttle 500 (m/watch *load-model-progress)))))
+
+(comment
+  (def repo "repo-1")
+  (def hnsw (ensure-hnsw-index! repo))
+  (def text-coll-100
+    (apply concat (repeatedly
+                   10
+                   (fn []
+                     ["The universe is constantly expanding, revealing new mysteries every day."
+                      "She decided to take a walk in the park to clear her mind."
+                      "Black holes are among the most fascinating and enigmatic objects in the cosmos."
+                      "The cat curled up on the windowsill, basking in the afternoon sun."
+                      "Scientists believe dark matter makes up a significant portion of the universe."
+                      "He practiced the piano diligently, hoping to master the piece by next week."
+                      "The stars twinkled brightly in the clear night sky, each one a distant sun."
+                      "They laughed together, sharing stories from their childhood."
+                      "The Milky Way is just one of billions of galaxies in the vast universe."
+                      "She opened the book and began reading, losing herself in the stor"]))))
+  (<text-embedding&store! repo text-coll-100 nil))

src/main/frontend/log.cljs

@@ -1,8 +1,8 @@
 (ns frontend.log
   "System-component-like ns that encapsulates logging functionality"
-  (:require [lambdaisland.glogi :as log]
-            [lambdaisland.glogi.console :as glogi-console]
-            [frontend.config :as config]))
+  (:require [frontend.config :as config]
+            [lambdaisland.glogi :as log]
+            [lambdaisland.glogi.console :as glogi-console]))
 
 ;; TODO: Move code below into a fn to behave like a system component
 ;; instead of having no control over its behavior at require time

src/main/frontend/mixins.cljs

@@ -9,7 +9,8 @@
 (defn detach
   "Detach all event listeners."
   [state]
-  (some-> state ::event-handler .removeAll))
+  (when-let [^EventHandler handler (some-> state ::event-handler)]
+    (.removeAll handler)))
 
 (defn listen
   "Register an event `handler` for events of `type` on `target`."