feat(full-text-search): support both ascii and non-ascii chars

src/main/frontend/search.cljs

@@ -14,7 +14,8 @@
             ["fuzzysort" :as fuzzy]
             ["flexsearch" :as flexsearch]
             [medley.core :as medley]
-            [promesa.core :as p]))
+            [promesa.core :as p]
+            ["/frontend/utils" :as utils]))
 
 (def fuzzy-go (gobj/get fuzzy "go"))
 (defonce prepare (gobj/get fuzzy "prepare"))
@@ -47,7 +48,9 @@
           indice (flexsearch.
                   (clj->js
                    {:encode "icase"
-                    :tokenize "full"
+                    ;; TODO: forward with multi-languages support
+                    ;; :tokenize "forward"
+                    :tokenize utils/searchTokenize
                     :doc {:id "id"
                           :field ["uuid" "content"]}
                     :async true}))]

src/main/frontend/utils.js

@@ -203,4 +203,8 @@ export const win32 = path => {
 
   // UNC paths are always absolute
   return Boolean(result[2] || isUnc);
-}
+};
+
+export const searchTokenize = str => {
+  return str.split(/\W+/).concat(str.replace(/[\x00-\x7F]/g, '').split('')).filter(e => !!e);
+};