refactor: mv common worker/frontend ns to frontend/common

Move common core.async, date and search utils to frontend/common.
Part of LOG-3171

.clj-kondo/config.edn

@@ -54,6 +54,10 @@
              frontend.components.query.result query-result
              frontend.components.class class-component
              frontend.components.property property-component
+             frontend.common.date common-date
+             frontend.common.missionary-util c.m
+             frontend.common.schema-register sr
+             frontend.common.search-fuzzy fuzzy
              frontend.config config
              frontend.date date
              frontend.db db

deps/common/.carve/config.edn

@@ -1,7 +1,5 @@
 {:paths ["src"]
  :api-namespaces [logseq.common.path
-                  logseq.common.fractional-index
-                  logseq.common.missionary-util
                   logseq.common.uuid
                   logseq.common.util.page-ref
                   logseq.common.util.block-ref

src/main/frontend/common/async_util.cljc

@@ -0,0 +1,90 @@
+(ns frontend.common.async-util
+  "Some cljs.core.async relate macros and fns, used in worker and frontend
+   namespaces. See also: https://gist.github.com/vvvvalvalval/f1250cec76d3719a8343"
+  #?(:cljs (:require [promesa.core :as p]
+                     [logseq.common.util :as common-util]
+                     [clojure.core.async :as async]
+                     [cljs.core.async.impl.channels :refer [ManyToManyChannel]])))
+
+#?(:cljs
+   (defn throw-err
+     [v]
+     (if (instance? ExceptionInfo v) (throw v) v)))
+
+(defmacro <?
+  [port]
+  `(throw-err (cljs.core.async/<! ~port)))
+
+#?(:cljs
+   (defn c->p
+     "Converts a Core.async channel to a Promise"
+     [chan]
+     (let [d (p/deferred)]
+       (if chan
+         (async/go
+           (let [result (async/<! chan)]
+             (if (instance? ExceptionInfo result)
+               (p/reject! d result)
+               (p/resolve! d result))))
+         (p/resolve! d nil))
+       d)))
+
+#?(:cljs
+   (defn drain-chan
+     "drop all stuffs in CH, and return all of them"
+     [ch]
+     (->> (repeatedly #(async/poll! ch))
+          (take-while identity))))
+
+#?(:cljs
+   (defn <ratelimit
+       "return a channel CH,
+  ratelimit flush items in in-ch every max-duration(ms),
+  opts:
+  - :filter-fn filter item before putting items into returned CH, (filter-fn item)
+               will poll it when its return value is channel,
+  - :flush-fn exec flush-fn when time to flush, (flush-fn item-coll)
+  - :stop-ch stop go-loop when stop-ch closed
+  - :distinct-key-fn distinct coll when put into CH
+  - :chan-buffer buffer of return CH, default use (async/chan 1000)
+  - :flush-now-ch flush the content in the queue immediately
+  - :refresh-timeout-ch refresh (timeout max-duration)"
+       [in-ch max-duration & {:keys [filter-fn flush-fn stop-ch distinct-key-fn chan-buffer flush-now-ch refresh-timeout-ch]}]
+       (let [ch (if chan-buffer (async/chan chan-buffer) (async/chan 1000))
+             stop-ch* (or stop-ch (async/chan))
+             flush-now-ch* (or flush-now-ch (async/chan))
+             refresh-timeout-ch* (or refresh-timeout-ch (async/chan))]
+         (async/go-loop [timeout-ch (async/timeout max-duration) coll []]
+           (let [{:keys [refresh-timeout timeout e stop flush-now]}
+                 (async/alt! refresh-timeout-ch* {:refresh-timeout true}
+                             timeout-ch {:timeout true}
+                             in-ch ([e] {:e e})
+                             stop-ch* {:stop true}
+                             flush-now-ch* {:flush-now true})]
+             (cond
+               refresh-timeout
+               (recur (async/timeout max-duration) coll)
+
+               (or flush-now timeout)
+               (do (async/onto-chan! ch coll false)
+                   (flush-fn coll)
+                   (drain-chan flush-now-ch*)
+                   (recur (async/timeout max-duration) []))
+
+               (some? e)
+               (let [filter-v (filter-fn e)
+                     filter-v* (if (instance? ManyToManyChannel filter-v)
+                                 (async/<! filter-v)
+                                 filter-v)]
+                 (if filter-v*
+                   (recur timeout-ch (cond->> (conj coll e)
+                                       distinct-key-fn (common-util/distinct-by distinct-key-fn)
+                                       true vec))
+                   (recur timeout-ch coll)))
+
+               (or stop
+                 ;; got nil from in-ch, means in-ch is closed
+                 ;; so we stop the whole go-loop
+                   (nil? e))
+               (async/close! ch))))
+         ch)))

src/main/frontend/worker/date.cljs → src/main/frontend/common/date.cljs

@@ -1,5 +1,6 @@
-(ns frontend.worker.date
-  "Date related fns that used by worker"
+(ns frontend.common.date
+  "Date related fns shared by worker and frontend namespaces. Eventually some
+   of this should go to logseq.common.util.date-time"
   (:require [cljs-time.format :as tf]
             [logseq.common.util :as common-util]))
 

src/main/frontend/common/search_fuzzy.cljs

@@ -2,7 +2,7 @@
   "fuzzy search. Used by frontend and worker namespaces"
   (:require [clojure.string :as string]
             [cljs-bean.core :as bean]
-            [frontend.worker.util :as worker-util]))
+            ["remove-accents" :as removeAccents]))
 
 (def MAX-STRING-LENGTH 1000.0)
 
@@ -56,15 +56,24 @@
                        (dec idx)
                        (- score 0.1)))))))
 
+(defn search-normalize
+  "Normalize string for searching (loose)"
+  [s remove-accents?]
+  (when s
+    (let [normalize-str (.normalize (string/lower-case s) "NFKC")]
+      (if remove-accents?
+        (removeAccents normalize-str)
+        normalize-str))))
+
 (defn fuzzy-search
   [data query & {:keys [limit extract-fn]
                  :or {limit 20}}]
-  (let [query (worker-util/search-normalize query true)]
+  (let [query (search-normalize query true)]
     (->> (take limit
                (sort-by :score (comp - compare)
                         (filter #(< 0 (:score %))
                                 (for [item data]
                                   (let [s (str (if extract-fn (extract-fn item) item))]
                                     {:data item
-                                     :score (score query (worker-util/search-normalize s true))})))))
+                                     :score (score query (search-normalize s true))})))))
          (map :data))))

src/main/frontend/components/repo.cljs

@@ -24,7 +24,7 @@
             [logseq.shui.ui :as shui]
             [frontend.handler.db-based.rtc :as rtc-handler]
             [frontend.handler.graph :as graph]
-            [frontend.worker.async-util :as async-util]))
+            [frontend.common.async-util :as async-util]))
 
 (rum/defc normalized-graph-label
   [{:keys [url remote? GraphName GraphUUID] :as graph} on-click]

src/main/frontend/date.cljs

@@ -10,7 +10,7 @@
             [logseq.common.util.date-time :as date-time-util]
             [goog.object :as gobj]
             [lambdaisland.glogi :as log]
-            [frontend.worker.date :as worker-date]))
+            [frontend.common.date :as common-date]))
 
 (defn nld-parse
   [s]
@@ -21,7 +21,7 @@
 
 (defn journal-title-formatters
   []
-  (worker-date/journal-title-formatters (state/get-date-formatter)))
+  (common-date/journal-title-formatters (state/get-date-formatter)))
 
 (defn get-date-time-string
   ([]
@@ -111,15 +111,15 @@
 
 (defn normalize-date
   [s]
-  (worker-date/normalize-date s (state/get-date-formatter)))
+  (common-date/normalize-date s (state/get-date-formatter)))
 
 (defn normalize-journal-title
   [title]
-  (worker-date/normalize-journal-title title (state/get-date-formatter)))
+  (common-date/normalize-journal-title title (state/get-date-formatter)))
 
 (defn valid-journal-title?
   [title]
-  (worker-date/valid-journal-title? title (state/get-date-formatter)))
+  (common-date/valid-journal-title? title (state/get-date-formatter)))
 
 (defn journal-title->
   ([journal-title then-fn]
@@ -144,7 +144,7 @@
   [journal-title]
   (journal-title-> journal-title #(tc/to-long %)))
 
-(def default-journal-filename-formatter worker-date/default-journal-filename-formatter)
+(def default-journal-filename-formatter common-date/default-journal-filename-formatter)
 
 (defn journal-title->default
   "Journal title to filename format"

src/main/frontend/db/transact.cljs

@@ -3,7 +3,7 @@
   (:require [clojure.core.async :as async]
             [clojure.core.async.interop :refer [p->c]]
             [promesa.core :as p]
-            [frontend.worker.async-util :include-macros true :refer [<?]]))
+            [frontend.common.async-util :include-macros true :refer [<?]]))
 
 (defonce *request-id (atom 0))
 (defonce requests (async/chan 1000))

src/main/frontend/fs/sync.cljs

@@ -15,6 +15,7 @@
             [clojure.set :as set]
             [clojure.string :as string]
             [electron.ipc :as ipc]
+            [frontend.common.async-util :as async-util]
             [frontend.config :as config]
             [frontend.context.i18n :refer [t]]
             [frontend.db :as db]
@@ -2680,7 +2681,7 @@
 
   (<ratelimit [this from-chan]
     (let [<fast-filter-e-fn (.filter-file-change-events-fn this)]
-      (util/<ratelimit
+      (async-util/<ratelimit
        from-chan rate
        :filter-fn
        (fn [e]

src/main/frontend/handler/search.cljs

@@ -2,6 +2,7 @@
   "Provides util handler fns for search"
   (:require [clojure.string :as string]
             [frontend.config :as config]
+            [frontend.common.search-fuzzy :as fuzzy]
             [frontend.db :as db]
             [frontend.handler.notification :as notification]
             [frontend.search :as search]
@@ -123,8 +124,8 @@
     content
     (when (and content q)
       (let [q-words (string/split q #" ")
-            lc-content (util/search-normalize content (state/enable-search-remove-accents?))
-            lc-q (util/search-normalize q (state/enable-search-remove-accents?))]
+            lc-content (fuzzy/search-normalize content (state/enable-search-remove-accents?))
+            lc-q (fuzzy/search-normalize q (state/enable-search-remove-accents?))]
         (if (and (string/includes? lc-content lc-q)
                  (not (util/safe-re-find #" " q)))
           (let [i (string/index-of lc-content lc-q)
@@ -140,8 +141,8 @@
                                 result []]
                            (if (and (seq words) content)
                              (let [word (first words)
-                                   lc-word (util/search-normalize word (state/enable-search-remove-accents?))
-                                   lc-content (util/search-normalize content (state/enable-search-remove-accents?))]
+                                   lc-word (fuzzy/search-normalize word (state/enable-search-remove-accents?))
+                                   lc-content (fuzzy/search-normalize content (state/enable-search-remove-accents?))]
                                (if-let [i (string/index-of lc-content lc-word)]
                                  (recur (rest words)
                                         (subs content (+ i (count word)))

src/main/frontend/search.cljs

@@ -25,7 +25,7 @@
 (defn block-search
   [repo q option]
   (when-let [engine (get-engine repo)]
-    (let [q (util/search-normalize q (state/enable-search-remove-accents?))]
+    (let [q (fuzzy/search-normalize q (state/enable-search-remove-accents?))]
       (when-not (string/blank? q)
         (protocol/query engine q option)))))
 

src/main/frontend/util.cljc

@@ -27,7 +27,6 @@
             [rum.core :as rum]
             [clojure.core.async :as async]
             [frontend.pubsub :as pubsub]
-            [frontend.worker.util :as worker-util]
             [datascript.impl.entity :as de]))
   #?(:cljs (:import [goog.async Debouncer]))
   (:require
@@ -1006,9 +1005,6 @@
      [string]
      (some-> string str (js/encodeURIComponent) (.replace "+" "%20"))))
 
-#?(:cljs
-   (def search-normalize worker-util/search-normalize))
-
 #?(:cljs
    (def page-name-sanity-lc
      "Delegate to common-util to loosely couple app usages to graph-parser"
@@ -1126,9 +1122,6 @@
      (->> (repeatedly #(async/poll! ch))
           (take-while identity))))
 
-#?(:cljs
-   (def <ratelimit worker-util/<ratelimit))
-
 #?(:cljs
    (defn trace!
      []

src/main/frontend/worker/async_util.cljc

@@ -1,28 +0,0 @@
-(ns frontend.worker.async-util
-  "Some cljs.core.async relate macros and fns.
-  see also: https://gist.github.com/vvvvalvalval/f1250cec76d3719a8343"
-  #?(:cljs (:require [promesa.core :as p]
-                     [clojure.core.async :as async])))
-
-#?(:cljs
-   (defn throw-err
-     [v]
-     (if (instance? ExceptionInfo v) (throw v) v)))
-
-(defmacro <?
-  [port]
-  `(throw-err (cljs.core.async/<! ~port)))
-
-#?(:cljs
-   (defn c->p
-     "Converts a Core.async channel to a Promise"
-     [chan]
-     (let [d (p/deferred)]
-       (if chan
-         (async/go
-           (let [result (async/<! chan)]
-             (if (instance? ExceptionInfo result)
-               (p/reject! d result)
-               (p/resolve! d result))))
-         (p/resolve! d nil))
-       d)))

src/main/frontend/worker/file.cljs

@@ -9,6 +9,7 @@
             [cljs-time.core :as t]
             [cljs-time.coerce :as tc]
             [frontend.worker.util :as worker-util]
+            [frontend.common.async-util :as async-util]
             [datascript.core :as d]
             [logseq.db :as ldb]
             [malli.core :as m]
@@ -116,7 +117,7 @@
 
 (defn <ratelimit-file-writes!
   []
-  (worker-util/<ratelimit file-writes-chan batch-write-interval
+  (async-util/<ratelimit file-writes-chan batch-write-interval
                           :filter-fn (fn [_] true)
                           :flush-fn
                           (fn [col]

src/main/frontend/worker/file/core.cljs

@@ -6,7 +6,7 @@
             [logseq.common.path :as path]
             [datascript.core :as d]
             [logseq.db :as ldb]
-            [frontend.worker.date :as worker-date]
+            [frontend.common.date :as common-date]
             [frontend.worker.util :as worker-util]
             [logseq.db.sqlite.util :as sqlite-util]))
 
@@ -136,11 +136,11 @@
           title (string/capitalize (:block/name page-block))
           whiteboard-page? (ldb/whiteboard? page-block)
           format (if whiteboard-page? "edn" format)
-          journal-page? (worker-date/valid-journal-title? title date-formatter)
-          journal-title (worker-date/normalize-journal-title title date-formatter)
+          journal-page? (common-date/valid-journal-title? title date-formatter)
+          journal-title (common-date/normalize-journal-title title date-formatter)
           journal-page? (and journal-page? (not (string/blank? journal-title)))
           filename (if journal-page?
-                     (worker-date/date->file-name journal-title (:journal-file-name-format context))
+                     (common-date/date->file-name journal-title (:journal-file-name-format context))
                      (-> (or (:block/title page-block) (:block/name page-block))
                          wfu/file-name-sanity))
           sub-dir (cond

src/main/frontend/worker/handler/page/file_based/page.cljs

@@ -9,7 +9,7 @@
             [logseq.graph-parser.text :as text]
             [logseq.common.util :as common-util]
             [logseq.common.config :as common-config]
-            [frontend.worker.date :as date]
+            [frontend.common.date :as common-date]
             [logseq.db.frontend.order :as db-order]))
 
 (defn- file-based-properties-block
@@ -74,7 +74,7 @@
                            :as options}]
   (let [date-formatter (common-config/get-date-formatter config)
         split-namespace? (not (or (string/starts-with? title "hls__")
-                                  (date/valid-journal-title? date-formatter title)))
+                                  (common-date/valid-journal-title? date-formatter title)))
         [title page-name] (get-title-and-pagename title)]
     (when-not (ldb/get-page @conn page-name)
       (let [pages    (if split-namespace?

src/main/frontend/worker/search.cljs

@@ -7,7 +7,6 @@
             [goog.object :as gobj]
             [datascript.core :as d]
             [frontend.common.search-fuzzy :as fuzzy]
-            [frontend.worker.util :as worker-util]
             [logseq.db.sqlite.util :as sqlite-util]
             [logseq.common.util :as common-util]
             [logseq.db :as ldb]))
@@ -172,8 +171,8 @@ DROP TRIGGER IF EXISTS blocks_au;
           (if (seq coll')
             (rest coll')
             (reduced false))))
-      (seq (worker-util/search-normalize match true))
-      (seq (worker-util/search-normalize q true))))))
+      (seq (fuzzy/search-normalize match true))
+      (seq (fuzzy/search-normalize q true))))))
 
 (defn- page-or-object?
   [entity]
@@ -194,7 +193,7 @@ DROP TRIGGER IF EXISTS blocks_au;
 (defn- sanitize
   [content]
   (some-> content
-          (worker-util/search-normalize true)))
+          (fuzzy/search-normalize true)))
 
 (defn block->index
   "Convert a block to the index for searching"
@@ -238,7 +237,7 @@ DROP TRIGGER IF EXISTS blocks_au;
   [repo db q {:keys [limit]
               :or {limit 100}}]
   (when repo
-    (let [q (worker-util/search-normalize q true)
+    (let [q (fuzzy/search-normalize q true)
           q (fuzzy/clean-str q)
           q (if (= \# (first q)) (subs q 1) q)]
       (when-not (string/blank? q)

src/main/frontend/worker/util.cljc

@@ -2,14 +2,10 @@
   "Worker utils"
   #?(:cljs (:require-macros [frontend.worker.util]))
   #?(:cljs (:refer-clojure :exclude [format]))
-  #?(:cljs (:require ["remove-accents" :as removeAccents]
-                     [cljs.core.async.impl.channels :refer [ManyToManyChannel]]
-                     [clojure.core.async :as async]
-                     [clojure.string :as string]
+  #?(:cljs (:require [clojure.string :as string]
                      [goog.crypt :as crypt]
                      [goog.crypt.Hmac]
                      [goog.crypt.Sha256]
-                     [logseq.common.util :as common-util]
                      [logseq.db :as ldb]
                      [logseq.db.sqlite.common-db :as sqlite-common-db])))
 
@@ -27,73 +23,6 @@
 
 #?(:cljs
    (do
-     (defn search-normalize
-       "Normalize string for searching (loose)"
-       [s remove-accents?]
-       (when s
-         (let [normalize-str (.normalize (string/lower-case s) "NFKC")]
-           (if remove-accents?
-             (removeAccents normalize-str)
-             normalize-str))))
-
-     (defn drain-chan
-       "drop all stuffs in CH, and return all of them"
-       [ch]
-       (->> (repeatedly #(async/poll! ch))
-            (take-while identity)))
-
-     (defn <ratelimit
-       "return a channel CH,
-  ratelimit flush items in in-ch every max-duration(ms),
-  opts:
-  - :filter-fn filter item before putting items into returned CH, (filter-fn item)
-               will poll it when its return value is channel,
-  - :flush-fn exec flush-fn when time to flush, (flush-fn item-coll)
-  - :stop-ch stop go-loop when stop-ch closed
-  - :distinct-key-fn distinct coll when put into CH
-  - :chan-buffer buffer of return CH, default use (async/chan 1000)
-  - :flush-now-ch flush the content in the queue immediately
-  - :refresh-timeout-ch refresh (timeout max-duration)"
-       [in-ch max-duration & {:keys [filter-fn flush-fn stop-ch distinct-key-fn chan-buffer flush-now-ch refresh-timeout-ch]}]
-       (let [ch (if chan-buffer (async/chan chan-buffer) (async/chan 1000))
-             stop-ch* (or stop-ch (async/chan))
-             flush-now-ch* (or flush-now-ch (async/chan))
-             refresh-timeout-ch* (or refresh-timeout-ch (async/chan))]
-         (async/go-loop [timeout-ch (async/timeout max-duration) coll []]
-           (let [{:keys [refresh-timeout timeout e stop flush-now]}
-                 (async/alt! refresh-timeout-ch* {:refresh-timeout true}
-                             timeout-ch {:timeout true}
-                             in-ch ([e] {:e e})
-                             stop-ch* {:stop true}
-                             flush-now-ch* {:flush-now true})]
-             (cond
-               refresh-timeout
-               (recur (async/timeout max-duration) coll)
-
-               (or flush-now timeout)
-               (do (async/onto-chan! ch coll false)
-                   (flush-fn coll)
-                   (drain-chan flush-now-ch*)
-                   (recur (async/timeout max-duration) []))
-
-               (some? e)
-               (let [filter-v (filter-fn e)
-                     filter-v* (if (instance? ManyToManyChannel filter-v)
-                                 (async/<! filter-v)
-                                 filter-v)]
-                 (if filter-v*
-                   (recur timeout-ch (cond->> (conj coll e)
-                                       distinct-key-fn (common-util/distinct-by distinct-key-fn)
-                                       true vec))
-                   (recur timeout-ch coll)))
-
-               (or stop
-                 ;; got nil from in-ch, means in-ch is closed
-                 ;; so we stop the whole go-loop
-                   (nil? e))
-               (async/close! ch))))
-         ch))
-
      (defn post-message
        [type data]
        (when (exists? js/self)