wip: async custom query

src/main/frontend/db/async/util.cljs

@@ -1,12 +1,13 @@
 (ns frontend.db.async.util
   "Async util helper"
-  (:require [frontend.persist-db.browser :as db-browser]
-            [cljs-bean.core :as bean]
-            [promesa.core :as p]))
+  (:require [frontend.state :as state]
+            [promesa.core :as p]
+            [clojure.edn :as edn]))
 
 (defn <q
   [graph & inputs]
   (assert (not-any? fn? inputs) "Async query inputs can't include fns because fn can't be serialized")
-  (when-let [sqlite @db-browser/*worker]
+  (when-let [sqlite @state/*db-worker]
     (p/let [result (.q sqlite graph (pr-str inputs))]
-      (bean/->clj result))))
+      (when result
+        (edn/read-string result)))))

src/main/frontend/db/model.cljs

@@ -336,25 +336,8 @@ independent of format as format specific heading characters are stripped"
     (->
      (react/q repo [:frontend.worker.react/block id]
               {:query-fn (fn [_]
-                           (let [e (db-utils/entity id)
-                                 children (map :db/id (sort-by-left (:block/_parent e) e))]
-                             [e {:name (:block/name e)
-                                 :original-name (:block/original-name e)
-                                 :link (:block/link e)
-                                 :namespace (:block/namespace e)
-                                 :types (:block/type e)
-                                 :schema (:block/schema e)
-                                 :content (:block/content e)
-                                 :marker (:block/marker e)
-                                 :priority (:block/priority e)
-                                 :properties (:block/properties e)
-                                 :properties-values (:block/properties-text-values e)
-                                 :alias (:block/alias e)
-                                 :tags (:block/tags e)
-                                 :children children
-                                 :collapsed? (:block/collapsed? e)
-                                 :collapsed-properties (:block/collapsed-properties e)
-                                 :refs-count (count (:block/_refs e))}]))}
+                           (let [e (db-utils/entity id)]
+                             [e (:block/tx-id e)]))}
               nil)
      react
      first)))

src/main/frontend/db/react.cljs

@@ -4,13 +4,14 @@
   It'll be great if we can find an automatically resolving and performant
   solution.
   "
-  (:require [datascript.core :as d]
-            [frontend.date :as date]
+  (:require [frontend.date :as date]
             [frontend.db.conn :as conn]
             [frontend.db.utils :as db-utils]
             [frontend.state :as state]
             [frontend.util :as util :refer [react]]
-            [clojure.core.async :as async]))
+            [clojure.core.async :as async]
+            [frontend.db.async.util :as db-async-util]
+            [promesa.core :as p]))
 
 ;; Query atom of map of Key ([repo q inputs]) -> atom
 ;; TODO: replace with LRUCache, only keep the latest 20 or 50 items?
@@ -43,15 +44,13 @@
   (reset! query-state {}))
 
 (defn add-q!
-  [k query time inputs result-atom transform-fn query-fn inputs-fn]
-  (let [time' (int (util/safe-parse-float time))] ;; for robustness. `time` should already be float
-    (swap! query-state assoc k {:query query
-                                :query-time time'
-                                :inputs inputs
-                                :result result-atom
-                                :transform-fn transform-fn
-                                :query-fn query-fn
-                                :inputs-fn inputs-fn}))
+  [k query inputs result-atom transform-fn query-fn inputs-fn]
+  (swap! query-state assoc k {:query query
+                              :inputs inputs
+                              :result result-atom
+                              :transform-fn transform-fn
+                              :query-fn query-fn
+                              :inputs-fn inputs-fn})
   result-atom)
 
 (defn remove-q!
@@ -85,16 +84,36 @@
   (when-let [result (get @query-state k)]
     (when (satisfies? IWithMeta @(:result result))
       (set! (.-state (:result result))
-           (with-meta @(:result result) {:query-time (:query-time result)})))
+            @(:result result)))
     (:result result)))
 
+(defn- <q-aux
+  [repo db query-fn inputs-fn k query inputs]
+  (let [kv? (and (vector? k) (= :kv (first k)))]
+    (when (or query-fn query kv?)
+      (cond
+        query-fn
+        (query-fn db nil)
+
+        kv?
+        (db-utils/entity db (last k))
+
+        inputs-fn
+        (let [inputs (inputs-fn)]
+          (apply db-async-util/<q repo (cons query inputs)))
+
+        (seq inputs)
+        (apply db-async-util/<q repo (cons query inputs))
+
+        :else
+        (apply db-async-util/<q repo [query])))))
+
 (defn q
   [repo k {:keys [use-cache? transform-fn query-fn inputs-fn disable-reactive?]
            :or {use-cache? true
                 transform-fn identity}} query & inputs]
   ;; {:pre [(s/valid? :frontend.worker.react/block k)]}
-  (let [kv? (and (vector? k) (= :kv (first k)))
-        origin-key k
+  (let [origin-key k
         k (vec (cons repo k))]
     (when-let [db (conn/get-db repo)]
       (let [result-atom (get-query-cached-result k)]
@@ -104,30 +123,20 @@
           (swap! queries conj origin-key))
         (if (and use-cache? result-atom)
           result-atom
-          (let [{:keys [result time]} (util/with-time
-                                        (-> (cond
-                                              query-fn
-                                              (query-fn db nil)
-
-                                              inputs-fn
-                                              (let [inputs (inputs-fn)]
-                                                (apply d/q query db inputs))
-
-                                              kv?
-                                              (db-utils/entity db (last k))
-
-                                              (seq inputs)
-                                              (apply d/q query db inputs)
-
-                                              :else
-                                              (d/q query db))
-                                            transform-fn))
-                result-atom (or result-atom (atom nil))]
-            ;; Don't notify watches now
-            (set! (.-state result-atom) result)
-            (if disable-reactive?
-              result-atom
-              (add-q! k query time inputs result-atom transform-fn query-fn inputs-fn))))))))
+          (let [result-atom (or result-atom (atom nil))
+                p-or-value (<q-aux repo db query-fn inputs-fn k query inputs)]
+            (when-not disable-reactive?
+              (add-q! k query inputs result-atom transform-fn query-fn inputs-fn))
+            (if (p/promise? p-or-value)
+              (do
+                (p/let [result p-or-value
+                        result' (transform-fn result)]
+                  (reset! result-atom result'))
+                result-atom)
+              (let [result' (transform-fn p-or-value)]
+                ;; Don't notify watches now
+                (set! (.-state result-atom) result')
+                result-atom))))))))
 
 (defn get-current-page
   []
@@ -146,35 +155,12 @@
         (db-utils/entity [:block/name page-name])))))
 
 (defn- execute-query!
-  [graph db k {:keys [query _query-time inputs transform-fn query-fn inputs-fn result]}
-   {:keys [_skip-query-time-check?]}]
-  ;; FIXME:
-  (when true
-      ;; (or skip-query-time-check?
-      ;;       (<= (or query-time 0) 80))
-    (let [new-result (->
-                      (cond
-                        query-fn
-                        (let [result (query-fn db result)]
-                          (if (coll? result)
-                            (doall result)
-                            result))
-
-                        inputs-fn
-                        (let [inputs (inputs-fn)]
-                          (apply d/q query db inputs))
-
-                        (keyword? query)
-                        (db-utils/get-key-value graph query)
-
-                        (seq inputs)
-                        (apply d/q query db inputs)
-
-                        :else
-                        (d/q query db))
-                      transform-fn)]
-      (when-not (= new-result result)
-       (set-new-result! k new-result)))))
+  [graph db k {:keys [query inputs transform-fn query-fn inputs-fn result]
+               :or {transform-fn identity}}]
+  (p/let [p-or-value (<q-aux graph db query-fn inputs-fn k query inputs)
+          result' (transform-fn p-or-value)]
+    (when-not (= result' result)
+      (set-new-result! k result'))))
 
 (defn refresh-affected-queries!
   [repo-url affected-keys]
@@ -198,7 +184,7 @@
                {:keys [custom-query?]} (state/edit-in-query-or-refs-component)]
            (when (or query query-fn)
              (try
-               (let [f #(execute-query! repo-url db (vec (cons repo-url k)) cache {:skip-query-time-check? custom-query?})]
+               (let [f #(execute-query! repo-url db (vec (cons repo-url k)) cache)]
                        ;; Detects whether user is editing in a custom query, if so, execute the query immediately
                  (if (and custom? (not custom-query?))
                    (async/put! (state/get-reactive-custom-queries-chan) [f query])

src/main/frontend/db_worker.cljs

@@ -283,7 +283,7 @@
      (when-let [conn (worker-state/get-datascript-conn repo)]
        (let [inputs (edn/read-string inputs-str)]
          (let [result (apply d/q (first inputs) @conn (rest inputs))]
-           (bean/->js result)))))
+           (pr-str result)))))
 
   (get-block-and-children
    [_this repo name children?]

src/main/frontend/persist_db/browser.cljs

@@ -15,7 +15,7 @@
             [logseq.db :as ldb]
             [frontend.date :as date]))
 
-(defonce *worker (atom nil))
+(defonce *worker state/*db-worker)
 
 (defn- ask-persist-permission!
   []

src/main/frontend/state.cljs

@@ -26,6 +26,8 @@
 
 (defonce *editor-editing-ref (atom nil))
 
+(defonce *db-worker (atom nil))
+
 ;; Stores main application state
 (defonce ^:large-vars/data-var state
   (let [document-mode? (or (storage/get :document/mode?) false)