Merge branch 'feat/db' into feat/capacitor-new

clj-e2e/src/logseq/e2e/rtc.clj

@@ -19,7 +19,7 @@
          tx# (max local-tx# remote-tx#)]
      ~@body
      (loop [i# 5]
-       (when (zero? i#) (throw (ex-info "wait-tx-updated failed" {:data m#})))
+       (when (zero? i#) (throw (ex-info "wait-tx-updated failed" {:old m# :new (get-rtc-tx)})))
        (util/wait-timeout 1000)
        (let [new-m# (get-rtc-tx)
              new-local-tx# (or (:local-tx new-m#) 0)

clj-e2e/test/logseq/e2e/custom_report.clj

@@ -1,5 +1,6 @@
 (ns logseq.e2e.custom-report
-  (:require [clojure.stacktrace :as stack]
+  (:require [clojure.pprint :as pp]
+            [clojure.stacktrace :as stack]
             [clojure.string :as string]
             [clojure.test :as t]
             [logseq.e2e.playwright-page :as pw-page])
@@ -10,6 +11,8 @@
   record all playwright contexts in this dynamic var"
   nil)
 
+(def ^:dynamic *pw-page->console-logs* nil)
+
 (defn screenshot
   [page test-name]
   (println :screenshot test-name)
@@ -37,7 +40,13 @@
   ;; screenshot for all pw pages when :error
   (when-let [all-contexts (seq *pw-contexts*)]
     (doseq [page (mapcat pw-page/get-pages all-contexts)]
-      (screenshot page (string/join "-" (map (comp str :name meta) t/*testing-vars*))))))
+      (screenshot page (string/join "-" (map (comp str :name meta) t/*testing-vars*)))))
+
+  ;; dump console logs
+  (when-let [pw-page->console-logs (some-> *pw-page->console-logs* deref)]
+    (doseq [[_pw-page logs] pw-page->console-logs]
+      (spit (format "e2e-dump/console-logs-%s.txt" (System/currentTimeMillis))
+            (with-out-str (pp/pprint logs))))))
 
 (defmethod t/report :fail
   [m]
@@ -52,4 +61,10 @@
   ;; screenshot for all pw pages when :fail
   (when-let [all-contexts (seq *pw-contexts*)]
     (doseq [page (mapcat pw-page/get-pages all-contexts)]
-      (screenshot page (string/join "-" (map (comp str :name meta) t/*testing-vars*))))))
+      (screenshot page (string/join "-" (map (comp str :name meta) t/*testing-vars*)))))
+
+;; dump console logs
+  (when-let [pw-page->console-logs (some-> *pw-page->console-logs* deref)]
+    (doseq [[_pw-page logs] pw-page->console-logs]
+      (spit (format "e2e-dump/console-logs-%s.txt" (System/currentTimeMillis))
+            (with-out-str (pp/pprint logs))))))

clj-e2e/test/logseq/e2e/fixtures.clj

@@ -16,11 +16,16 @@
                   :persistent false
                   :slow-mo @config/*slow-mo})
     (w/grant-permissions :clipboard-write :clipboard-read)
-    (binding [custom-report/*pw-contexts* #{(.context (w/get-page))}]
+    (binding [custom-report/*pw-contexts* #{(.context (w/get-page))}
+              custom-report/*pw-page->console-logs* (atom {})]
       (w/navigate (str "http://localhost:" (or port @config/*port)))
       (settings/developer-mode)
       (w/refresh)
       (assert/assert-graph-loaded?)
+      (let [p (w/get-page)]
+        (.onConsoleMessage p (fn [msg]
+                               (when custom-report/*pw-page->console-logs*
+                                 (swap! custom-report/*pw-page->console-logs* update p conj (.text msg))))))
       (f))))
 
 (def *page1 (atom nil))
@@ -39,12 +44,20 @@
     (reset! *page1 p1)
     (reset! *page2 p2)
     (binding [custom-report/*pw-contexts* (set [(.context @p1) (.context @p2)])
+              custom-report/*pw-page->console-logs* (atom {})
               w/*page* (delay (throw (ex-info "Don't use *page*, use *page1* and *page2* instead" {})))]
       (run!
        #(w/with-page %
           (w/navigate (str "http://localhost:" port'))
           (settings/developer-mode)
-          (w/refresh))
+          (w/refresh)
+          (assert/assert-graph-loaded?)
+          (let [p (w/get-page)]
+            (.onConsoleMessage
+             p
+             (fn [msg]
+               (when custom-report/*pw-page->console-logs*
+                 (swap! custom-report/*pw-page->console-logs* update p conj (.text msg)))))))
        [p1 p2])
       (f))
 

deps/db/src/logseq/db/common/entity_plus.cljc

@@ -93,14 +93,16 @@
       (get-journal-title db e)
       (or
        (get (.-kv e) k)
-       (let [result (lookup-entity e k default-value)
+       (if db-based?
+         (let [result (lookup-entity e k default-value)
              ;; Replace title for pages only, otherwise it'll recursively
              ;; replace block id refs if there're cycle references of blocks
-             refs (:block/refs e)
-             result' (if (and (string? result) refs)
-                       (db-content/id-ref->title-ref result refs)
-                       result)]
-         (or result' default-value))))))
+               refs (:block/refs e)
+               result' (if (and (string? result) refs)
+                         (db-content/id-ref->title-ref result refs)
+                         result)]
+           (or result' default-value))
+         (lookup-entity e k default-value))))))
 
 (defn- lookup-kv-with-default-value
   [db ^Entity e k default-value]

deps/db/src/logseq/db/frontend/validate.cljs

@@ -38,10 +38,10 @@
                               ;; remove :db/id as it adds needless declarations to schema
                               #(validator [(dissoc % :db/id)])
                               ent-maps)]
-        (js/console.log "changed eids:" changed-ids tx-meta)
+        (prn "changed eids:" changed-ids :tx-meta tx-meta)
         (if (seq invalid-ent-maps)
           (let [explainer (get-schema-explainer (:closed-schema? validate-options))]
-            (js/console.error "Invalid datascript entities detected amongst changed entity ids:" changed-ids)
+            (prn "Invalid datascript entities detected amongst changed entity ids:" changed-ids)
             (doseq [m invalid-ent-maps]
               (let [m' (update m :block/properties (fn [properties]
                                                      (map (fn [[p v]]

shadow-cljs.edn

@@ -49,7 +49,8 @@
                                       :redef false}
                            :cross-chunk-method-motion false}
         :build-hooks [(shadow.hooks/git-revision-hook "--long --always --dirty")]
-        :closure-defines {goog.debug.LOGGING_ENABLED true
+        :closure-defines {lambdaisland.glogi.console/colorize "false"
+                          goog.debug.LOGGING_ENABLED true
                           frontend.modules.instrumentation.sentry/SENTRY-DSN #shadow/env "LOGSEQ_SENTRY_DSN"
                           frontend.modules.instrumentation.posthog/POSTHOG-TOKEN #shadow/env "LOGSEQ_POSTHOG_TOKEN"
                           frontend.config/ENABLE-PLUGINS #shadow/env ["ENABLE_PLUGINS" :as :bool :default true]

src/main/frontend/handler/editor.cljs

@@ -1343,11 +1343,18 @@
    (doseq [[block value] blocks]
      (save-block-if-changed! block value))))
 
+(defonce *auto-save-timeout (atom nil))
+(defn- clear-block-auto-save-timeout!
+  []
+  (when @*auto-save-timeout
+    (js/clearTimeout @*auto-save-timeout)))
+
 (defn save-current-block!
   "skip-properties? if set true, when editing block is likely be properties, skip saving"
   ([]
    (save-current-block! {}))
   ([{:keys [force? skip-properties? current-block] :as opts}]
+   (clear-block-auto-save-timeout!)
    ;; non English input method
    (when-not (or (state/editor-in-composition?)
                  (state/get-editor-action))
@@ -1803,15 +1810,13 @@
             new-value (string/replace value full_text new-full-text)]
         (save-block-aux! block new-value {})))))
 
-(defonce *auto-save-timeout (atom nil))
 (defn edit-box-on-change!
   [e block id]
   (when (= (:db/id block) (:db/id (state/get-edit-block)))
     (let [value (util/evalue e)
           repo (state/get-current-repo)]
       (state/set-edit-content! id value false)
-      (when @*auto-save-timeout
-        (js/clearTimeout @*auto-save-timeout))
+      (clear-block-auto-save-timeout!)
       (block-handler/mark-last-input-time! repo)
       (reset! *auto-save-timeout
               (js/setTimeout
@@ -1821,7 +1826,7 @@
                             (not (and
                                   (config/db-based-graph? repo)
                                   (re-find #"#\S+" value))))
-                 ; don't auto-save for page's properties block
+                   ; don't auto-save for page's properties block
                    (save-current-block! {:skip-properties? true})))
                450)))))
 

src/main/frontend/worker/db_listener.cljs

@@ -16,6 +16,7 @@
 (defn- sync-db-to-main-thread
   "Return tx-report"
   [repo conn {:keys [tx-meta] :as tx-report}]
+  (when repo (worker-state/set-db-latest-tx-time! repo))
   (let [{:keys [from-disk?]} tx-meta
         result (worker-pipeline/invoke-hooks repo conn tx-report (worker-state/get-context))
         tx-report' (:tx-report result)]

src/main/frontend/worker/file.cljs

@@ -147,7 +147,7 @@
 
 (defn do-write-file!
   [repo conn page-db-id outliner-op context request-id]
-  (let [page-block (d/pull @conn '[*] page-db-id)
+  (let [page-block (d/entity @conn page-db-id)
         page-db-id (:db/id page-block)
         whiteboard? (file-entity-util/whiteboard? page-block)
         blocks-count (ldb/get-page-blocks-count @conn page-db-id)
@@ -157,8 +157,10 @@
       (if (and (or (> blocks-count 500) whiteboard?)
                (not (worker-state/tx-idle? repo {:diff 3000})))
         (async/put! file-writes-chan [repo page-db-id outliner-op (tc/to-long (t/now)) request-id])
-        (let [pull-keys (if whiteboard? whiteboard-blocks-pull-keys-with-persisted-ids '[*])
-              blocks (ldb/get-page-blocks @conn (:db/id page-block) {:pull-keys pull-keys})
+        (let [blocks (if whiteboard?
+                       (ldb/get-page-blocks @conn (:db/id page-block)
+                                            {:pull-keys whiteboard-blocks-pull-keys-with-persisted-ids})
+                       (:block/_page page-block))
               blocks (if whiteboard? (map cleanup-whiteboard-block blocks) blocks)]
           (if (and (= 1 (count blocks))
                    (string/blank? (:block/title (first blocks)))

src/main/frontend/worker/rtc/core.cljs

@@ -480,7 +480,7 @@
             (m/?<
              (m/latest
               (fn [rtc-state rtc-auto-push? rtc-remote-profile?
-                   rtc-lock online-users pending-local-ops-count local-tx remote-tx]
+                   rtc-lock online-users pending-local-ops-count [local-tx remote-tx]]
                 {:graph-uuid graph-uuid
                  :local-graph-schema-version (db-schema/schema-version->string local-graph-schema-version)
                  :remote-graph-schema-version (db-schema/schema-version->string remote-graph-schema-version)
@@ -498,8 +498,7 @@
               (m/watch *rtc-auto-push?) (m/watch *rtc-remote-profile?)
               (m/watch *rtc-lock') (m/watch *online-users)
               (client-op/create-pending-block-ops-count-flow repo)
-              (rtc-log-and-state/create-local-t-flow graph-uuid)
-              (rtc-log-and-state/create-remote-t-flow graph-uuid))))
+              (rtc-log-and-state/create-local&remote-t-flow graph-uuid))))
           (catch Cancelled _))))))
 
 (def ^:private create-get-state-flow (c.m/throttle 300 create-get-state-flow*))

src/main/frontend/worker/rtc/log_and_state.cljs

@@ -64,26 +64,48 @@
     (string? v) (uuid v)
     :else       (throw (ex-info "illegal value" {:data v}))))
 
-(defn create-local-t-flow
+(defn- create-local-t-flow
   [graph-uuid]
   (->> (m/watch *graph-uuid->local-t)
        (m/eduction (keep (fn [m] (get m (ensure-uuid graph-uuid)))))
        c.m/continue-flow))
 
-(defn create-remote-t-flow
+(defn- create-remote-t-flow
   [graph-uuid]
-  {:pre [(some? graph-uuid)]}
   (->> (m/watch *graph-uuid->remote-t)
        (m/eduction (keep (fn [m] (get m (ensure-uuid graph-uuid)))))
        c.m/continue-flow))
 
+(defn create-local&remote-t-flow
+  "ensure local-t <= remote-t"
+  [graph-uuid]
+  (assert (some? graph-uuid))
+  (->> (m/latest vector (create-local-t-flow graph-uuid) (create-remote-t-flow graph-uuid))
+       (m/eduction (filter (fn [[local-t remote-t]] (>= remote-t local-t))))))
+
 (defn update-local-t
   [graph-uuid local-t]
-  (swap! *graph-uuid->local-t assoc (ensure-uuid graph-uuid) local-t))
+  (let [graph-uuid (ensure-uuid graph-uuid)
+        current-remote-t (get @*graph-uuid->remote-t graph-uuid)
+        current-local-t (get @*graph-uuid->local-t graph-uuid)]
+    (when (and current-remote-t current-local-t)
+      (assert (and (>= local-t current-local-t) (<= local-t current-remote-t))
+              {:local-t local-t
+               :current-local-t current-local-t
+               :current-remote-t current-remote-t}))
+    (swap! *graph-uuid->local-t assoc graph-uuid local-t)))
 
 (defn update-remote-t
   [graph-uuid remote-t]
-  (swap! *graph-uuid->remote-t assoc (ensure-uuid graph-uuid) remote-t))
+  (let [graph-uuid (ensure-uuid graph-uuid)
+        current-remote-t (get @*graph-uuid->remote-t graph-uuid)
+        current-local-t (get @*graph-uuid->local-t graph-uuid)]
+    (when (and current-remote-t current-local-t)
+      (assert (and (>= remote-t current-remote-t) (>= remote-t current-local-t))
+              {:remote-t remote-t
+               :current-local-t current-local-t
+               :current-remote-t current-remote-t}))
+    (swap! *graph-uuid->remote-t assoc graph-uuid remote-t)))
 
 ;;; subscribe-logs, push to frontend
 ;;; TODO: refactor by using c.m/run-background-task