Merge branch 'feat/db' into refactor/db-remove-block-name-unique

.clj-kondo/config.edn

@@ -182,6 +182,7 @@
            frontend.test.helper/deftest-async clojure.test/deftest
            frontend.test.helper/with-reset cljs.test/async
            frontend.worker.rtc.idb-keyval-mock/with-reset-idb-keyval-mock cljs.test/async
-           frontend.react/defc clojure.core/defn}
+           frontend.react/defc clojure.core/defn
+           frontend.schema-register/defkeyword cljs.spec.alpha/def}
  :skip-comments true
  :output {:progress true}}

.github/stale-issues.yml

@@ -46,7 +46,7 @@ jobs:
               - **Blog**: https://blog.logseq.com
               - **Docs**: https://docs.logseq.com
 
-            Thanks for your contributions to Logseq! If you have any other [**issues**](https://github.com/logseq/logseq/issues/new/choose) or [**feature requests**](https://discuss.logseq.com/c/feature-requests/), please don't hesitate to [let us know](https://github.com/logseq/logseq/issues/new/choose). We always welcome pull requests too!
+            Thanks for your contributions to Logseq! If you have any other [**issues**](https://github.com/logseq/logseq/issues/new/choose) or [**feature requests**](https://discuss.logseq.com/c/feedback/feature-requests/), please don't hesitate to [let us know](https://github.com/logseq/logseq/issues/new/choose). We always welcome pull requests too!
 
       - name: 'Print outputs'
         run: |
@@ -81,7 +81,7 @@ jobs:
               - **Blog**: https://blog.logseq.com
               - **Docs**: https://docs.logseq.com
 
-            Thanks for your contributions to Logseq! If you have any other [**issues**](https://github.com/logseq/logseq/issues/new/choose) or [**feature requests**](https://discuss.logseq.com/c/feature-requests/), please don't hesitate to [let us know](https://github.com/logseq/logseq/issues/new/choose). We always welcome pull requests too!
+            Thanks for your contributions to Logseq! If you have any other [**issues**](https://github.com/logseq/logseq/issues/new/choose) or [**feature requests**](https://discuss.logseq.com/c/feedback/feature-requests/), please don't hesitate to [let us know](https://github.com/logseq/logseq/issues/new/choose). We always welcome pull requests too!
 
       - name: 'Print outputs'
         run: |

.github/workflows/build-desktop-release.yml

@@ -356,13 +356,6 @@ jobs:
         run: yarn run postinstall
         working-directory: ./static/node_modules/dugite/
 
-      - name: Prepare Code Sign
-        if: ${{ github.repository == 'logseq/logseq' }}
-        run: |
-          [IO.File]::WriteAllBytes($(Get-Location).Path + "\codesign.pfx", [Convert]::FromBase64String($env:CERTIFICATE))
-        env:
-          CERTIFICATE: ${{ secrets.CODE_SIGN_CERTIFICATE }}
-
       - name: Build/Release Electron app
         run: yarn electron:make
         working-directory: ./static
@@ -550,9 +543,33 @@ jobs:
       ANDROID_KEYSTORE_PASSWORD: "${{ secrets.ANDROID_KEYSTORE_PASSWORD }}"
       SENTRY_AUTH_TOKEN: "${{ secrets.SENTRY_AUTH_TOKEN }}"
 
+  codesign-windows:
+    if: ${{ github.event_name == 'schedule' || github.event.inputs.build-target == 'nightly' || github.event.inputs.build-target == 'beta' }}
+    needs: [ build-windows ]
+    runs-on: [self-hosted, macos, token]
+    steps:
+      - name: Download Windows Artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: logseq-win64-builds
+          path: ./builds
+
+      - name: Sign Windows Executable
+        run: |
+          ls -lah ./builds
+          jsign --storetype ETOKEN --storepass "${PASS}" -t http://timestamp.digicert.com ./builds/*.exe
+        env:
+          PASS: ${{ secrets.CODE_SIGN_CERTIFICATE_PASSWORD }}
+
+      - name: Upload Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: logseq-win64-signed-builds
+          path: builds
+
   nightly-release:
     if: ${{ github.event_name == 'schedule' || github.event.inputs.build-target == 'nightly' }}
-    needs: [ build-macos-x64, build-macos-arm64, build-linux-x64, build-linux-arm64, build-windows, build-android, e2e-test ]
+    needs: [ build-macos-x64, build-macos-arm64, build-linux-x64, build-linux-arm64, codesign-windows, build-android, e2e-test ]
     runs-on: ubuntu-20.04
     steps:
       - name: Download MacOS x64 Artifacts
@@ -579,10 +596,10 @@ jobs:
           name: logseq-linux-arm64-builds
           path: ./
 
-      - name: Download The Windows Artifact
+      - name: Download The Windows Artifact (Signed)
         uses: actions/download-artifact@v3
         with:
-          name: logseq-win64-builds
+          name: logseq-win64-signed-builds
           path: ./
 
       - name: Download Android Artifacts
@@ -625,7 +642,7 @@ jobs:
   release:
     # NOTE: For now, we only have beta channel to be released on Github
     if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.build-target == 'beta' }}
-    needs: [ build-macos-x64, build-macos-arm64, build-linux-x64, build-linux-arm64, build-windows, e2e-test ]
+    needs: [ build-macos-x64, build-macos-arm64, build-linux-x64, build-linux-arm64, codesign-windows, build-android, e2e-test ]
     runs-on: ubuntu-20.04
     steps:
       - name: Download MacOS x64 Artifacts
@@ -655,7 +672,7 @@ jobs:
       - name: Download The Windows Artifact
         uses: actions/download-artifact@v3
         with:
-          name: logseq-win64-builds
+          name: logseq-win64-signed-builds
           path: ./
 
       - name: Download Android Artifacts

.github/workflows/stale-issues.yml

@@ -46,7 +46,7 @@ jobs:
               - **Blog**: https://blog.logseq.com
               - **Docs**: https://docs.logseq.com
 
-            Thanks for your contributions to Logseq! If you have any other [**issues**](https://github.com/logseq/logseq/issues/new/choose) or [**feature requests**](https://discuss.logseq.com/c/feature-requests/), please don't hesitate to [let us know](https://github.com/logseq/logseq/issues/new/choose). We always welcome pull requests too!
+            Thanks for your contributions to Logseq! If you have any other [**issues**](https://github.com/logseq/logseq/issues/new/choose) or [**feature requests**](https://discuss.logseq.com/c/feedback/feature-requests/), please don't hesitate to [let us know](https://github.com/logseq/logseq/issues/new/choose). We always welcome pull requests too!
 
       - name: '🧹 Close stale awaiting response issues'
         id: awaiting_issues
@@ -74,5 +74,5 @@ jobs:
               - **Blog**: https://blog.logseq.com
               - **Docs**: https://docs.logseq.com
 
-            Thanks for your contributions to Logseq! If you have any other [**issues**](https://github.com/logseq/logseq/issues/new/choose) or [**feature requests**](https://discuss.logseq.com/c/feature-requests/), please don't hesitate to [let us know](https://github.com/logseq/logseq/issues/new/choose). We always welcome pull requests too!
+            Thanks for your contributions to Logseq! If you have any other [**issues**](https://github.com/logseq/logseq/issues/new/choose) or [**feature requests**](https://discuss.logseq.com/c/feedback/feature-requests/), please don't hesitate to [let us know](https://github.com/logseq/logseq/issues/new/choose). We always welcome pull requests too!
 

CONTRIBUTING.md

@@ -168,7 +168,7 @@ Your contributions to open source, large or small, make great projects like this
 [github]: https://github.com/logseq/logseq "Logseq Repo"
 [discord]: https://discord.gg/KpN4eHY "Logseq Discord Server"
 [individual-cla]: https://cla-assistant.io/logseq/logseq "Individual CLA"
-[feature-request]: https://discuss.logseq.com/c/feature-requests/ "Submit Feature Request"
+[feature-request]: https://discuss.logseq.com/c/feedback/feature-requests/ "Submit Feature Request"
 [forum]: https://discuss.logseq.com "Logseq Forum"
 [search-pr]: https://github.com/logseq/logseq/pulls "Search open PRs"
 [new-issue]: https://github.com/logseq/logseq/issues/new?assignees=&labels=&template=bug_report.yaml "Submit a New issue"

bb.edn

@@ -136,6 +136,12 @@
   dev:lint
   logseq.tasks.dev/lint
 
+  dev:test
+  logseq.tasks.dev/test
+
+  dev:lint-and-test
+  logseq.tasks.dev/lint-and-test
+
   dev:gen-malli-kondo-config
   logseq.tasks.dev/gen-malli-kondo-config
 

deps/graph-parser/src/logseq/graph_parser/block.cljs

@@ -156,6 +156,19 @@
      properties)
     []))
 
+(defn- extract-refs-from-property-value
+  [value format]
+  (cond
+    (coll? value)
+    (filter (fn [v] (and (string? v) (not (string/blank? v)))) value)
+    (and (string? value) (= \" (first value) (last value)))
+    nil
+    (string? value)
+    (let [ast (gp-mldoc/inline->edn value (gp-mldoc/default-config format))]
+      (text/extract-refs-from-mldoc-ast ast))
+    :else
+    nil))
+
 (defn- get-page-ref-names-from-properties
   [properties user-config]
   (let [page-refs (->>
@@ -169,8 +182,9 @@
                               (keyword k))))
                    ;; get links ast
                    (map last)
-                   (mapcat (or (:extract-refs-from-property-value-fn user-config)
-                               text/extract-refs-from-mldoc-ast))
+                   (mapcat (fn [value]
+                             (let [f (or (:extract-refs-from-property-value-fn user-config) extract-refs-from-property-value)]
+                               (f value (get user-config :format :markdown)))))
                    ;; comma separated collections
                    (concat (->> (map second properties)
                                 (filter coll?)
@@ -488,11 +502,7 @@
                          id (get-custom-id-or-new-id {:properties properties})
                          property-refs (->> (get-page-refs-from-properties
                                              properties db date-formatter
-                                             (assoc user-config
-                                                    :extract-refs-from-property-value-fn
-                                                    (fn [refs]
-                                                      (when (coll? refs)
-                                                        refs))))
+                                             user-config)
                                             (map :block/original-name))
                          pre-block? (if (:heading properties) false true)
                          block {:block/uuid id

deps/shui/src/logseq/shui/dialog/core.cljs

@@ -104,8 +104,11 @@
 ;; components
 (rum/defc modal-inner
   [config]
-  (let [{:keys [id title description content footer on-open-change open?]} config
-        props (dissoc config :id :title :description :content :footer :on-open-change :open?)]
+  (let [{:keys [id title description content footer on-open-change align open?]} config
+        props (dissoc config
+                :id :title :description :content :footer
+                :align :on-open-change :open?)
+        props (assoc-in props [:overlay-props :data-align] (name (or align :center)))]
 
     (rum/use-effect!
       (fn []

package.json

@@ -61,7 +61,7 @@
         "cljs:electron-watch": "clojure -M:cljs watch app electron --config-merge \"{:asset-path \\\"./js\\\"}\"",
         "cljs:release": "clojure -M:cljs release app publishing electron",
         "cljs:release-electron": "clojure -M:cljs release app electron --debug && clojure -M:cljs release publishing",
-        "cljs:release-app": "clojure -M:cljs release app --config-merge \"{:compiler-options {:output-feature-set :es6}}\"",
+        "cljs:release-app": "clojure -M:cljs release app",
         "cljs:release-publishing": "clojure -M:cljs release publishing",
         "cljs:test": "clojure -M:test compile test",
         "cljs:run-test": "node static/tests.js",

packages/ui/@/components/ui/dialog.tsx

@@ -1,7 +1,7 @@
 import * as React from 'react'
 import * as DialogPrimitive from '@radix-ui/react-dialog'
 import { X } from 'lucide-react'
-import { cn } from '../../lib/utils'
+import { cn } from '@/lib/utils'
 
 const Dialog = DialogPrimitive.Root
 
@@ -19,7 +19,7 @@ const DialogOverlay = React.forwardRef<
     ref={ref}
     className={cn(
       'ui__dialog-overlay',
-      'fixed inset-0 z-50 bg-background/90 data-[state=open]:animate-in ' +
+      'fixed inset-0 z-50 bg-background/90 data-[state=open]:animate-in flex justify-center items-center ' +
       'data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0',
       className
     )}
@@ -30,20 +30,17 @@ DialogOverlay.displayName = DialogPrimitive.Overlay.displayName
 
 const DialogContent = React.forwardRef<
   React.ElementRef<typeof DialogPrimitive.Content>,
-  React.ComponentPropsWithoutRef<typeof DialogPrimitive.Content>
->(({ className, children, ...props }, ref) => (
+  React.ComponentPropsWithoutRef<typeof DialogPrimitive.Content & any>
+>(({ className, children, overlayProps, ...props }, ref) => (
   <DialogPortal>
-    <DialogOverlay>
+    <DialogOverlay {...overlayProps}>
       <DialogPrimitive.Content
         ref={ref}
         className={cn(
           'ui__dialog-content',
-          'fixed left-[50%] top-[50%] z-50 grid w-full max-w-2xl lg:max-w-3xl translate-x-[-50%] translate-y-[-50%] gap-4 border ' +
-          'bg-background p-6 shadow-lg duration-200 data-[state=open]:animate-in data-[state=closed]:animate-out ' +
-          'data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 ' +
-          'data-[state=open]:zoom-in-95 data-[state=closed]:slide-out-to-left-1/2 ' +
-          'data-[state=closed]:slide-out-to-top-[48%] data-[state=open]:slide-in-from-left-1/2 data-[state=open]:slide-in-from-top-[48%] ' +
-          'sm:rounded-lg',
+          'relative grid w-full max-w-2xl lg:max-w-3xl gap-4 border sm:rounded-lg ' +
+          'bg-background p-6 shadow-lg duration-200 data-[state=open]:animate-in ' +
+          'data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95 ',
           className
         )}
         {...props}

resources/css/shui.css

@@ -225,13 +225,14 @@ html[data-theme=dark] {
   }
 }
 
-.ui__dialog-content {
-  &[side=start] {
-    @apply translate-y-0 top-[56px];
-  }
-
-  &[side=end] {
+.ui__dialog-overlay {
+  &[data-align=start],
+  &[data-align=top] {
+    @apply !items-start;
 
+    .ui__dialog-content {
+      @apply top-20;
+    }
   }
 }
 

scripts/src/logseq/tasks/dev.clj

@@ -27,6 +27,19 @@
     (println cmd)
     (shell cmd)))
 
+(defn test
+  "Run tests. Pass args through to cmd 'yarn cljs:run-test'"
+  [& args]
+  (shell "yarn cljs:test")
+  (apply shell "yarn cljs:run-test" args))
+
+(defn lint-and-test
+  "Run all lint tasks, then run tests(exclude testcases taged by :long).
+  pass args through to cmd 'yarn cljs:run-test'"
+  []
+  (lint)
+  (test "-e" "long"))
+
 
 (defn gen-malli-kondo-config
   "Generate clj-kondo type-mismatch config from malli schema

src/main/frontend/common_keywords.cljs

@@ -0,0 +1,39 @@
+(ns frontend.common-keywords
+  "There are some keywords scattered throughout the codebase."
+  (:require [frontend.schema-register :include-macros true :as sr]))
+
+
+(sr/defkeyword :block/uuid
+  "block's uuid"
+  :uuid)
+
+(sr/defkeyword :block/name
+  "block name, lowercase, only page-blocks have this attr"
+  :string)
+
+(sr/defkeyword :block/original-name
+  "like `:block/name`, but not unified into lowercase"
+  :string)
+
+(sr/defkeyword :block/type
+  "block type"
+  [:enum #{"property"} #{"class"} #{"whiteboard"} #{"hidden"}])
+
+(sr/defkeyword :block/parent
+  "page blocks don't have this attr")
+
+(sr/defkeyword :block/left
+  "
+- page blocks don't have this attr
+- some no-order blocks don't have this attr too,
+  TODO: list these types")
+
+(sr/defkeyword :block/content
+  "content string of the blocks.
+in db-version, page-references(e.g. [[page-name]]) are stored as [[~^uuid]]."
+  :string)
+
+(sr/defkeyword :block/raw-content
+  "like `:block/content`,
+but when eval `(:block/raw-content block-entity)`, return raw-content of this block"
+  :string)

src/main/frontend/components/container.cljs

@@ -756,7 +756,7 @@
    {:title "Documentation" :icon "help" :href "https://docs.logseq.com/"}
    :hr
    {:title "Report bug" :icon "bug" :on-click #(rfe/push-state :bug-report)}
-   {:title "Request feature" :icon "git-pull-request" :href "https://discuss.logseq.com/c/feature-requests/"}
+   {:title "Request feature" :icon "git-pull-request" :href "https://discuss.logseq.com/c/feedback/feature-requests/"}
    {:title "Submit feedback" :icon "messages" :href "https://discuss.logseq.com/c/feedback/13"}
    :hr
    {:title "Ask the community" :icon "brand-discord" :href "https://discord.com/invite/KpN4eHY"}

src/main/frontend/components/onboarding.cljs

@@ -34,7 +34,7 @@
           {:title (t :help/title-development)
            :children [[(t :help/roadmap) "https://trello.com/b/8txSM12G/roadmap"]
                       [(t :help/bug) "https://github.com/logseq/logseq/issues/new?labels=from:in-app&template=bug_report.yaml"]
-                      [(t :help/feature) "https://discuss.logseq.com/c/feature-requests/"]
+                      [(t :help/feature) "https://discuss.logseq.com/c/feedback/feature-requests/"]
                       [(t :help/changelog) "https://docs.logseq.com/#/page/changelog"]]}
 
           {:title (t :help/title-about)

src/main/frontend/components/plugins.cljs

@@ -377,19 +377,19 @@
                     (reset! *search-key nil)
                     (.focus target))}
       (ui/icon "x")])
-   [:input.form-input.is-small
-    {:placeholder (t :plugin/search-plugin)
-     :ref         *search-ref
-     :auto-focus  true
-     :on-key-down (fn [^js e]
-                    (when (= 27 (.-keyCode e))
-                      (util/stop e)
-                      (if (string/blank? search-key)
-                        (some-> (js/document.querySelector ".cp__plugins-page") (.focus))
-                        (reset! *search-key nil))))
-     :on-change   #(let [^js target (.-target %)]
-                     (reset! *search-key (some-> (.-value target) (string/triml))))
-     :value       (or search-key "")}]])
+   (shui/input
+     {:placeholder (t :plugin/search-plugin)
+      :ref *search-ref
+      :auto-focus true
+      :on-key-down (fn [^js e]
+                     (when (= 27 (.-keyCode e))
+                       (util/stop e)
+                       (if (string/blank? search-key)
+                         (some-> (js/document.querySelector ".cp__plugins-page") (.focus))
+                         (reset! *search-key nil))))
+      :on-change #(let [^js target (.-target %)]
+                    (reset! *search-key (some-> (.-value target) (string/triml))))
+      :value (or search-key "")})])
 
 (rum/defc panel-tab-developer
   []
@@ -1407,7 +1407,7 @@
   (shui/dialog-open!
     (plugins-page)
     {:label :plugins-dashboard
-     :side :center}))
+     :align :start}))
 
 (defn open-waiting-updates-modal!
   []
@@ -1431,6 +1431,7 @@
       [:div.settings-modal.of-plugins
        (focused-settings-content title)])
     {:label   "plugin-settings-modal"
+     :align   :start
      :id      "ls-focused-settings-modal"}))
 
 (defn hook-custom-routes

src/main/frontend/components/plugins.css

@@ -150,10 +150,11 @@
         z-index: 1;
       }
 
-      .form-input {
+      .ui__input {
         background-color: var(--ls-primary-background-color);
         padding: 6px 7px 5px 29px;
-        opacity: .5;
+        opacity: .7;
+        height: 30px;
 
         &:focus {
           background-color: var(--ls-secondary-background-color);
@@ -963,8 +964,7 @@ html[data-theme='dark'] {
 }
 
 .ui__dialog-content[label=plugin-settings-modal] {
-  @apply w-auto lg:max-w-5xl p-0 gap-0
-  top-[10%] translate-y-0 !animate-none;
+  @apply w-auto lg:max-w-5xl p-0 gap-0;
 }
 
 .ui__dialog-content[label=plugins-dashboard] {

src/main/frontend/core.cljs

@@ -1,21 +1,23 @@
 (ns frontend.core
   "Entry ns for the mobile, browser and electron frontend apps"
   {:dev/always true}
-  (:require [rum.core :as rum]
+  (:require [frontend.common-keywords]
+            [frontend.components.plugins :as plugins]
+            [frontend.config :as config]
+            [frontend.fs.sync :as sync]
             [frontend.handler :as handler]
             [frontend.handler.plugin :as plugin-handler]
             [frontend.handler.route :as route-handler]
-            [frontend.components.plugins :as plugins]
+            [frontend.log]
             [frontend.page :as page]
             [frontend.routes :as routes]
+            [frontend.schema-register :as sr]
             [frontend.spec]
-            [frontend.log]
+            [logseq.api]
+            [malli.dev.cljs :as md]
             [reitit.frontend :as rf]
             [reitit.frontend.easy :as rfe]
-            [logseq.api]
-            [frontend.fs.sync :as sync]
-            [frontend.config :as config]
-            [malli.dev.cljs :as md]))
+            [rum.core :as rum]))
 
 (defn set-router!
   []
@@ -47,6 +49,7 @@
 (defn ^:export start []
   (when config/dev?
     (md/start!))
+  (frontend.schema-register/init)
   (when-let [node (.getElementById js/document "root")]
     (set-router!)
     (rum/mount (page/current-page) node)

src/main/frontend/db_worker.cljs

@@ -18,13 +18,13 @@
             [frontend.worker.handler.page.file-based.rename :as file-worker-page-rename]
             [frontend.worker.handler.page.db-based.rename :as db-worker-page-rename]
             [frontend.worker.rtc.core :as rtc-core]
-            [frontend.worker.rtc.db-listener :as rtc-db-listener]
+            [frontend.worker.rtc.db-listener]
             [frontend.worker.rtc.full-upload-download-graph :as rtc-updown]
             [frontend.worker.rtc.op-mem-layer :as op-mem-layer]
             [frontend.worker.rtc.snapshot :as rtc-snapshot]
             [frontend.worker.search :as search]
             [frontend.worker.state :as worker-state]
-            [frontend.worker.undo-redo]
+            [frontend.worker.undo-redo :as undo-redo]
             [frontend.worker.util :as worker-util]
             [logseq.db :as ldb]
             [logseq.db.sqlite.common-db :as sqlite-common-db]
@@ -179,9 +179,7 @@
             conn (sqlite-common-db/get-storage-conn storage schema)]
         (swap! *datascript-conns assoc repo conn)
         (p/let [_ (op-mem-layer/<init-load-from-indexeddb! repo)]
-          (rtc-db-listener/listen-to-db-changes! repo conn)
-          (db-listener/listen-db-changes! repo conn))
-        ))))
+          (db-listener/listen-db-changes! repo conn))))))
 
 (defn- iter->vec [iter]
   (when iter
@@ -605,7 +603,6 @@
          (try
            (let [state (<? (rtc-core/<init-state token false))
                  r (<? (rtc-updown/<async-upload-graph state repo conn remote-graph-name))]
-             (rtc-db-listener/listen-db-to-generate-ops repo conn)
              (p/resolve! d r))
            (catch :default e
              (worker-util/post-message :notification
@@ -694,6 +691,14 @@
    [_this block-uuid]
    (transit/write transit-w (rtc-core/get-block-update-log (uuid block-uuid))))
 
+  (undo
+   [_this repo]
+   (undo-redo/undo repo)
+   nil)
+  (redo
+   [_this repo]
+   (undo-redo/redo repo))
+
   (keep-alive
    [_this]
    "alive")

src/main/frontend/handler/history.cljs

@@ -1,13 +1,14 @@
 (ns ^:no-doc frontend.handler.history
-  (:require [frontend.db :as db]
+  (:require [frontend.config :as config]
+            [frontend.db :as db]
+            [frontend.db.transact :as db-transact]
             [frontend.handler.editor :as editor]
+            [frontend.handler.route :as route-handler]
             [frontend.modules.editor.undo-redo :as undo-redo]
             [frontend.state :as state]
             [frontend.util :as util]
-            [frontend.handler.route :as route-handler]
             [goog.dom :as gdom]
-            [promesa.core :as p]
-            [frontend.db.transact :as db-transact]))
+            [promesa.core :as p]))
 
 (defn restore-cursor!
   [{:keys [last-edit-block container pos end-pos]} undo?]
@@ -40,20 +41,28 @@
 
 (defn undo!
   [e]
-  (when (db-transact/request-finished?)
-    (util/stop e)
-    (p/do!
-     (state/set-state! [:editor/last-replace-ref-content-tx (state/get-current-repo)] nil)
-     (editor/save-current-block!)
-     (state/clear-editor-action!)
-     (state/set-block-op-type! nil)
-     (let [cursor-state (undo-redo/undo)]
-       (state/set-state! :ui/restore-cursor-state (select-keys cursor-state [:editor-cursor :app-state]))))))
+  (when-let [repo (state/get-current-repo)]
+    (when (db-transact/request-finished?)
+      (util/stop e)
+      (p/do!
+       (state/set-state! [:editor/last-replace-ref-content-tx repo] nil)
+       (editor/save-current-block!)
+       (state/clear-editor-action!)
+       (state/set-block-op-type! nil)
+       (if (config/db-based-graph? repo)
+         (let [^js worker @state/*db-worker]
+           (.undo worker repo))
+         (let [cursor-state (undo-redo/undo)]
+           (state/set-state! :ui/restore-cursor-state (select-keys cursor-state [:editor-cursor :app-state]))))))))
 
 (defn redo!
   [e]
-  (when (db-transact/request-finished?)
-    (util/stop e)
-    (state/clear-editor-action!)
-    (let [cursor-state (undo-redo/redo)]
-      (state/set-state! :ui/restore-cursor-state (select-keys cursor-state [:editor-cursor :app-state])))))
+  (when-let [repo (state/get-current-repo)]
+    (when (db-transact/request-finished?)
+      (util/stop e)
+      (state/clear-editor-action!)
+      (if (config/db-based-graph? repo)
+        (let [^js worker @state/*db-worker]
+          (.redo worker repo))
+        (let [cursor-state (undo-redo/redo)]
+          (state/set-state! :ui/restore-cursor-state (select-keys cursor-state [:editor-cursor :app-state])))))))

src/main/frontend/schema_register.clj

@@ -0,0 +1,12 @@
+(ns frontend.schema-register
+  "Macro 'defkeyword' to def keyword with docstring and malli-schema")
+
+
+(defmacro defkeyword
+  "Define keyword with docstring and malli-schema"
+  [kw docstring & [optional-malli-schema]]
+  (assert (keyword? kw) "must be keyword")
+  (assert (some? docstring) "must have 'docstring' arg")
+  (when optional-malli-schema
+    `(do (assert (frontend.schema-register/not-register-yet? ~kw) (str "Already registered: " ~kw))
+         (frontend.schema-register/register! ~kw ~optional-malli-schema))))

src/main/frontend/schema_register.cljs

@@ -0,0 +1,23 @@
+(ns frontend.schema-register
+  "Set malli default registry to a mutable one,
+  and use `register!` to add schemas dynamically."
+  (:require [malli.core :as m]
+            [malli.registry :as mr]))
+
+(def *malli-registry (atom {}))
+
+(defn register!
+  [type schema]
+  (swap! *malli-registry assoc type schema))
+
+(defn not-register-yet?
+  [type]
+  (boolean (nil? (@*malli-registry type))))
+
+(defn init
+  []
+  (reset! *malli-registry {})
+  (mr/set-default-registry!
+   (mr/composite-registry
+    (m/default-schemas)
+    (mr/mutable-registry *malli-registry))))

src/main/frontend/worker/db_listener.cljs

@@ -1,6 +1,12 @@
 (ns frontend.worker.db-listener
   "Db listeners for worker-db."
-  (:require [datascript.core :as d]))
+  (:require [cljs-bean.core :as bean]
+            [datascript.core :as d]
+            [frontend.worker.pipeline :as worker-pipeline]
+            [frontend.worker.search :as search]
+            [frontend.worker.state :as worker-state]
+            [frontend.worker.util :as worker-util]
+            [promesa.core :as p]))
 
 
 (defn- entity-datoms=>attr->datom
@@ -27,6 +33,31 @@
 (defmulti listen-db-changes
   (fn [listen-key & _] listen-key))
 
+(defmethod listen-db-changes :sync-db-to-main-thread
+  [_ {:keys [tx-meta repo conn] :as tx-report}]
+  (let [{:keys [pipeline-replace? from-disk?]} tx-meta
+        result (worker-pipeline/invoke-hooks repo conn tx-report (worker-state/get-context))
+        tx-report' (or (:tx-report result) tx-report)]
+    (when-not pipeline-replace?
+      (let [data (merge
+                  {:request-id (:request-id tx-meta)
+                   :repo repo
+                   :tx-data (:tx-data tx-report')
+                   :tx-meta tx-meta}
+                  (dissoc result :tx-report))]
+        (worker-util/post-message :sync-db-changes data))
+
+      (when-not from-disk?
+        (p/do!
+         (let [{:keys [blocks-to-remove-set blocks-to-add]} (search/sync-search-indice repo tx-report')
+               ^js wo (worker-state/get-worker-object)]
+           (when wo
+             (when (seq blocks-to-remove-set)
+               (.search-delete-blocks wo repo (bean/->js blocks-to-remove-set)))
+             (when (seq blocks-to-add)
+               (.search-upsert-blocks wo repo (bean/->js blocks-to-add))))))))))
+
+
 (defn listen-db-changes!
   [repo conn]
   (let [handlers (methods listen-db-changes)]
@@ -38,9 +69,11 @@
                        id->same-entity-datoms (group-by first datom-vec-coll)
                        id-order (distinct (map first datom-vec-coll))
                        same-entity-datoms-coll (map id->same-entity-datoms id-order)
-                       id->attr->datom (update-vals id->same-entity-datoms entity-datoms=>attr->datom)]
+                       id->attr->datom (update-vals id->same-entity-datoms entity-datoms=>attr->datom)
+                       args* (assoc args
+                                    :repo repo
+                                    :conn conn
+                                    :id->attr->datom id->attr->datom
+                                    :same-entity-datoms-coll same-entity-datoms-coll)]
                    (doseq [[k handler-fn] handlers]
-                     (handler-fn k (assoc args
-                                          :repo repo
-                                          :id->attr->datom id->attr->datom
-                                          :same-entity-datoms-coll same-entity-datoms-coll))))))))
+                     (handler-fn k args*)))))))

src/main/frontend/worker/handler/page.cljs

@@ -138,7 +138,8 @@
                                              page-txs
                                              first-block-tx)]
                                (when (seq txs)
-                                 [page-uuid (ldb/transact! conn txs (cond-> {:persist-op? persist-op?}
+                                 [page-uuid (ldb/transact! conn txs (cond-> {:persist-op? persist-op?
+                                                                             :outliner-op :create-page}
                                                                       today-journal?
                                                                       (assoc :create-today-journal? true
                                                                              :today-journal-name page-name)))])))] ;; FIXME: prettier validation

src/main/frontend/worker/rtc/core.cljs

@@ -125,6 +125,7 @@
 (defmethod transact-db! :delete-blocks [_ & args]
   (outliner-tx/transact!
    {:persist-op? false
+    :gen-undo-op? false
     :outliner-op :delete-blocks
     :transact-opts {:repo (first args)
                     :conn (second args)}}
@@ -133,6 +134,7 @@
 (defmethod transact-db! :move-blocks [_ & args]
   (outliner-tx/transact!
    {:persist-op? false
+    :gen-undo-op? false
     :outliner-op :move-blocks
     :transact-opts {:repo (first args)
                     :conn (second args)}}
@@ -141,6 +143,7 @@
 (defmethod transact-db! :move-blocks&persist-op [_ & args]
   (outliner-tx/transact!
    {:persist-op? true
+    :gen-undo-op? false
     :outliner-op :move-blocks
     :transact-opts {:repo (first args)
                     :conn (second args)}}
@@ -149,6 +152,7 @@
 (defmethod transact-db! :insert-blocks [_ & args]
   (outliner-tx/transact!
       {:persist-op? false
+       :gen-undo-op? false
        :outliner-op :insert-blocks
        :transact-opts {:repo (first args)
                        :conn (second args)}}
@@ -163,12 +167,14 @@
                           ;; must be `logseq.db.frontend.malli-schema.closed-value-block`
                           :block/type #{"closed value"}})
                        block-uuids)
-                 {:persist-op? false}))
+                 {:persist-op? false
+                  :gen-undo-op? false}))
 
 
 (defmethod transact-db! :save-block [_ & args]
   (outliner-tx/transact!
       {:persist-op? false
+       :gen-undo-op? false
        :outliner-op :save-block
        :transact-opts {:repo (first args)
                        :conn (second args)}}
@@ -177,10 +183,12 @@
 (defmethod transact-db! :delete-whiteboard-blocks [_ conn block-uuids]
   (ldb/transact! conn
                  (mapv (fn [block-uuid] [:db/retractEntity [:block/uuid block-uuid]]) block-uuids)
-                 {:persist-op? false}))
+                 {:persist-op? false
+                  :gen-undo-op? false}))
 
 (defmethod transact-db! :upsert-whiteboard-block [_ conn blocks]
-  (ldb/transact! conn blocks {:persist-op? false}))
+  (ldb/transact! conn blocks {:persist-op? false
+                              :gen-undo-op? false}))
 
 (defn- whiteboard-page-block?
   [block]
@@ -444,7 +452,8 @@
                      [:db/retract db-id :block/journal-day]
                      [:db/retract db-id :block/journal?]))
             (when (seq @*other-tx-data)
-              (ldb/transact! conn @*other-tx-data {:persist-op? false}))
+              (ldb/transact! conn @*other-tx-data {:persist-op? false
+                                                   :gen-undo-op? false}))
             (transact-db! :save-block repo conn date-formatter new-block)))))))
 
 (defn apply-remote-move-ops
@@ -480,6 +489,7 @@
     (when (and (seq blocks) target-page-block)
       (outliner-tx/transact!
        {:persist-op? true
+        :gen-undo-op? false
         :transact-opts {:repo repo
                         :conn conn}}
        (outliner-core/move-blocks! repo conn blocks target-page-block false)))))

src/main/frontend/worker/rtc/db_listener.cljs

@@ -5,34 +5,8 @@
             [clojure.data :as data]
             [clojure.set :as set]
             [datascript.core :as d]
-            [frontend.worker.rtc.op-mem-layer :as op-mem-layer]
-            [frontend.worker.state :as worker-state]
-            [frontend.worker.pipeline :as worker-pipeline]
-            [frontend.worker.search :as search]
-            [frontend.worker.util :as worker-util]
-            [promesa.core :as p]
-            [cljs-bean.core :as bean]))
-
-
-(defn- entity-datoms=>attr->datom
-  [entity-datoms]
-  (reduce
-   (fn [m datom]
-     (let [[_e a _v t add?] datom]
-       (if-let [[_e _a _v old-t old-add?] (get m a)]
-         (cond
-           (and (= old-t t)
-                (true? add?)
-                (false? old-add?))
-           (assoc m a datom)
-
-           (< old-t t)
-           (assoc m a datom)
-
-           :else
-           m)
-         (assoc m a datom))))
-   {} entity-datoms))
+            [frontend.worker.db-listener :as db-listener]
+            [frontend.worker.rtc.op-mem-layer :as op-mem-layer]))
 
 
 (defn- diff-value-of-set-type-attr
@@ -60,11 +34,11 @@
       (seq retract-uuids) (conj [:retract retract-uuids]))))
 
 (defn- entity-datoms=>ops
-  [db-before db-after entity-datoms]
-  (let [attr->datom (entity-datoms=>attr->datom entity-datoms)]
+  [db-before db-after id->attr->datom entity-datoms]
+  (let [e (ffirst entity-datoms)
+        attr->datom (id->attr->datom e)]
     (when (seq attr->datom)
       (let [updated-key-set (set (keys attr->datom))
-            e (some-> attr->datom first second first)
             {[_e _a block-uuid _t add1?] :block/uuid
              [_e _a _v _t add2?]         :block/name
              [_e _a _v _t add3?]         :block/parent
@@ -153,36 +127,32 @@
         ops*))))
 
 (defn- entity-datoms=>asset-op
-  [db-after entity-datoms]
-  (let [attr->datom (entity-datoms=>attr->datom entity-datoms)]
-    (when (seq attr->datom)
-      (let [e (some-> attr->datom first second first)
-            {[_e _a asset-uuid _t add1?] :asset/uuid
-             [_e _a asset-meta _t add2?] :asset/meta}
-            attr->datom
-            op (cond
-                 (or (and add1? asset-uuid)
-                     (and add2? asset-meta))
-                 [:update-asset]
-
-                 (and (not add1?) asset-uuid)
-                 [:remove-asset asset-uuid])]
-        (when op
-          (let [asset-uuid (some-> (d/entity db-after e) :asset/uuid str)]
-            (case (first op)
-              :update-asset (when asset-uuid ["update-asset" {:asset-uuid asset-uuid}])
-              :remove-asset ["remove-asset" {:asset-uuid (str (second op))}])))))))
+  [db-after id->attr->datom entity-datoms]
+  (when-let [e (ffirst entity-datoms)]
+    (let [attr->datom (id->attr->datom e)]
+      (when (seq attr->datom)
+        (let [{[_e _a asset-uuid _t add1?] :asset/uuid
+               [_e _a asset-meta _t add2?] :asset/meta}
+              attr->datom
+              op (cond
+                   (or (and add1? asset-uuid)
+                       (and add2? asset-meta))
+                   [:update-asset]
+
+                   (and (not add1?) asset-uuid)
+                   [:remove-asset asset-uuid])]
+          (when op
+            (let [asset-uuid (some-> (d/entity db-after e) :asset/uuid str)]
+              (case (first op)
+                :update-asset (when asset-uuid ["update-asset" {:asset-uuid asset-uuid}])
+                :remove-asset ["remove-asset" {:asset-uuid (str (second op))}]))))))))
 
 
 (defn generate-rtc-ops
-  [repo db-before db-after datoms]
-  (let [datom-vec-coll (map vec datoms)
-        id->same-entity-datoms (group-by first datom-vec-coll)
-        id-order (distinct (map first datom-vec-coll))
-        same-entity-datoms-coll (map id->same-entity-datoms id-order)
-        asset-ops (keep (partial entity-datoms=>asset-op db-after) same-entity-datoms-coll)
+  [repo db-before db-after same-entity-datoms-coll id->attr->datom]
+  (let [asset-ops (keep (partial entity-datoms=>asset-op db-after id->attr->datom) same-entity-datoms-coll)
         ops (when (empty asset-ops)
-              (mapcat (partial entity-datoms=>ops db-before db-after) same-entity-datoms-coll))
+              (mapcat (partial entity-datoms=>ops db-before db-after id->attr->datom) same-entity-datoms-coll))
         now-epoch*1000 (* 1000 (tc/to-long (t/now)))
         ops* (map-indexed (fn [idx op]
                             [(first op) (assoc (second op) :epoch (+ idx now-epoch*1000))]) ops)
@@ -195,56 +165,9 @@
       (op-mem-layer/add-asset-ops! repo asset-ops*))))
 
 
-
-(defn listen-db-to-generate-ops
-  [repo conn]
-  (d/listen! conn :gen-ops
-             (fn [{:keys [tx-data tx-meta db-before db-after]}]
-               (when (:persist-op? tx-meta true)
-                 (generate-rtc-ops repo db-before db-after tx-data)))))
-
-(comment
-  (defn listen-db-to-batch-txs
-   [conn]
-   (d/listen! conn :batch-txs
-              (fn [{:keys [tx-data]}]
-                (when (worker-state/batch-tx-mode?)
-                  (worker-state/conj-batch-txs! tx-data))))))
-
-(defn sync-db-to-main-thread
-  [repo conn]
-  (d/listen! conn :sync-db
-             (fn [{:keys [tx-meta] :as tx-report}]
-               (let [{:keys [pipeline-replace? from-disk?]} tx-meta
-                     result (worker-pipeline/invoke-hooks repo conn tx-report (worker-state/get-context))
-                     tx-report' (or (:tx-report result) tx-report)]
-                 (when-not pipeline-replace?
-                   (let [data (merge
-                               {:request-id (:request-id tx-meta)
-                                :repo repo
-                                :tx-data (:tx-data tx-report')
-                                :tx-meta tx-meta}
-                               (dissoc result :tx-report))]
-                     (worker-util/post-message :sync-db-changes data))
-
-                   (when-not from-disk?
-                     (p/do!
-                      (let [{:keys [blocks-to-remove-set blocks-to-add]} (search/sync-search-indice repo tx-report')
-                            ^js wo (worker-state/get-worker-object)]
-                        (when wo
-                          (when (seq blocks-to-remove-set)
-                            (.search-delete-blocks wo repo (bean/->js blocks-to-remove-set)))
-                          (when (seq blocks-to-add)
-                            (.search-upsert-blocks wo repo (bean/->js blocks-to-add))))))))))))
-
-(defn listen-to-db-changes!
-  [repo conn]
-  (d/unlisten! conn :gen-ops)
-  (d/unlisten! conn :sync-db)
-  (sync-db-to-main-thread repo conn)
-  (when (op-mem-layer/rtc-db-graph? repo)
-    (listen-db-to-generate-ops repo conn)
-    ;; (rtc-db-listener/listen-db-to-batch-txs conn)
-    )
-
-  )
+(defmethod db-listener/listen-db-changes :gen-rtc-ops
+  [_ {:keys [_tx-data tx-meta db-before db-after
+             repo id->attr->datom same-entity-datoms-coll]}]
+  (when (and (op-mem-layer/rtc-db-graph? repo)
+             (:persist-op? tx-meta true))
+    (generate-rtc-ops repo db-before db-after same-entity-datoms-coll id->attr->datom)))

src/main/frontend/worker/undo_redo.cljs

@@ -1,6 +1,7 @@
 (ns frontend.worker.undo-redo
   "undo/redo related fns and op-schema"
   (:require [datascript.core :as d]
+            [frontend.schema-register :include-macros true :as sr]
             [frontend.worker.db-listener :as db-listener]
             [frontend.worker.state :as worker-state]
             [logseq.common.config :as common-config]
@@ -9,22 +10,53 @@
             [malli.core :as m]
             [malli.util :as mu]))
 
-(def undo-op-schema
+(sr/defkeyword :gen-undo-op?
+  "tx-meta option, generate undo ops from tx-data when true (default true)")
+
+(sr/defkeyword :gen-undo-boundary-op?
+  "tx-meta option, generate `::boundary` undo-op when true (default true).
+usually every transaction's tx-data will generate ops like: [<boundary> <op1> <op2> ...],
+push to undo-stack, result in [...<boundary> <op0> <boundary> <op1> <op2> ...].
+
+when this option is false, only generate [<op1> <op2> ...]. undo-stack: [...<boundary> <op0> <op1> <op2> ...]
+so when undo, it will undo [<op0> <op1> <op2>] instead of [<op1> <op2>]")
+
+(sr/defkeyword ::boundary
+  "boundary of one or more undo-ops.
+when one undo/redo will operate on all ops between two ::boundary")
+
+(sr/defkeyword ::insert-block
+  "when a block is inserted, generate a ::insert-block undo-op.
+when undo this op, the related block will be removed.")
+
+(sr/defkeyword ::move-block
+  "when a block is moved, generate a ::move-block undo-op.")
+
+(sr/defkeyword ::remove-block
+  "when a block is removed, generate a ::remove-block undo-op.
+when undo this op, this original entity-map will be transacted back into db")
+
+(sr/defkeyword ::update-block
+  "when a block is updated, generate a ::update-block undo-op.")
+
+(def ^:private boundary [::boundary])
+
+(def ^:private undo-op-schema
   (mu/closed-schema
    [:multi {:dispatch first}
-    [:boundary
+    [::boundary
      [:cat :keyword]]
-    [:insert-block
+    [::insert-block
      [:cat :keyword
       [:map
        [:block-uuid :uuid]]]]
-    [:move-block
+    [::move-block
      [:cat :keyword
       [:map
        [:block-uuid :uuid]
        [:block-origin-left :uuid]
        [:block-origin-parent :uuid]]]]
-    [:remove-block
+    [::remove-block
      [:cat :keyword
       [:map
        [:block-uuid :uuid]
@@ -34,11 +66,11 @@
          [:block/left :uuid]
          [:block/parent :uuid]
          [:block/content :string]
-         [:block/created-at :int]
-         [:block/updated-at :int]
-         [:block/format :any]
+         [:block/created-at {:optional true} :int]
+         [:block/updated-at {:optional true} :int]
+         [:block/format {:optional true} :any]
          [:block/tags {:optional true} [:sequential :uuid]]]]]]]
-    [:update-block
+    [::update-block
      [:cat :keyword
       [:map
        [:block-uuid :uuid]
@@ -46,74 +78,103 @@
        ;; TODO: add more attrs
        ]]]]))
 
-(def undo-ops-validator (m/validator [:sequential undo-op-schema]))
+(def ^:private undo-ops-validator (m/validator [:sequential undo-op-schema]))
 
-(defn reverse-op
+(def ^:private entity-map-pull-pattern
+  [:block/uuid
+   {:block/left [:block/uuid]}
+   {:block/parent [:block/uuid]}
+   :block/content
+   :block/created-at
+   :block/updated-at
+   :block/format
+   {:block/tags [:block/uuid]}])
+
+(defn- ->block-entity-map
+  [db eid]
+  (let [m (d/pull db entity-map-pull-pattern eid)]
+    (cond-> m
+      true                  (update :block/left :block/uuid)
+      true                  (update :block/parent :block/uuid)
+      (seq (:block/tags m)) (update :block/tags (partial mapv :block/uuid)))))
+
+(defn- reverse-op
   [db op]
   (let [block-uuid (:block-uuid (second op))]
     (case (first op)
-      :boundary op
+      ::boundary op
 
-      :insert-block
-      [:remove-block
+      ::insert-block
+      [::remove-block
        {:block-uuid block-uuid
-        :block-entity-map (d/pull db [:block/uuid
-                                      {:block/left [:block/uuid]}
-                                      {:block/parent [:block/uuid]}
-                                      :block/created-at
-                                      :block/updated-at
-                                      :block/format
-                                      :block/properties
-                                      {:block/tags [:block/uuid]}
-                                      :block/content
-                                      {:block/page [:block/uuid]}]
-                                  [:block/uuid block-uuid])}]
-
-      :move-block
+        :block-entity-map (->block-entity-map db [:block/uuid block-uuid])}]
+
+      ::move-block
       (let [b (d/entity db [:block/uuid block-uuid])]
-        [:move-block
+        [::move-block
          {:block-uuid block-uuid
           :block-origin-left (:block/uuid (:block/left b))
           :block-origin-parent (:block/uuid (:block/parent b))}])
 
-      :remove-block
-      [:insert-block {:block-uuid block-uuid}]
+      ::remove-block
+      [::insert-block {:block-uuid block-uuid}]
 
-      :update-block
-      (let [block-origin-content (when (:block-origin-content op)
+      ::update-block
+      (let [block-origin-content (when (:block-origin-content (second op))
                                    (:block/content (d/entity db [:block/uuid block-uuid])))]
-        [:update-block
+        [::update-block
          (cond-> {:block-uuid block-uuid}
            block-origin-content (assoc :block-origin-content block-origin-content))]))))
 
-
 (def ^:private apply-conj-vec (partial apply (fnil conj [])))
 
 (defn- push-undo-ops
   [repo ops]
+  (assert (undo-ops-validator ops) ops)
   (swap! (:undo/repo->undo-stack @worker-state/*state) update repo apply-conj-vec ops))
 
-(defn- pop-undo-op
+(defn- pop-ops-helper
+  [stack]
+  (let [[ops i]
+        (loop [i (dec (count stack)) r []]
+          (let [peek-op (nth stack i nil)]
+            (cond
+              (neg? i)
+              [r 0]
+
+              (nil? peek-op)
+              [r i]
+
+              (= boundary peek-op)
+              [r i]
+
+              :else
+              (recur (dec i) (conj r peek-op)))))]
+    [ops (subvec stack 0 i)]))
+
+(defn- pop-undo-ops
   [repo]
-  (let [repo->undo-stack (:undo/repo->undo-stack @worker-state/*state)]
-    (when-let [peek-op (peek (@repo->undo-stack repo))]
-      (swap! repo->undo-stack update repo pop)
-      peek-op)))
+  (let [repo->undo-stack (:undo/repo->undo-stack @worker-state/*state)
+        undo-stack (@repo->undo-stack repo)
+        [ops undo-stack*] (pop-ops-helper undo-stack)]
+    (swap! repo->undo-stack assoc repo undo-stack*)
+    ops))
 
 (defn- push-redo-ops
   [repo ops]
+  (assert (undo-ops-validator ops) ops)
   (swap! (:undo/repo->redo-stack @worker-state/*state) update repo apply-conj-vec ops))
 
-(defn- pop-redo-op
+(defn- pop-redo-ops
   [repo]
-  (let [repo->redo-stack (:undo/repo->redo-stack @worker-state/*state)]
-    (when-let [peek-op (peek (@repo->redo-stack repo))]
-      (swap! repo->redo-stack update repo pop)
-      peek-op)))
-
-
-(defmulti reverse-apply-op (fn [op _conn _repo] (first op)))
-(defmethod reverse-apply-op :remove-block
+  (let [repo->redo-stack (:undo/repo->redo-stack @worker-state/*state)
+        redo-stack (@repo->redo-stack repo)
+        [ops redo-stack*] (pop-ops-helper redo-stack)]
+    (swap! repo->redo-stack assoc repo redo-stack*)
+    ops))
+
+(defmulti ^:private reverse-apply-op (fn [op _conn _repo] (first op)))
+(defmethod reverse-apply-op ::remove-block
   [op conn repo]
   (let [[_ {:keys [block-uuid block-entity-map]}] op]
     (when-let [left-entity (d/entity @conn [:block/uuid (:block/left block-entity-map)])]
@@ -126,17 +187,20 @@
          (outliner-core/insert-blocks! repo conn
                                        [(cond-> {:block/uuid block-uuid
                                                  :block/content (:block/content block-entity-map)
-                                                 :block/created-at (:block/created-at block-entity-map)
-                                                 :block/updated-at (:block/updated-at block-entity-map)
                                                  :block/format :markdown}
+                                          (:block/created-at block-entity-map)
+                                          (assoc :block/created-at (:block/created-at block-entity-map))
+
+                                          (:block/updated-at block-entity-map)
+                                          (assoc :block/updated-at (:block/updated-at block-entity-map))
+
                                           (seq (:block/tags block-entity-map))
                                           (assoc :block/tags (mapv (partial vector :block/uuid)
                                                                    (:block/tags block-entity-map))))]
                                        left-entity {:sibling? sibling? :keep-uuid? true}))
-        :push-undo-redo
-        ))))
+        :push-undo-redo))))
 
-(defmethod reverse-apply-op :insert-block
+(defmethod reverse-apply-op ::insert-block
   [op conn repo]
   (let [[_ {:keys [block-uuid]}] op]
     (when-let [block-entity (d/entity @conn [:block/uuid block-uuid])]
@@ -152,7 +216,7 @@
                                        {:children? false}))
         :push-undo-redo))))
 
-(defmethod reverse-apply-op :move-block
+(defmethod reverse-apply-op ::move-block
   [op conn repo]
   (let [[_ {:keys [block-uuid block-origin-left block-origin-parent]}] op]
     (when-let [block-entity (d/entity @conn [:block/uuid block-uuid])]
@@ -166,7 +230,7 @@
            (outliner-core/move-blocks! repo conn [block-entity] left-entity sibling?))
           :push-undo-redo)))))
 
-(defmethod reverse-apply-op :update-block
+(defmethod reverse-apply-op ::update-block
   [op conn repo]
   (let [[_ {:keys [block-uuid block-origin-content]}] op]
     (when-let [block-entity (d/entity @conn [:block/uuid block-uuid])]
@@ -181,51 +245,40 @@
                                     new-block))
         :push-undo-redo))))
 
-
 (defn undo
   [repo]
-  (when-let [op (pop-undo-op repo)]
+  (if-let [ops (not-empty (pop-undo-ops repo))]
     (let [conn (worker-state/get-datascript-conn repo)
-          rev-op (reverse-op @conn op)]
-      (when (= :push-undo-redo (reverse-apply-op op conn repo))
-        (push-redo-ops repo [rev-op])))))
+          redo-ops-to-push (transient [])]
+      (doseq [op ops]
+        (let [rev-op (reverse-op @conn op)]
+          (when (= :push-undo-redo (reverse-apply-op op conn repo))
+            (conj! redo-ops-to-push rev-op))))
+      (when-let [rev-ops (not-empty (persistent! redo-ops-to-push))]
+        (push-redo-ops repo (cons boundary rev-ops))))
+    (prn "No further undo infomation")))
 
 (defn redo
   [repo]
-  (when-let [op (pop-redo-op repo)]
+  (if-let [ops (not-empty (pop-redo-ops repo))]
     (let [conn (worker-state/get-datascript-conn repo)
-          rev-op (reverse-op @conn op)]
-      (when (= :push-undo-redo (reverse-apply-op op conn repo))
-        (push-undo-ops repo [rev-op])))))
+          undo-ops-to-push (transient [])]
+      (doseq [op ops]
+        (let [rev-op (reverse-op @conn op)]
+          (when (= :push-undo-redo (reverse-apply-op op conn repo))
+            (conj! undo-ops-to-push rev-op))))
+      (when-let [rev-ops (not-empty (persistent! undo-ops-to-push))]
+        (push-undo-ops repo (cons boundary rev-ops))))
+    (prn "No further redo infomation")))
 
 
 ;;; listen db changes and push undo-ops
 
-(def ^:private entity-map-pull-pattern
-  [:block/uuid
-   {:block/left [:block/uuid]}
-   {:block/parent [:block/uuid]}
-   :block/content
-   :block/created-at
-   :block/updated-at
-   :block/format
-   {:block/tags [:block/uuid]}])
-
-(defn- ->block-entity-map
-  [db eid]
-  (let [m (-> (d/pull db entity-map-pull-pattern eid)
-              (update :block/left :block/uuid)
-              (update :block/parent :block/uuid))]
-    (if (seq (:block/tags m))
-      (update m :block/tags (partial mapv :block/uuid))
-      m)))
-
 (defn- normal-block?
   [entity]
   (and (:block/parent entity)
        (:block/left entity)))
 
-
 (defn- entity-datoms=>ops
   [db-before db-after id->attr->datom entity-datoms]
   (when-let [e (ffirst entity-datoms)]
@@ -240,43 +293,65 @@
           (cond
             (and (not add1?) block-uuid
                  (normal-block? entity-before))
-            [[:remove-block
+            [[::remove-block
               {:block-uuid (:block/uuid entity-before)
                :block-entity-map (->block-entity-map db-before e)}]]
 
             (and add1? block-uuid
                  (normal-block? entity-after))
-            [[:insert-block {:block-uuid (:block/uuid entity-after)}]]
+            [[::insert-block {:block-uuid (:block/uuid entity-after)}]]
 
             (and (or add3? add4?)
                  (normal-block? entity-after))
-            (cond-> [[:move-block
+            (cond-> [[::move-block
                       {:block-uuid (:block/uuid entity-after)
                        :block-origin-left (:block/uuid (:block/left entity-before))
                        :block-origin-parent (:block/uuid (:block/parent entity-before))}]]
               (and add2? block-content)
-              (conj [:update-block
+              (conj [::update-block
                      {:block-uuid (:block/uuid entity-after)
                       :block-origin-content (:block/content entity-before)}]))
 
             (and add2? block-content
                  (normal-block? entity-after))
-            [[:update-block
+            [[::update-block
               {:block-uuid (:block/uuid entity-after)
                :block-origin-content (:block/content entity-before)}]]))))))
 
 (defn- generate-undo-ops
-  [repo db-before db-after same-entity-datoms-coll id->attr->datom]
+  [repo db-before db-after same-entity-datoms-coll id->attr->datom gen-boundary-op?]
   (let [ops (mapcat (partial entity-datoms=>ops db-before db-after id->attr->datom) same-entity-datoms-coll)]
-    (assert (undo-ops-validator ops) ops)
     (when (seq ops)
-      (push-undo-ops repo ops))))
-
+      (push-undo-ops repo (if gen-boundary-op? (cons boundary ops) ops)))))
 
 (defmethod db-listener/listen-db-changes :gen-undo-ops
   [_ {:keys [_tx-data tx-meta db-before db-after
              repo id->attr->datom same-entity-datoms-coll]}]
   (when (:gen-undo-op? tx-meta true)
-    (generate-undo-ops repo db-before db-after same-entity-datoms-coll id->attr->datom)))
+    (generate-undo-ops repo db-before db-after same-entity-datoms-coll id->attr->datom
+                       (:gen-undo-boundary-op? tx-meta true))))
 
 ;;; listen db changes and push undo-ops (ends)
+
+(comment
+  (defn- clear-undo-redo-stack
+    []
+    (reset! (:undo/repo->undo-stack @worker-state/*state) {})
+    (reset! (:undo/repo->redo-stack @worker-state/*state) {}))
+  (clear-undo-redo-stack)
+  (add-watch (:undo/repo->undo-stack @worker-state/*state)
+             :xxx
+             (fn [_ _ o n]
+               (cljs.pprint/pprint {:k :undo
+                                    :o o
+                                    :n n})))
+
+  (add-watch (:undo/repo->redo-stack @worker-state/*state)
+             :xxx
+             (fn [_ _ o n]
+               (cljs.pprint/pprint {:k :redo
+                                    :o o
+                                    :n n})))
+
+  (remove-watch (:undo/repo->undo-stack @worker-state/*state) :xxx)
+  (remove-watch (:undo/repo->redo-stack @worker-state/*state) :xxx))

src/test/frontend/worker/rtc/db_listener_test.cljs

@@ -1,6 +1,7 @@
 (ns frontend.worker.rtc.db-listener-test
   (:require [cljs.test :as t :refer [deftest is testing]]
             [datascript.core :as d]
+            [frontend.worker.db-listener :as worker-db-listener]
             [frontend.worker.rtc.db-listener :as subject]
             [logseq.db.frontend.schema :as db-schema]))
 
@@ -8,6 +9,12 @@
 (def empty-db (d/empty-db db-schema/schema-for-db-based-graph))
 
 
+(defn- tx-data=>id->attr->datom
+  [tx-data]
+  (let [datom-vec-coll (map vec tx-data)
+        id->same-entity-datoms (group-by first datom-vec-coll)]
+    (update-vals id->same-entity-datoms #'worker-db-listener/entity-datoms=>attr->datom)))
+
 (deftest entity-datoms=>ops-test
   (testing "remove whiteboard page-block"
     (let [conn (d/conn-from-db empty-db)
@@ -22,4 +29,5 @@
       (is (= [["remove-page" {:block-uuid (str block-uuid)}]]
              (#'subject/entity-datoms=>ops (:db-before remove-whiteboard-page-block)
                                            (:db-after remove-whiteboard-page-block)
+                                           (tx-data=>id->attr->datom (:tx-data remove-whiteboard-page-block))
                                            (map vec (:tx-data remove-whiteboard-page-block))))))))

src/test/frontend/worker/rtc/fixture.cljs

@@ -1,16 +1,17 @@
 (ns frontend.worker.rtc.fixture
-  (:require [cljs.test :as t]
-            [cljs.core.async :as async :refer [<! >! chan go]]
-            [frontend.worker.rtc.mock :as rtc-mock]
-            [frontend.worker.rtc.core :as rtc-core]
-            [frontend.worker.rtc.asset-sync :as asset-sync]
-            [frontend.test.helper :as test-helper]
+  (:require [cljs.core.async :as async :refer [<! >! chan go]]
+            [cljs.test :as t]
             [datascript.core :as d]
+            [frontend.db :as db]
             [frontend.db.conn :as conn]
+            [frontend.state :as state]
+            [frontend.test.helper :as test-helper]
+            [frontend.worker.db-listener :as worker-db-listener]
+            [frontend.worker.rtc.asset-sync :as asset-sync]
+            [frontend.worker.rtc.core :as rtc-core]
             [frontend.worker.rtc.db-listener :as db-listener]
-            [frontend.worker.rtc.op-mem-layer :as op-mem-layer]
-            [frontend.db :as db]
-            [frontend.state :as state]))
+            [frontend.worker.rtc.mock :as rtc-mock]
+            [frontend.worker.rtc.op-mem-layer :as op-mem-layer]))
 
 (def *test-rtc-state (atom nil))
 (def *test-asset-sync-state (atom nil))
@@ -94,8 +95,16 @@
       (d/listen! test-db-conn
                  ::gen-ops
                  (fn [{:keys [tx-data tx-meta db-before db-after]}]
-                   (when (:persist-op? tx-meta true)
-                     (db-listener/generate-rtc-ops test-helper/test-db-name-db-version db-before db-after tx-data)))))
+                   (let [datom-vec-coll (map vec tx-data)
+                         id->same-entity-datoms (group-by first datom-vec-coll)
+                         id-order (distinct (map first datom-vec-coll))
+                         same-entity-datoms-coll (map id->same-entity-datoms id-order)
+                         id->attr->datom (update-vals
+                                          id->same-entity-datoms
+                                          #'worker-db-listener/entity-datoms=>attr->datom)]
+                     (when (:persist-op? tx-meta true)
+                       (db-listener/generate-rtc-ops test-helper/test-db-name-db-version db-before db-after
+                                                     same-entity-datoms-coll id->attr->datom))))))
    :after
    #(when-let [test-db-conn (conn/get-db test-helper/test-db-name-db-version false)]
       (d/unlisten! test-db-conn ::gen-ops))})

src/test/frontend/worker/undo_redo_test.cljs

@@ -6,8 +6,8 @@
 
 (deftest reverse-op-test
   ;; TODO: add tests for undo-redo
-  undo-redo/undo-op-schema
-  undo-redo/reverse-op
+  #'undo-redo/undo-op-schema
+  #'undo-redo/reverse-op
   undo-redo/undo
   undo-redo/redo
   )