Merge branch 'feat/db' into refactor/db-properties-schema

.clj-kondo/config.edn

@@ -126,6 +126,7 @@
              frontend.worker.handler.page worker-page
              frontend.worker.handler.page.rename worker-page-rename
              frontend.worker.handler.file.util wfu
+             frontend.worker.batch-tx batch-tx
              lambdaisland.glogi log
              logseq.common.config common-config
              logseq.common.graph common-graph
@@ -182,6 +183,7 @@
            frontend.test.helper/deftest-async clojure.test/deftest
            frontend.test.helper/with-reset cljs.test/async
            frontend.worker.rtc.idb-keyval-mock/with-reset-idb-keyval-mock cljs.test/async
-           frontend.react/defc clojure.core/defn}
+           frontend.react/defc clojure.core/defn
+           frontend.schema-register/defkeyword cljs.spec.alpha/def}
  :skip-comments true
  :output {:progress true}}

.github/stale-issues.yml

@@ -46,7 +46,7 @@ jobs:
               - **Blog**: https://blog.logseq.com
               - **Docs**: https://docs.logseq.com
 
-            Thanks for your contributions to Logseq! If you have any other [**issues**](https://github.com/logseq/logseq/issues/new/choose) or [**feature requests**](https://discuss.logseq.com/c/feature-requests/), please don't hesitate to [let us know](https://github.com/logseq/logseq/issues/new/choose). We always welcome pull requests too!
+            Thanks for your contributions to Logseq! If you have any other [**issues**](https://github.com/logseq/logseq/issues/new/choose) or [**feature requests**](https://discuss.logseq.com/c/feedback/feature-requests/), please don't hesitate to [let us know](https://github.com/logseq/logseq/issues/new/choose). We always welcome pull requests too!
 
       - name: 'Print outputs'
         run: |
@@ -81,7 +81,7 @@ jobs:
               - **Blog**: https://blog.logseq.com
               - **Docs**: https://docs.logseq.com
 
-            Thanks for your contributions to Logseq! If you have any other [**issues**](https://github.com/logseq/logseq/issues/new/choose) or [**feature requests**](https://discuss.logseq.com/c/feature-requests/), please don't hesitate to [let us know](https://github.com/logseq/logseq/issues/new/choose). We always welcome pull requests too!
+            Thanks for your contributions to Logseq! If you have any other [**issues**](https://github.com/logseq/logseq/issues/new/choose) or [**feature requests**](https://discuss.logseq.com/c/feedback/feature-requests/), please don't hesitate to [let us know](https://github.com/logseq/logseq/issues/new/choose). We always welcome pull requests too!
 
       - name: 'Print outputs'
         run: |

.github/workflows/build-desktop-release.yml

@@ -356,13 +356,6 @@ jobs:
         run: yarn run postinstall
         working-directory: ./static/node_modules/dugite/
 
-      - name: Prepare Code Sign
-        if: ${{ github.repository == 'logseq/logseq' }}
-        run: |
-          [IO.File]::WriteAllBytes($(Get-Location).Path + "\codesign.pfx", [Convert]::FromBase64String($env:CERTIFICATE))
-        env:
-          CERTIFICATE: ${{ secrets.CODE_SIGN_CERTIFICATE }}
-
       - name: Build/Release Electron app
         run: yarn electron:make
         working-directory: ./static
@@ -550,9 +543,33 @@ jobs:
       ANDROID_KEYSTORE_PASSWORD: "${{ secrets.ANDROID_KEYSTORE_PASSWORD }}"
       SENTRY_AUTH_TOKEN: "${{ secrets.SENTRY_AUTH_TOKEN }}"
 
+  codesign-windows:
+    if: ${{ github.event_name == 'schedule' || github.event.inputs.build-target == 'nightly' || github.event.inputs.build-target == 'beta' }}
+    needs: [ build-windows ]
+    runs-on: [self-hosted, macos, token]
+    steps:
+      - name: Download Windows Artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: logseq-win64-builds
+          path: ./builds
+
+      - name: Sign Windows Executable
+        run: |
+          ls -lah ./builds
+          jsign --storetype ETOKEN --storepass "${PASS}" -t http://timestamp.digicert.com ./builds/*.exe
+        env:
+          PASS: ${{ secrets.CODE_SIGN_CERTIFICATE_PASSWORD }}
+
+      - name: Upload Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: logseq-win64-signed-builds
+          path: builds
+
   nightly-release:
     if: ${{ github.event_name == 'schedule' || github.event.inputs.build-target == 'nightly' }}
-    needs: [ build-macos-x64, build-macos-arm64, build-linux-x64, build-linux-arm64, build-windows, build-android, e2e-test ]
+    needs: [ build-macos-x64, build-macos-arm64, build-linux-x64, build-linux-arm64, codesign-windows, build-android, e2e-test ]
     runs-on: ubuntu-20.04
     steps:
       - name: Download MacOS x64 Artifacts
@@ -579,10 +596,10 @@ jobs:
           name: logseq-linux-arm64-builds
           path: ./
 
-      - name: Download The Windows Artifact
+      - name: Download The Windows Artifact (Signed)
         uses: actions/download-artifact@v3
         with:
-          name: logseq-win64-builds
+          name: logseq-win64-signed-builds
           path: ./
 
       - name: Download Android Artifacts
@@ -625,7 +642,7 @@ jobs:
   release:
     # NOTE: For now, we only have beta channel to be released on Github
     if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.build-target == 'beta' }}
-    needs: [ build-macos-x64, build-macos-arm64, build-linux-x64, build-linux-arm64, build-windows, e2e-test ]
+    needs: [ build-macos-x64, build-macos-arm64, build-linux-x64, build-linux-arm64, codesign-windows, build-android, e2e-test ]
     runs-on: ubuntu-20.04
     steps:
       - name: Download MacOS x64 Artifacts
@@ -655,7 +672,7 @@ jobs:
       - name: Download The Windows Artifact
         uses: actions/download-artifact@v3
         with:
-          name: logseq-win64-builds
+          name: logseq-win64-signed-builds
           path: ./
 
       - name: Download Android Artifacts

.github/workflows/stale-issues.yml

@@ -46,7 +46,7 @@ jobs:
               - **Blog**: https://blog.logseq.com
               - **Docs**: https://docs.logseq.com
 
-            Thanks for your contributions to Logseq! If you have any other [**issues**](https://github.com/logseq/logseq/issues/new/choose) or [**feature requests**](https://discuss.logseq.com/c/feature-requests/), please don't hesitate to [let us know](https://github.com/logseq/logseq/issues/new/choose). We always welcome pull requests too!
+            Thanks for your contributions to Logseq! If you have any other [**issues**](https://github.com/logseq/logseq/issues/new/choose) or [**feature requests**](https://discuss.logseq.com/c/feedback/feature-requests/), please don't hesitate to [let us know](https://github.com/logseq/logseq/issues/new/choose). We always welcome pull requests too!
 
       - name: '🧹 Close stale awaiting response issues'
         id: awaiting_issues
@@ -74,5 +74,5 @@ jobs:
               - **Blog**: https://blog.logseq.com
               - **Docs**: https://docs.logseq.com
 
-            Thanks for your contributions to Logseq! If you have any other [**issues**](https://github.com/logseq/logseq/issues/new/choose) or [**feature requests**](https://discuss.logseq.com/c/feature-requests/), please don't hesitate to [let us know](https://github.com/logseq/logseq/issues/new/choose). We always welcome pull requests too!
+            Thanks for your contributions to Logseq! If you have any other [**issues**](https://github.com/logseq/logseq/issues/new/choose) or [**feature requests**](https://discuss.logseq.com/c/feedback/feature-requests/), please don't hesitate to [let us know](https://github.com/logseq/logseq/issues/new/choose). We always welcome pull requests too!
 

CONTRIBUTING.md

@@ -168,7 +168,7 @@ Your contributions to open source, large or small, make great projects like this
 [github]: https://github.com/logseq/logseq "Logseq Repo"
 [discord]: https://discord.gg/KpN4eHY "Logseq Discord Server"
 [individual-cla]: https://cla-assistant.io/logseq/logseq "Individual CLA"
-[feature-request]: https://discuss.logseq.com/c/feature-requests/ "Submit Feature Request"
+[feature-request]: https://discuss.logseq.com/c/feedback/feature-requests/ "Submit Feature Request"
 [forum]: https://discuss.logseq.com "Logseq Forum"
 [search-pr]: https://github.com/logseq/logseq/pulls "Search open PRs"
 [new-issue]: https://github.com/logseq/logseq/issues/new?assignees=&labels=&template=bug_report.yaml "Submit a New issue"

bb.edn

@@ -136,6 +136,12 @@
   dev:lint
   logseq.tasks.dev/lint
 
+  dev:test
+  logseq.tasks.dev/test
+
+  dev:lint-and-test
+  logseq.tasks.dev/lint-and-test
+
   dev:gen-malli-kondo-config
   logseq.tasks.dev/gen-malli-kondo-config
 

deps/graph-parser/src/logseq/graph_parser/block.cljs

@@ -155,6 +155,19 @@
      properties)
     []))
 
+(defn- extract-refs-from-property-value
+  [value format]
+  (cond
+    (coll? value)
+    (filter (fn [v] (and (string? v) (not (string/blank? v)))) value)
+    (and (string? value) (= \" (first value) (last value)))
+    nil
+    (string? value)
+    (let [ast (gp-mldoc/inline->edn value (gp-mldoc/default-config format))]
+      (text/extract-refs-from-mldoc-ast ast))
+    :else
+    nil))
+
 (defn- get-page-ref-names-from-properties
   [properties user-config]
   (let [page-refs (->>
@@ -168,8 +181,9 @@
                               (keyword k))))
                    ;; get links ast
                    (map last)
-                   (mapcat (or (:extract-refs-from-property-value-fn user-config)
-                               text/extract-refs-from-mldoc-ast))
+                   (mapcat (fn [value]
+                             (let [f (or (:extract-refs-from-property-value-fn user-config) extract-refs-from-property-value)]
+                               (f value (get user-config :format :markdown)))))
                    ;; comma separated collections
                    (concat (->> (map second properties)
                                 (filter coll?)
@@ -530,11 +544,7 @@
                          id (get-custom-id-or-new-id {:properties properties})
                          property-refs (->> (get-page-refs-from-properties
                                              properties db date-formatter
-                                             (assoc user-config
-                                                    :extract-refs-from-property-value-fn
-                                                    (fn [refs]
-                                                      (when (coll? refs)
-                                                        refs))))
+                                             user-config)
                                             (map :block/original-name))
                          pre-block? (if (:heading properties) false true)
                          block {:block/uuid id

deps/shui/src/logseq/shui/dialog/core.cljs

@@ -104,8 +104,11 @@
 ;; components
 (rum/defc modal-inner
   [config]
-  (let [{:keys [id title description content footer on-open-change open?]} config
-        props (dissoc config :id :title :description :content :footer :on-open-change :open?)]
+  (let [{:keys [id title description content footer on-open-change align open?]} config
+        props (dissoc config
+                :id :title :description :content :footer
+                :align :on-open-change :open?)
+        props (assoc-in props [:overlay-props :data-align] (name (or align :center)))]
 
     (rum/use-effect!
       (fn []

deps/shui/src/logseq/shui/popup/core.cljs

@@ -196,4 +196,4 @@
     [:<>
      (for [config popups
            :when (and (map? config) (:id config))]
-       (x-popup config))]))
+       (rum/with-key (x-popup config) (:id config)))]))

libs/src/LSPlugin.ts

@@ -623,6 +623,8 @@ export interface IEditorProxy extends Record<string, any> {
 
   getSelectedBlocks: () => Promise<Array<BlockEntity> | null>
 
+  clearSelectedBlocks: () => Promise<void>
+
   /**
    * get all blocks of the current page as a tree structure
    *

libs/src/modules/LSPlugin.Experiments.ts

@@ -21,7 +21,7 @@ export class LSPluginExperiments {
   get Components() {
     const exper = this.ensureHostScope().logseq.sdk.experiments
     return {
-      Editor: exper.cp_page_editor as (props: { page: string }) => any
+      Editor: exper.cp_page_editor as (props: { page: string } & any) => any
     }
   }
 

package.json

@@ -61,7 +61,7 @@
         "cljs:electron-watch": "clojure -M:cljs watch app electron --config-merge \"{:asset-path \\\"./js\\\"}\"",
         "cljs:release": "clojure -M:cljs release app publishing electron",
         "cljs:release-electron": "clojure -M:cljs release app electron --debug && clojure -M:cljs release publishing",
-        "cljs:release-app": "clojure -M:cljs release app --config-merge \"{:compiler-options {:output-feature-set :es6}}\"",
+        "cljs:release-app": "clojure -M:cljs release app",
         "cljs:release-publishing": "clojure -M:cljs release publishing",
         "cljs:test": "clojure -M:test compile test",
         "cljs:run-test": "node static/tests.js",

packages/ui/@/components/ui/dialog.tsx

@@ -1,7 +1,7 @@
 import * as React from 'react'
 import * as DialogPrimitive from '@radix-ui/react-dialog'
 import { X } from 'lucide-react'
-import { cn } from '../../lib/utils'
+import { cn } from '@/lib/utils'
 
 const Dialog = DialogPrimitive.Root
 
@@ -19,7 +19,7 @@ const DialogOverlay = React.forwardRef<
     ref={ref}
     className={cn(
       'ui__dialog-overlay',
-      'fixed inset-0 z-50 bg-background/90 data-[state=open]:animate-in ' +
+      'fixed inset-0 z-50 bg-background/90 data-[state=open]:animate-in flex justify-center items-center ' +
       'data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0',
       className
     )}
@@ -30,20 +30,17 @@ DialogOverlay.displayName = DialogPrimitive.Overlay.displayName
 
 const DialogContent = React.forwardRef<
   React.ElementRef<typeof DialogPrimitive.Content>,
-  React.ComponentPropsWithoutRef<typeof DialogPrimitive.Content>
->(({ className, children, ...props }, ref) => (
+  React.ComponentPropsWithoutRef<typeof DialogPrimitive.Content & any>
+>(({ className, children, overlayProps, ...props }, ref) => (
   <DialogPortal>
-    <DialogOverlay>
+    <DialogOverlay {...overlayProps}>
       <DialogPrimitive.Content
         ref={ref}
         className={cn(
           'ui__dialog-content',
-          'fixed left-[50%] top-[50%] z-50 grid w-full max-w-2xl lg:max-w-3xl translate-x-[-50%] translate-y-[-50%] gap-4 border ' +
-          'bg-background p-6 shadow-lg duration-200 data-[state=open]:animate-in data-[state=closed]:animate-out ' +
-          'data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 ' +
-          'data-[state=open]:zoom-in-95 data-[state=closed]:slide-out-to-left-1/2 ' +
-          'data-[state=closed]:slide-out-to-top-[48%] data-[state=open]:slide-in-from-left-1/2 data-[state=open]:slide-in-from-top-[48%] ' +
-          'sm:rounded-lg',
+          'relative grid w-full max-w-2xl lg:max-w-3xl gap-4 border sm:rounded-lg ' +
+          'bg-background p-6 shadow-lg duration-200 data-[state=open]:animate-in ' +
+          'data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95 ',
           className
         )}
         {...props}

packages/ui/@/components/ui/input.tsx

@@ -6,14 +6,15 @@ import { cn } from '@/lib/utils'
 export interface InputProps
   extends React.InputHTMLAttributes<HTMLInputElement> {}
 
-const Input = React.forwardRef<HTMLInputElement, InputProps>(
-  ({ className, type, ...props }, ref) => {
+const Input = React.forwardRef<HTMLInputElement, InputProps & any>(
+  ({ className, type, small, ...props }, ref) => {
     return (
       <input
         type={type}
         className={cn(
           'ui__input',
-          'flex h-10 w-full rounded-md border border-input bg-background px-3 py-2 text-sm ring-offset-background ' +
+          (small ? 'h-8 py-1 px-2' : 'h-10 px-3 py-2'),
+          'flex w-full rounded-md border border-input bg-background text-sm ring-offset-background ' +
           'file:border-0 file:bg-transparent file:text-sm file:font-medium focus:border-input ' +
           'placeholder:text-muted-foreground focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring ' +
           'focus-visible:ring-offset-2 focus-visible:ring-offset-background disabled:cursor-not-allowed disabled:opacity-50',

resources/css/shui.css

@@ -225,13 +225,14 @@ html[data-theme=dark] {
   }
 }
 
-.ui__dialog-content {
-  &[side=start] {
-    @apply translate-y-0 top-[56px];
-  }
-
-  &[side=end] {
+.ui__dialog-overlay {
+  &[data-align=start],
+  &[data-align=top] {
+    @apply !items-start;
 
+    .ui__dialog-content {
+      @apply top-20;
+    }
   }
 }
 

scripts/src/logseq/tasks/dev.clj

@@ -27,6 +27,19 @@
     (println cmd)
     (shell cmd)))
 
+(defn test
+  "Run tests. Pass args through to cmd 'yarn cljs:run-test'"
+  [& args]
+  (shell "yarn cljs:test")
+  (apply shell "yarn cljs:run-test" args))
+
+(defn lint-and-test
+  "Run all lint tasks, then run tests(exclude testcases tagged by :long).
+  pass args through to cmd 'yarn cljs:run-test'"
+  []
+  (lint)
+  (test "-e" "long"))
+
 
 (defn gen-malli-kondo-config
   "Generate clj-kondo type-mismatch config from malli schema

src/main/frontend/common_keywords.cljs

@@ -0,0 +1,39 @@
+(ns frontend.common-keywords
+  "There are some keywords scattered throughout the codebase."
+  (:require [frontend.schema-register :include-macros true :as sr]))
+
+
+(sr/defkeyword :block/uuid
+  "block's uuid"
+  :uuid)
+
+(sr/defkeyword :block/name
+  "block name, lowercase, only page-blocks have this attr"
+  :string)
+
+(sr/defkeyword :block/original-name
+  "like `:block/name`, but not unified into lowercase"
+  :string)
+
+(sr/defkeyword :block/type
+  "block type"
+  [:enum #{"property"} #{"class"} #{"whiteboard"} #{"hidden"}])
+
+(sr/defkeyword :block/parent
+  "page blocks don't have this attr")
+
+(sr/defkeyword :block/left
+  "
+- page blocks don't have this attr
+- some no-order blocks don't have this attr too,
+  TODO: list these types")
+
+(sr/defkeyword :block/content
+  "content string of the blocks.
+in db-version, page-references(e.g. [[page-name]]) are stored as [[~^uuid]]."
+  :string)
+
+(sr/defkeyword :block/raw-content
+  "like `:block/content`,
+but when eval `(:block/raw-content block-entity)`, return raw-content of this block"
+  :string)

src/main/frontend/components/block.cljs

@@ -1814,11 +1814,14 @@
                        (state/toggle-collapsed-block! uuid)
                        (if collapsed?
                          (editor-handler/expand-block! uuid)
-                         (editor-handler/collapse-block! uuid))))}
+                         (editor-handler/collapse-block! uuid)))
+                     ;; debug config context
+                     (when (and (state/developer-mode?) (.-metaKey event))
+                       (js/console.debug "[block config]==" config)))}
         [:span {:class (if (or (and control-show?
-                                    (or collapsed?
-                                        (editor-handler/collapsable? uuid {:semantic? true})))
-                               (and collapsed? (or order-list? config/publishing?)))
+                                 (or collapsed?
+                                   (editor-handler/collapsable? uuid {:semantic? true})))
+                             (and collapsed? (or order-list? config/publishing?)))
                          "control-show cursor-pointer"
                          "control-hide")}
          (ui/rotating-arrow collapsed?)]])
@@ -2545,9 +2548,12 @@
                                                    (p/do!
                                                     (state/set-editor-op! :escape)
                                                     (editor-handler/save-block! (editor-handler/get-state) value)
-                                                    (js/setTimeout #(editor-handler/escape-editing select?) 10))))}
-                                     edit-input-id
-                                     config))]
+                                                    (js/setTimeout (fn []
+                                                                     (editor-handler/escape-editing select?)
+                                                                     (some-> config :on-escape-editing
+                                                                       (apply [(str uuid) (= event :esc)]))) 10))))}
+                           edit-input-id
+                           config))]
           [:div.flex.flex-1.flex-row.gap-1.items-start
            editor-cp
            (when (and (seq (:block/tags block)) db-based?)
@@ -2559,8 +2565,8 @@
           [:div.flex.flex-row
            [:div.flex-1.w-full {:style {:display (if (:slide? config) "block" "flex")}}
             (ui/catch-error
-             (ui/block-error "Block Render Error:"
-                             {:content (:block/content block)
+              (ui/block-error "Block Render Error:"
+                {:content (:block/content block)
                               :section-attrs
                               {:on-click #(let [content (or (:block/original-name block)
                                                             (:block/content block))]

src/main/frontend/components/container.cljs

@@ -763,7 +763,7 @@
    {:title "Documentation" :icon "help" :href "https://docs.logseq.com/"}
    :hr
    {:title "Report bug" :icon "bug" :on-click #(rfe/push-state :bug-report)}
-   {:title "Request feature" :icon "git-pull-request" :href "https://discuss.logseq.com/c/feature-requests/"}
+   {:title "Request feature" :icon "git-pull-request" :href "https://discuss.logseq.com/c/feedback/feature-requests/"}
    {:title "Submit feedback" :icon "messages" :href "https://discuss.logseq.com/c/feedback/13"}
    :hr
    {:title "Ask the community" :icon "brand-discord" :href "https://discord.com/invite/KpN4eHY"}

src/main/frontend/components/container.css

@@ -602,8 +602,8 @@
   user-select: none;
 
   .resizer {
-    @apply absolute top-0 bottom-0 touch-none left-[1px] w-[3px] select-none !bg-primary;
-    @apply !cursor-col-resize hover:bg-primary/90 focus:bg-primary/90 active:bg-primary/90;
+    @apply absolute top-0 bottom-0 touch-none left-[1px] w-[3px] select-none;
+    @apply cursor-col-resize hover:bg-primary/90 focus:bg-primary/90 active:bg-primary/90;
     @apply z-[1000] delay-300 transition-[background-color] duration-300;
   }
 

src/main/frontend/components/journal.css

@@ -6,10 +6,14 @@
   }
 
   .journal-item {
-    @apply border-t min-h-[250px];
+    @apply border-b min-h-[250px] pb-[64px] mb-[38px];
 
     &:first-child {
-      @apply pt-0 border-none min-h-[500px];
+      @apply pt-0 min-h-[500px];
+    }
+
+    &:last-child {
+      @apply border-none;
     }
   }
 }

src/main/frontend/components/onboarding.cljs

@@ -34,7 +34,7 @@
           {:title (t :help/title-development)
            :children [[(t :help/roadmap) "https://trello.com/b/8txSM12G/roadmap"]
                       [(t :help/bug) "https://github.com/logseq/logseq/issues/new?labels=from:in-app&template=bug_report.yaml"]
-                      [(t :help/feature) "https://discuss.logseq.com/c/feature-requests/"]
+                      [(t :help/feature) "https://discuss.logseq.com/c/feedback/feature-requests/"]
                       [(t :help/changelog) "https://docs.logseq.com/#/page/changelog"]]}
 
           {:title (t :help/title-about)

src/main/frontend/components/page.cljs

@@ -449,7 +449,7 @@
                  page-name' (get-sanity-page-name state page-name)]
              (db-async/<get-block (state/get-current-repo) page-name')
              (assoc state ::page-name page-name')))}
-  [state {:keys [repo page-name preview? sidebar?] :as option}]
+  [state {:keys [repo page-name config preview? sidebar?] :as option}]
   (let [loading? (when (::page-name state)  (state/sub-async-query-loading (::page-name state)))]
     (when-let [path-page-name (get-path-page-name state page-name)]
       (let [current-repo (state/sub :git/current-repo)
@@ -537,7 +537,7 @@
                      (let [_ (and block? page (reset! *current-block-page (:block/name (:block/page page))))
                            _ (when (and block? (not page))
                                (route-handler/redirect-to-page! @*current-block-page))]
-                       (page-blocks-cp repo page {:sidebar? sidebar? :whiteboard? whiteboard?})))]])
+                       (page-blocks-cp repo page (merge config {:sidebar? sidebar? :whiteboard? whiteboard?}))))]])
 
                (when today?
                  (today-queries repo today? sidebar?))

src/main/frontend/components/page.css

@@ -416,7 +416,7 @@ html.is-native-ios {
   }
 
   .ls-new-property {
-    @apply mt-1;
+    @apply mt-[3px];
   }
 
   &.is-collapsed {
@@ -452,6 +452,12 @@ html.is-native-ios {
   .info-title {
     @apply relative min-h-[28px] flex items-center pl-1;
   }
+
+  .ls-properties-area {
+    &:has(.property-pair) {
+      @apply pt-2.5;
+    }
+  }
 }
 
 .page-info-title-placeholder {

src/main/frontend/components/plugins.cljs

@@ -377,19 +377,19 @@
                     (reset! *search-key nil)
                     (.focus target))}
       (ui/icon "x")])
-   [:input.form-input.is-small
-    {:placeholder (t :plugin/search-plugin)
-     :ref         *search-ref
-     :auto-focus  true
-     :on-key-down (fn [^js e]
-                    (when (= 27 (.-keyCode e))
-                      (util/stop e)
-                      (if (string/blank? search-key)
-                        (some-> (js/document.querySelector ".cp__plugins-page") (.focus))
-                        (reset! *search-key nil))))
-     :on-change   #(let [^js target (.-target %)]
-                     (reset! *search-key (some-> (.-value target) (string/triml))))
-     :value       (or search-key "")}]])
+   (shui/input
+     {:placeholder (t :plugin/search-plugin)
+      :ref *search-ref
+      :auto-focus true
+      :on-key-down (fn [^js e]
+                     (when (= 27 (.-keyCode e))
+                       (util/stop e)
+                       (if (string/blank? search-key)
+                         (some-> (js/document.querySelector ".cp__plugins-page") (.focus))
+                         (reset! *search-key nil))))
+      :on-change #(let [^js target (.-target %)]
+                    (reset! *search-key (some-> (.-value target) (string/triml))))
+      :value (or search-key "")})])
 
 (rum/defc panel-tab-developer
   []
@@ -1407,7 +1407,7 @@
   (shui/dialog-open!
     (plugins-page)
     {:label :plugins-dashboard
-     :side :center}))
+     :align :start}))
 
 (defn open-waiting-updates-modal!
   []
@@ -1431,6 +1431,7 @@
       [:div.settings-modal.of-plugins
        (focused-settings-content title)])
     {:label   "plugin-settings-modal"
+     :align   :start
      :id      "ls-focused-settings-modal"}))
 
 (defn hook-custom-routes

src/main/frontend/components/plugins.css

@@ -150,10 +150,11 @@
         z-index: 1;
       }
 
-      .form-input {
+      .ui__input {
         background-color: var(--ls-primary-background-color);
         padding: 6px 7px 5px 29px;
-        opacity: .5;
+        opacity: .7;
+        height: 30px;
 
         &:focus {
           background-color: var(--ls-secondary-background-color);
@@ -963,8 +964,7 @@ html[data-theme='dark'] {
 }
 
 .ui__dialog-content[label=plugin-settings-modal] {
-  @apply w-auto lg:max-w-5xl p-0 gap-0
-  top-[10%] translate-y-0 !animate-none;
+  @apply w-auto lg:max-w-5xl p-0 gap-0;
 }
 
 .ui__dialog-content[label=plugins-dashboard] {

src/main/frontend/components/property.css

@@ -47,7 +47,7 @@
 }
 
 .ls-properties-area {
-    @apply grid gap-0.5 pt-2 pb-1.5;
+    @apply grid gap-0.5 pt-1.5 pb-1.5;
 
     .property-pair {
         @apply grid grid-cols-5 gap-1;

src/main/frontend/components/property/value.cljs

@@ -755,7 +755,7 @@
                     (if (seq items)
                       (concat
                        (for [item items]
-                         (select-item property type item opts))
+                         (rum/with-key (select-item property type item opts) (or (:block/uuid item) (str item))))
                        (when date?
                          [(property-value-date-picker block property nil {:toggle-fn toggle-fn})]))
                       (when-not editing?

src/main/frontend/components/reference.cljs

@@ -14,6 +14,7 @@
             [frontend.search :as search]
             [frontend.state :as state]
             [frontend.ui :as ui]
+            [logseq.shui.ui :as shui]
             [frontend.util :as util]
             [rum.core :as rum]
             [frontend.modules.outliner.tree :as tree]
@@ -78,11 +79,11 @@
           [:div.flex.flex-row.flex-wrap
            [:div.mr-1.font-medium.py-1 (t :linked-references/filter-excludes)]
            (filtered-refs page-name filters filters-atom excludes)])])
-     [:div.cp__filters-input-panel.flex
+     [:div.cp__filters-input-panel.flex.focus-within:bg-gray-03
       (ui/icon "search")
-      [:input.cp__filters-input.w-full
+      [:input.cp__filters-input.w-full.bg-transparent
        {:placeholder (t :linked-references/filter-search)
-        :auto-focus true
+        :autofocus true
         :on-change (fn [e]
                      (reset! filter-search (util/evalue e)))}]]
      (let [all-filters (set (keys filters))
@@ -150,10 +151,9 @@
                            ;; expand
                            (reset! @*collapsed? false)))
         :on-pointer-down (fn [e]
-                         (util/stop-propagation e))
-        :on-click (fn []
-                    (state/set-modal! (filter-dialog filters-atom *ref-pages page-name)
-                                      {:center? true}))}
+                           (util/stop-propagation e)
+                           (shui/dialog-open!
+                             (filter-dialog filters-atom *ref-pages page-name)))}
        (ui/icon "filter" {:class (cond
                                    (empty? filter-state)
                                    "opacity-60 hover:opacity-100"

src/main/frontend/components/reference.css

@@ -1,5 +1,4 @@
 .cp__filters-input {
-  background-color: var(--ls-primary-background-color);
   padding: 0.5rem;
   outline: none;
 }

src/main/frontend/core.cljs

@@ -1,21 +1,23 @@
 (ns frontend.core
   "Entry ns for the mobile, browser and electron frontend apps"
   {:dev/always true}
-  (:require [rum.core :as rum]
+  (:require [frontend.common-keywords]
+            [frontend.components.plugins :as plugins]
+            [frontend.config :as config]
+            [frontend.fs.sync :as sync]
             [frontend.handler :as handler]
             [frontend.handler.plugin :as plugin-handler]
             [frontend.handler.route :as route-handler]
-            [frontend.components.plugins :as plugins]
+            [frontend.log]
             [frontend.page :as page]
             [frontend.routes :as routes]
+            [frontend.schema-register :as sr]
             [frontend.spec]
-            [frontend.log]
+            [logseq.api]
+            [malli.dev.cljs :as md]
             [reitit.frontend :as rf]
             [reitit.frontend.easy :as rfe]
-            [logseq.api]
-            [frontend.fs.sync :as sync]
-            [frontend.config :as config]
-            [malli.dev.cljs :as md]))
+            [rum.core :as rum]))
 
 (defn set-router!
   []
@@ -47,6 +49,7 @@
 (defn ^:export start []
   (when config/dev?
     (md/start!))
+  (frontend.schema-register/init)
   (when-let [node (.getElementById js/document "root")]
     (set-router!)
     (rum/mount (page/current-page) node)

src/main/frontend/db_worker.cljs

@@ -17,13 +17,13 @@
             [frontend.worker.handler.page :as worker-page]
             [frontend.worker.handler.page.rename :as worker-page-rename]
             [frontend.worker.rtc.core :as rtc-core]
-            [frontend.worker.rtc.db-listener :as rtc-db-listener]
+            [frontend.worker.rtc.db-listener]
             [frontend.worker.rtc.full-upload-download-graph :as rtc-updown]
             [frontend.worker.rtc.op-mem-layer :as op-mem-layer]
             [frontend.worker.rtc.snapshot :as rtc-snapshot]
             [frontend.worker.search :as search]
             [frontend.worker.state :as worker-state]
-            [frontend.worker.undo-redo]
+            [frontend.worker.undo-redo :as undo-redo]
             [frontend.worker.util :as worker-util]
             [logseq.db :as ldb]
             [logseq.db.sqlite.common-db :as sqlite-common-db]
@@ -176,9 +176,7 @@
             conn (sqlite-common-db/get-storage-conn storage schema)]
         (swap! *datascript-conns assoc repo conn)
         (p/let [_ (op-mem-layer/<init-load-from-indexeddb! repo)]
-          (rtc-db-listener/listen-to-db-changes! repo conn)
-          (db-listener/listen-db-changes! repo conn))
-        ))))
+          (db-listener/listen-db-changes! repo conn))))))
 
 (defn- iter->vec [iter]
   (when iter
@@ -588,7 +586,6 @@
          (try
            (let [state (<? (rtc-core/<init-state token false))
                  r (<? (rtc-updown/<async-upload-graph state repo conn remote-graph-name))]
-             (rtc-db-listener/listen-db-to-generate-ops repo conn)
              (p/resolve! d r))
            (catch :default e
              (worker-util/post-message :notification
@@ -677,6 +674,17 @@
    [_this block-uuid]
    (transit/write transit-w (rtc-core/get-block-update-log (uuid block-uuid))))
 
+  (undo
+   [_this repo]
+   (when-let [conn (worker-state/get-datascript-conn repo)]
+     (undo-redo/undo repo conn))
+   nil)
+
+  (redo
+   [_this repo]
+   (when-let [conn (worker-state/get-datascript-conn repo)]
+     (undo-redo/redo repo conn)))
+
   (keep-alive
    [_this]
    "alive")

src/main/frontend/handler/history.cljs

@@ -1,13 +1,14 @@
 (ns ^:no-doc frontend.handler.history
-  (:require [frontend.db :as db]
+  (:require [frontend.config :as config]
+            [frontend.db :as db]
+            [frontend.db.transact :as db-transact]
             [frontend.handler.editor :as editor]
+            [frontend.handler.route :as route-handler]
             [frontend.modules.editor.undo-redo :as undo-redo]
             [frontend.state :as state]
             [frontend.util :as util]
-            [frontend.handler.route :as route-handler]
             [goog.dom :as gdom]
-            [promesa.core :as p]
-            [frontend.db.transact :as db-transact]))
+            [promesa.core :as p]))
 
 (defn restore-cursor!
   [{:keys [last-edit-block container pos end-pos]} undo?]
@@ -41,20 +42,28 @@
 
 (defn undo!
   [e]
-  (when (db-transact/request-finished?)
-    (util/stop e)
-    (p/do!
-     (state/set-state! [:editor/last-replace-ref-content-tx (state/get-current-repo)] nil)
-     (editor/save-current-block!)
-     (state/clear-editor-action!)
-     (state/set-block-op-type! nil)
-     (let [cursor-state (undo-redo/undo)]
-       (state/set-state! :ui/restore-cursor-state (select-keys cursor-state [:editor-cursor :app-state]))))))
+  (when-let [repo (state/get-current-repo)]
+    (when (db-transact/request-finished?)
+      (util/stop e)
+      (p/do!
+       (state/set-state! [:editor/last-replace-ref-content-tx repo] nil)
+       (editor/save-current-block!)
+       (state/clear-editor-action!)
+       (state/set-block-op-type! nil)
+       (if (config/db-based-graph? repo)
+         (let [^js worker @state/*db-worker]
+           (.undo worker repo))
+         (let [cursor-state (undo-redo/undo)]
+           (state/set-state! :ui/restore-cursor-state (select-keys cursor-state [:editor-cursor :app-state]))))))))
 
 (defn redo!
   [e]
-  (when (db-transact/request-finished?)
-    (util/stop e)
-    (state/clear-editor-action!)
-    (let [cursor-state (undo-redo/redo)]
-      (state/set-state! :ui/restore-cursor-state (select-keys cursor-state [:editor-cursor :app-state])))))
+  (when-let [repo (state/get-current-repo)]
+    (when (db-transact/request-finished?)
+      (util/stop e)
+      (state/clear-editor-action!)
+      (if (config/db-based-graph? repo)
+        (let [^js worker @state/*db-worker]
+          (.redo worker repo))
+        (let [cursor-state (undo-redo/redo)]
+          (state/set-state! :ui/restore-cursor-state (select-keys cursor-state [:editor-cursor :app-state])))))))

src/main/frontend/schema_register.clj

@@ -0,0 +1,12 @@
+(ns frontend.schema-register
+  "Macro 'defkeyword' to def keyword with docstring and malli-schema")
+
+
+(defmacro defkeyword
+  "Define keyword with docstring and malli-schema"
+  [kw docstring & [optional-malli-schema]]
+  (assert (keyword? kw) "must be keyword")
+  (assert (some? docstring) "must have 'docstring' arg")
+  (when optional-malli-schema
+    `(do (assert (frontend.schema-register/not-register-yet? ~kw) (str "Already registered: " ~kw))
+         (frontend.schema-register/register! ~kw ~optional-malli-schema))))

src/main/frontend/schema_register.cljs

@@ -0,0 +1,23 @@
+(ns frontend.schema-register
+  "Set malli default registry to a mutable one,
+  and use `register!` to add schemas dynamically."
+  (:require [malli.core :as m]
+            [malli.registry :as mr]))
+
+(def *malli-registry (atom {}))
+
+(defn register!
+  [type schema]
+  (swap! *malli-registry assoc type schema))
+
+(defn not-register-yet?
+  [type]
+  (boolean (nil? (@*malli-registry type))))
+
+(defn init
+  []
+  (reset! *malli-registry {})
+  (mr/set-default-registry!
+   (mr/composite-registry
+    (m/default-schemas)
+    (mr/mutable-registry *malli-registry))))

src/main/frontend/ui.css

@@ -315,11 +315,9 @@ html.is-mobile {
 }
 
 .form-input {
-  @apply block w-full pl-2 sm:text-sm sm:leading-5 rounded bg-background border border-gray-07;
-
-  &:focus {
-    box-shadow: 0 0 0 2px var(--tw-shadow-color, rgba(164, 202, 254, 0.45));
-  }
+  @apply block w-full mt-1 pl-2 sm:text-sm sm:leading-5 rounded bg-background border border-gray-07;
+  @apply focus:border-input focus-visible:ring-ring focus-visible:outline-none focus-visible:ring-offset-2;
+  @apply focus-visible:ring-2 ring-offset-background;
 
   &.is-small {
     @apply py-1.5 sm:leading-4 sm:text-xs;

src/main/frontend/worker/batch_tx.clj

@@ -0,0 +1,18 @@
+(ns frontend.worker.batch-tx
+  "Macro for batch-tx fns")
+
+(defmacro with-batch-tx-mode
+  "1. start batch-tx mode
+  2. run body
+  3. exit batch-tx mode
+  4. refresh-ui"
+  [conn & body]
+  `(do (frontend.worker.batch-tx/start-batch-tx-mode)
+       ~@body
+       (let [txs# (frontend.worker.batch-tx/get-batch-txs)]
+         (frontend.worker.batch-tx/exit-batch-tx-mode)
+         (when (seq txs#)
+           (when-let [affected-keys# (not-empty
+                                      (frontend.worker.react/get-affected-queries-keys
+                                       {:db-after @~conn :tx-data txs#}))]
+             (frontend.worker.util/post-message :refresh-ui {:affected-keys affected-keys#}))))))

src/main/frontend/worker/batch_tx.cljs

@@ -0,0 +1,34 @@
+(ns frontend.worker.batch-tx
+  "Batch process multiple transactions.
+  When batch-processing, don't refresh ui."
+  (:require [frontend.worker.state :as worker-state]
+            [frontend.schema-register :include-macros true :as sr]))
+
+
+(sr/defkeyword :tx/batch-processing?
+  "will not sync worker-db-changes to UI when true")
+
+(sr/defkeyword :tx/batch-txs
+  "store all tx-data when batch-processing")
+
+
+(defn start-batch-tx-mode
+  []
+  (swap! worker-state/*state assoc :tx/batch-processing? true))
+
+(defn tx-batch-processing?
+  []
+  (:tx/batch-processing? @worker-state/*state))
+
+(defn get-batch-txs
+  []
+  (:tx/batch-txs @worker-state/*state))
+
+(defn conj-batch-txs!
+  [tx-data]
+  (swap! worker-state/*state update :tx/batch-txs (fn [data] (into data tx-data))))
+
+(defn exit-batch-tx-mode
+  []
+  (swap! worker-state/*state assoc :tx/batch-processing? false)
+  (swap! worker-state/*state assoc :tx/batch-txs nil))

src/main/frontend/worker/db_listener.cljs

@@ -1,6 +1,12 @@
 (ns frontend.worker.db-listener
   "Db listeners for worker-db."
-  (:require [datascript.core :as d]))
+  (:require [cljs-bean.core :as bean]
+            [datascript.core :as d]
+            [frontend.worker.pipeline :as worker-pipeline]
+            [frontend.worker.search :as search]
+            [frontend.worker.state :as worker-state]
+            [frontend.worker.util :as worker-util]
+            [promesa.core :as p]))
 
 
 (defn- entity-datoms=>attr->datom
@@ -27,6 +33,31 @@
 (defmulti listen-db-changes
   (fn [listen-key & _] listen-key))
 
+(defmethod listen-db-changes :sync-db-to-main-thread
+  [_ {:keys [tx-meta repo conn] :as tx-report}]
+  (let [{:keys [pipeline-replace? from-disk?]} tx-meta
+                     result (worker-pipeline/invoke-hooks repo conn tx-report (worker-state/get-context))
+                     tx-report' (or (:tx-report result) tx-report)]
+                 (when-not pipeline-replace?
+                   (let [data (merge
+                               {:request-id (:request-id tx-meta)
+                                :repo repo
+                                :tx-data (:tx-data tx-report')
+                                :tx-meta tx-meta}
+                               (dissoc result :tx-report))]
+                     (worker-util/post-message :sync-db-changes data))
+
+                   (when-not from-disk?
+                     (p/do!
+                      (let [{:keys [blocks-to-remove-set blocks-to-add]} (search/sync-search-indice repo tx-report')
+                            ^js wo (worker-state/get-worker-object)]
+                        (when wo
+                          (when (seq blocks-to-remove-set)
+                            (.search-delete-blocks wo repo (bean/->js blocks-to-remove-set)))
+                          (when (seq blocks-to-add)
+                            (.search-upsert-blocks wo repo (bean/->js blocks-to-add))))))))))
+
+
 (defn listen-db-changes!
   [repo conn]
   (let [handlers (methods listen-db-changes)]
@@ -38,9 +69,11 @@
                        id->same-entity-datoms (group-by first datom-vec-coll)
                        id-order (distinct (map first datom-vec-coll))
                        same-entity-datoms-coll (map id->same-entity-datoms id-order)
-                       id->attr->datom (update-vals id->same-entity-datoms entity-datoms=>attr->datom)]
+                       id->attr->datom (update-vals id->same-entity-datoms entity-datoms=>attr->datom)
+                       args* (assoc args
+                                    :repo repo
+                                    :conn conn
+                                    :id->attr->datom id->attr->datom
+                                    :same-entity-datoms-coll same-entity-datoms-coll)]
                    (doseq [[k handler-fn] handlers]
-                     (handler-fn k (assoc args
-                                          :repo repo
-                                          :id->attr->datom id->attr->datom
-                                          :same-entity-datoms-coll same-entity-datoms-coll))))))))
+                     (handler-fn k args*)))))))

src/main/frontend/worker/handler/page.cljs

@@ -128,7 +128,8 @@
                                  page-txs
                                  first-block-tx)]
                    (when (seq txs)
-                     (ldb/transact! conn txs (cond-> {:persist-op? persist-op?}
+                     (ldb/transact! conn txs (cond-> {:persist-op? persist-op?
+                                                      :outliner-op :create-page}
                                                today-journal?
                                                (assoc :create-today-journal? true
                                                       :today-journal-name page-name))))))] ;; FIXME: prettier validation

src/main/frontend/worker/pipeline.cljs

@@ -1,16 +1,16 @@
 (ns frontend.worker.pipeline
   "Pipeline work after transaction"
   (:require [datascript.core :as d]
-            [logseq.outliner.datascript-report :as ds-report]
-            [logseq.outliner.pipeline :as outliner-pipeline]
-            [frontend.worker.react :as worker-react]
+            [frontend.worker.batch-tx :as batch-tx]
+            [frontend.worker.db.fix :as db-fix]
             [frontend.worker.file :as file]
+            [frontend.worker.react :as worker-react]
             [frontend.worker.util :as worker-util]
-            [frontend.worker.state :as worker-state]
+            [logseq.db :as ldb]
             [logseq.db.frontend.validate :as db-validate]
             [logseq.db.sqlite.util :as sqlite-util]
-            [frontend.worker.db.fix :as db-fix]
-            [logseq.db :as ldb]))
+            [logseq.outliner.datascript-report :as ds-report]
+            [logseq.outliner.pipeline :as outliner-pipeline]))
 
 (defn- path-refs-need-recalculated?
   [tx-meta]
@@ -117,12 +117,12 @@
               final-tx-report (assoc tx-report'
                                      :tx-data full-tx-data
                                      :db-before (:db-before tx-report))
-              batch-processing? (worker-state/rtc-batch-processing?)
+              batch-processing? (batch-tx/tx-batch-processing?)
               affected-query-keys (when-not (or (:importing? context)
                                                 batch-processing?)
                                     (worker-react/get-affected-queries-keys final-tx-report))]
           (when batch-processing?
-            (worker-state/conj-batch-txs! full-tx-data))
+            (batch-tx/conj-batch-txs! full-tx-data))
           {:tx-report final-tx-report
            :affected-keys affected-query-keys
            :deleted-block-uuids deleted-block-uuids

src/main/frontend/worker/rtc/core.cljs

@@ -10,10 +10,10 @@
             [cognitect.transit :as transit]
             [datascript.core :as d]
             [frontend.worker.async-util :include-macros true :refer [<? go-try]]
+            [frontend.worker.batch-tx :include-macros true :as batch-tx]
             [frontend.worker.db-metadata :as worker-db-metadata]
             [frontend.worker.handler.page :as worker-page]
             [frontend.worker.handler.page.rename :as worker-page-rename]
-            [frontend.worker.react :as worker-react]
             [frontend.worker.rtc.asset-sync :as asset-sync]
             [frontend.worker.rtc.const :as rtc-const]
             [frontend.worker.rtc.op-mem-layer :as op-mem-layer]
@@ -125,6 +125,7 @@
 (defmethod transact-db! :delete-blocks [_ & args]
   (outliner-tx/transact!
    {:persist-op? false
+    :gen-undo-op? false
     :outliner-op :delete-blocks
     :transact-opts {:repo (first args)
                     :conn (second args)}}
@@ -133,6 +134,7 @@
 (defmethod transact-db! :move-blocks [_ & args]
   (outliner-tx/transact!
    {:persist-op? false
+    :gen-undo-op? false
     :outliner-op :move-blocks
     :transact-opts {:repo (first args)
                     :conn (second args)}}
@@ -141,6 +143,7 @@
 (defmethod transact-db! :move-blocks&persist-op [_ & args]
   (outliner-tx/transact!
    {:persist-op? true
+    :gen-undo-op? false
     :outliner-op :move-blocks
     :transact-opts {:repo (first args)
                     :conn (second args)}}
@@ -149,6 +152,7 @@
 (defmethod transact-db! :insert-blocks [_ & args]
   (outliner-tx/transact!
       {:persist-op? false
+       :gen-undo-op? false
        :outliner-op :insert-blocks
        :transact-opts {:repo (first args)
                        :conn (second args)}}
@@ -163,12 +167,14 @@
                           ;; must be `logseq.db.frontend.malli-schema.closed-value-block`
                           :block/type #{"closed value"}})
                        block-uuids)
-                 {:persist-op? false}))
+                 {:persist-op? false
+                  :gen-undo-op? false}))
 
 
 (defmethod transact-db! :save-block [_ & args]
   (outliner-tx/transact!
       {:persist-op? false
+       :gen-undo-op? false
        :outliner-op :save-block
        :transact-opts {:repo (first args)
                        :conn (second args)}}
@@ -177,10 +183,12 @@
 (defmethod transact-db! :delete-whiteboard-blocks [_ conn block-uuids]
   (ldb/transact! conn
                  (mapv (fn [block-uuid] [:db/retractEntity [:block/uuid block-uuid]]) block-uuids)
-                 {:persist-op? false}))
+                 {:persist-op? false
+                  :gen-undo-op? false}))
 
 (defmethod transact-db! :upsert-whiteboard-block [_ conn blocks]
-  (ldb/transact! conn blocks {:persist-op? false}))
+  (ldb/transact! conn blocks {:persist-op? false
+                              :gen-undo-op? false}))
 
 (defn- whiteboard-page-block?
   [block]
@@ -444,7 +452,8 @@
                      [:db/retract db-id :block/journal-day]
                      [:db/retract db-id :block/journal?]))
             (when (seq @*other-tx-data)
-              (ldb/transact! conn @*other-tx-data {:persist-op? false}))
+              (ldb/transact! conn @*other-tx-data {:persist-op? false
+                                                   :gen-undo-op? false}))
             (transact-db! :save-block repo conn date-formatter new-block)))))))
 
 (defn apply-remote-move-ops
@@ -480,6 +489,7 @@
     (when (and (seq blocks) target-page-block)
       (outliner-tx/transact!
        {:persist-op? true
+        :gen-undo-op? false
         :transact-opts {:repo repo
                         :conn conn}}
        (outliner-core/move-blocks! repo conn blocks target-page-block false)))))
@@ -631,22 +641,14 @@
               update-page-ops (vals update-page-ops-map)
               remove-page-ops (vals remove-page-ops-map)]
 
-          (worker-state/start-batch-tx-mode!)
-          (js/console.groupCollapsed "rtc/apply-remote-ops-log")
-          (worker-util/profile :apply-remote-update-page-ops (apply-remote-update-page-ops repo conn date-formatter update-page-ops))
-          (worker-util/profile :apply-remote-remove-ops (apply-remote-remove-ops repo conn date-formatter remove-ops))
-          (worker-util/profile :apply-remote-move-ops (apply-remote-move-ops repo conn date-formatter sorted-move-ops))
-          (worker-util/profile :apply-remote-update-ops (apply-remote-update-ops repo conn date-formatter update-ops))
-          (worker-util/profile :apply-remote-remove-page-ops (apply-remote-remove-page-ops repo conn remove-page-ops))
-          (js/console.groupEnd)
-          (let [txs (worker-state/get-batch-txs)]
-            (worker-state/exit-batch-tx-mode!)
-            (when (seq txs)
-              (let [affected-keys (worker-react/get-affected-queries-keys {:db-after @conn
-                                                                           :tx-data txs})]
-                (when (seq affected-keys)
-                  (worker-util/post-message :refresh-ui
-                                            {:affected-keys affected-keys})))))
+          (batch-tx/with-batch-tx-mode conn
+            (js/console.groupCollapsed "rtc/apply-remote-ops-log")
+            (worker-util/profile :apply-remote-update-page-ops (apply-remote-update-page-ops repo conn date-formatter update-page-ops))
+            (worker-util/profile :apply-remote-remove-ops (apply-remote-remove-ops repo conn date-formatter remove-ops))
+            (worker-util/profile :apply-remote-move-ops (apply-remote-move-ops repo conn date-formatter sorted-move-ops))
+            (worker-util/profile :apply-remote-update-ops (apply-remote-update-ops repo conn date-formatter update-ops))
+            (worker-util/profile :apply-remote-remove-page-ops (apply-remote-remove-page-ops repo conn remove-page-ops))
+            (js/console.groupEnd))
 
           (op-mem-layer/update-local-tx! repo remote-t)
           (update-log state {:remote-update-map affected-blocks-map}))

src/main/frontend/worker/rtc/db_listener.cljs

@@ -5,34 +5,9 @@
             [clojure.data :as data]
             [clojure.set :as set]
             [datascript.core :as d]
-            [frontend.worker.rtc.op-mem-layer :as op-mem-layer]
-            [frontend.worker.state :as worker-state]
-            [frontend.worker.pipeline :as worker-pipeline]
-            [frontend.worker.search :as search]
-            [frontend.worker.util :as worker-util]
-            [promesa.core :as p]
-            [cljs-bean.core :as bean]))
-
-
-(defn- entity-datoms=>attr->datom
-  [entity-datoms]
-  (reduce
-   (fn [m datom]
-     (let [[_e a _v t add?] datom]
-       (if-let [[_e _a _v old-t old-add?] (get m a)]
-         (cond
-           (and (= old-t t)
-                (true? add?)
-                (false? old-add?))
-           (assoc m a datom)
-
-           (< old-t t)
-           (assoc m a datom)
-
-           :else
-           m)
-         (assoc m a datom))))
-   {} entity-datoms))
+            [frontend.schema-register :include-macros true :as sr]
+            [frontend.worker.db-listener :as db-listener]
+            [frontend.worker.rtc.op-mem-layer :as op-mem-layer]))
 
 
 (defn- diff-value-of-set-type-attr
@@ -60,11 +35,11 @@
       (seq retract-uuids) (conj [:retract retract-uuids]))))
 
 (defn- entity-datoms=>ops
-  [db-before db-after entity-datoms]
-  (let [attr->datom (entity-datoms=>attr->datom entity-datoms)]
+  [db-before db-after id->attr->datom entity-datoms]
+  (let [e (ffirst entity-datoms)
+        attr->datom (id->attr->datom e)]
     (when (seq attr->datom)
       (let [updated-key-set (set (keys attr->datom))
-            e (some-> attr->datom first second first)
             {[_e _a block-uuid _t add1?] :block/uuid
              [_e _a _v _t add2?]         :block/name
              [_e _a _v _t add3?]         :block/parent
@@ -153,36 +128,32 @@
         ops*))))
 
 (defn- entity-datoms=>asset-op
-  [db-after entity-datoms]
-  (let [attr->datom (entity-datoms=>attr->datom entity-datoms)]
-    (when (seq attr->datom)
-      (let [e (some-> attr->datom first second first)
-            {[_e _a asset-uuid _t add1?] :asset/uuid
-             [_e _a asset-meta _t add2?] :asset/meta}
-            attr->datom
-            op (cond
-                 (or (and add1? asset-uuid)
-                     (and add2? asset-meta))
-                 [:update-asset]
-
-                 (and (not add1?) asset-uuid)
-                 [:remove-asset asset-uuid])]
-        (when op
-          (let [asset-uuid (some-> (d/entity db-after e) :asset/uuid str)]
-            (case (first op)
-              :update-asset (when asset-uuid ["update-asset" {:asset-uuid asset-uuid}])
-              :remove-asset ["remove-asset" {:asset-uuid (str (second op))}])))))))
+  [db-after id->attr->datom entity-datoms]
+  (when-let [e (ffirst entity-datoms)]
+    (let [attr->datom (id->attr->datom e)]
+      (when (seq attr->datom)
+        (let [{[_e _a asset-uuid _t add1?] :asset/uuid
+               [_e _a asset-meta _t add2?] :asset/meta}
+              attr->datom
+              op (cond
+                   (or (and add1? asset-uuid)
+                       (and add2? asset-meta))
+                   [:update-asset]
+
+                   (and (not add1?) asset-uuid)
+                   [:remove-asset asset-uuid])]
+          (when op
+            (let [asset-uuid (some-> (d/entity db-after e) :asset/uuid str)]
+              (case (first op)
+                :update-asset (when asset-uuid ["update-asset" {:asset-uuid asset-uuid}])
+                :remove-asset ["remove-asset" {:asset-uuid (str (second op))}]))))))))
 
 
 (defn generate-rtc-ops
-  [repo db-before db-after datoms]
-  (let [datom-vec-coll (map vec datoms)
-        id->same-entity-datoms (group-by first datom-vec-coll)
-        id-order (distinct (map first datom-vec-coll))
-        same-entity-datoms-coll (map id->same-entity-datoms id-order)
-        asset-ops (keep (partial entity-datoms=>asset-op db-after) same-entity-datoms-coll)
+  [repo db-before db-after same-entity-datoms-coll id->attr->datom]
+  (let [asset-ops (keep (partial entity-datoms=>asset-op db-after id->attr->datom) same-entity-datoms-coll)
         ops (when (empty asset-ops)
-              (mapcat (partial entity-datoms=>ops db-before db-after) same-entity-datoms-coll))
+              (mapcat (partial entity-datoms=>ops db-before db-after id->attr->datom) same-entity-datoms-coll))
         now-epoch*1000 (* 1000 (tc/to-long (t/now)))
         ops* (map-indexed (fn [idx op]
                             [(first op) (assoc (second op) :epoch (+ idx now-epoch*1000))]) ops)
@@ -195,55 +166,12 @@
       (op-mem-layer/add-asset-ops! repo asset-ops*))))
 
 
+(sr/defkeyword :persist-op?
+  "tx-meta option, generate rtc ops when not nil (default true)")
 
-(defn listen-db-to-generate-ops
-  [repo conn]
-  (d/listen! conn :gen-ops
-             (fn [{:keys [tx-data tx-meta db-before db-after]}]
-               (when (:persist-op? tx-meta true)
-                 (generate-rtc-ops repo db-before db-after tx-data)))))
-
-(comment
-  (defn listen-db-to-batch-txs
-   [conn]
-   (d/listen! conn :batch-txs
-              (fn [{:keys [tx-data]}]
-                (when (worker-state/batch-tx-mode?)
-                  (worker-state/conj-batch-txs! tx-data))))))
-
-(defn sync-db-to-main-thread
-  [repo conn]
-  (d/listen! conn :sync-db
-             (fn [{:keys [tx-meta] :as tx-report}]
-               (let [{:keys [pipeline-replace? from-disk?]} tx-meta
-                     result (worker-pipeline/invoke-hooks repo conn tx-report (worker-state/get-context))
-                     tx-report' (or (:tx-report result) tx-report)]
-                 (when-not pipeline-replace?
-                   (let [data (merge
-                               {:request-id (:request-id tx-meta)
-                                :repo repo
-                                :tx-data (:tx-data tx-report')
-                                :tx-meta tx-meta}
-                               (dissoc result :tx-report))]
-                     (worker-util/post-message :sync-db-changes data))
-
-                   (when-not from-disk?
-                     (p/do!
-                      (let [{:keys [blocks-to-remove-set blocks-to-add]} (search/sync-search-indice repo tx-report')
-                            ^js wo (worker-state/get-worker-object)]
-                        (when wo
-                          (when (seq blocks-to-remove-set)
-                            (.search-delete-blocks wo repo (bean/->js blocks-to-remove-set)))
-                          (when (seq blocks-to-add)
-                            (.search-upsert-blocks wo repo (bean/->js blocks-to-add))))))))))))
-
-(defn listen-to-db-changes!
-  [repo conn]
-  (d/unlisten! conn :gen-ops)
-  (d/unlisten! conn :sync-db)
-  (when (op-mem-layer/rtc-db-graph? repo)
-    (listen-db-to-generate-ops repo conn)
-    ;; (rtc-db-listener/listen-db-to-batch-txs conn)
-    )
-
-  (sync-db-to-main-thread repo conn))
+(defmethod db-listener/listen-db-changes :gen-rtc-ops
+  [_ {:keys [_tx-data tx-meta db-before db-after
+             repo id->attr->datom same-entity-datoms-coll]}]
+  (when (and (op-mem-layer/rtc-db-graph? repo)
+             (:persist-op? tx-meta true))
+    (generate-rtc-ops repo db-before db-after same-entity-datoms-coll id->attr->datom)))

src/main/frontend/worker/state.cljs

@@ -1,7 +1,14 @@
 (ns frontend.worker.state
   "State hub for worker"
   (:require [logseq.common.util :as common-util]
-            [logseq.common.config :as common-config]))
+            [logseq.common.config :as common-config]
+            [frontend.schema-register :include-macros true :as sr]))
+
+(sr/defkeyword :undo/repo->undo-stack
+  "{repo [first-op, second-op, ...]}")
+
+(sr/defkeyword :undo/repo->undo-stack
+  "{repo [first-op, second-op, ...]}")
 
 (defonce *state (atom {:worker/object nil
 
@@ -11,8 +18,10 @@
                        ;; FIXME: this name :config is too general
                        :config {}
                        :git/current-repo nil
-                       :rtc/batch-processing? false
-                       :rtc/remote-batch-txs nil
+
+                       :tx/batch-processing? false
+                       :tx/batch-txs nil
+
                        :rtc/downloading-graph? false
 
                        :undo/repo->undo-stack (atom {})
@@ -98,24 +107,3 @@
 (defn rtc-downloading-graph?
   []
   (:rtc/downloading-graph? @*state))
-
-(defn start-batch-tx-mode!
-  []
-  (swap! *state assoc :rtc/batch-processing? true))
-
-(defn rtc-batch-processing?
-  []
-  (:rtc/batch-processing? @*state))
-
-(defn get-batch-txs
-  []
-  (:rtc/remote-batch-txs @*state))
-
-(defn conj-batch-txs!
-  [tx-data]
-  (swap! *state update :rtc/remote-batch-txs (fn [data] (into data tx-data))))
-
-(defn exit-batch-tx-mode!
-  []
-  (swap! *state assoc :rtc/batch-processing? false)
-  (swap! *state assoc :rtc/remote-batch-txs nil))

src/main/frontend/worker/undo_redo.cljs

@@ -1,6 +1,8 @@
 (ns frontend.worker.undo-redo
   "undo/redo related fns and op-schema"
   (:require [datascript.core :as d]
+            [frontend.schema-register :include-macros true :as sr]
+            [frontend.worker.batch-tx :include-macros true :as batch-tx]
             [frontend.worker.db-listener :as db-listener]
             [frontend.worker.state :as worker-state]
             [logseq.common.config :as common-config]
@@ -9,22 +11,59 @@
             [malli.core :as m]
             [malli.util :as mu]))
 
-(def undo-op-schema
+(sr/defkeyword :gen-undo-op?
+  "tx-meta option, generate undo ops from tx-data when true (default true)")
+
+(sr/defkeyword :gen-undo-boundary-op?
+  "tx-meta option, generate `::boundary` undo-op when true (default true).
+usually every transaction's tx-data will generate ops like: [<boundary> <op1> <op2> ...],
+push to undo-stack, result in [...<boundary> <op0> <boundary> <op1> <op2> ...].
+
+when this option is false, only generate [<op1> <op2> ...]. undo-stack: [...<boundary> <op0> <op1> <op2> ...]
+so when undo, it will undo [<op0> <op1> <op2>] instead of [<op1> <op2>]")
+
+(sr/defkeyword ::boundary
+  "boundary of one or more undo-ops.
+when one undo/redo will operate on all ops between two ::boundary")
+
+(sr/defkeyword ::insert-block
+  "when a block is inserted, generate a ::insert-block undo-op.
+when undo this op, the related block will be removed.")
+
+(sr/defkeyword ::move-block
+  "when a block is moved, generate a ::move-block undo-op.")
+
+(sr/defkeyword ::remove-block
+  "when a block is removed, generate a ::remove-block undo-op.
+when undo this op, this original entity-map will be transacted back into db")
+
+(sr/defkeyword ::update-block
+  "when a block is updated, generate a ::update-block undo-op.")
+
+(sr/defkeyword ::empty-undo-stack
+  "return by undo, when no more undo ops")
+
+(sr/defkeyword ::empty-redo-stack
+  "return by redo, when no more redo ops")
+
+(def ^:private boundary [::boundary])
+
+(def ^:private undo-op-schema
   (mu/closed-schema
    [:multi {:dispatch first}
-    [:boundary
+    [::boundary
      [:cat :keyword]]
-    [:insert-block
+    [::insert-block
      [:cat :keyword
       [:map
        [:block-uuid :uuid]]]]
-    [:move-block
+    [::move-block
      [:cat :keyword
       [:map
        [:block-uuid :uuid]
        [:block-origin-left :uuid]
        [:block-origin-parent :uuid]]]]
-    [:remove-block
+    [::remove-block
      [:cat :keyword
       [:map
        [:block-uuid :uuid]
@@ -34,11 +73,11 @@
          [:block/left :uuid]
          [:block/parent :uuid]
          [:block/content :string]
-         [:block/created-at :int]
-         [:block/updated-at :int]
-         [:block/format :any]
+         [:block/created-at {:optional true} :int]
+         [:block/updated-at {:optional true} :int]
+         [:block/format {:optional true} :any]
          [:block/tags {:optional true} [:sequential :uuid]]]]]]]
-    [:update-block
+    [::update-block
      [:cat :keyword
       [:map
        [:block-uuid :uuid]
@@ -46,101 +85,148 @@
        ;; TODO: add more attrs
        ]]]]))
 
-(def undo-ops-validator (m/validator [:sequential undo-op-schema]))
+(def ^:private undo-ops-validator (m/validator [:sequential undo-op-schema]))
+
+(def ^:private entity-map-pull-pattern
+  [:block/uuid
+   {:block/left [:block/uuid]}
+   {:block/parent [:block/uuid]}
+   :block/content
+   :block/created-at
+   :block/updated-at
+   :block/format
+   {:block/tags [:block/uuid]}])
+
+(defn- ->block-entity-map
+  [db eid]
+  (let [m (d/pull db entity-map-pull-pattern eid)]
+    (cond-> m
+      true                  (update :block/left :block/uuid)
+      true                  (update :block/parent :block/uuid)
+      (seq (:block/tags m)) (update :block/tags (partial mapv :block/uuid)))))
 
-(defn reverse-op
+(defn- reverse-op
   [db op]
   (let [block-uuid (:block-uuid (second op))]
     (case (first op)
-      :boundary op
+      ::boundary op
 
-      :insert-block
-      [:remove-block
+      ::insert-block
+      [::remove-block
        {:block-uuid block-uuid
-        :block-entity-map (d/pull db [:block/uuid
-                                      {:block/left [:block/uuid]}
-                                      {:block/parent [:block/uuid]}
-                                      :block/created-at
-                                      :block/updated-at
-                                      :block/format
-                                      :block/properties
-                                      {:block/tags [:block/uuid]}
-                                      :block/content
-                                      {:block/page [:block/uuid]}]
-                                  [:block/uuid block-uuid])}]
-
-      :move-block
+        :block-entity-map (->block-entity-map db [:block/uuid block-uuid])}]
+
+      ::move-block
       (let [b (d/entity db [:block/uuid block-uuid])]
-        [:move-block
+        [::move-block
          {:block-uuid block-uuid
           :block-origin-left (:block/uuid (:block/left b))
           :block-origin-parent (:block/uuid (:block/parent b))}])
 
-      :remove-block
-      [:insert-block {:block-uuid block-uuid}]
+      ::remove-block
+      [::insert-block {:block-uuid block-uuid}]
 
-      :update-block
-      (let [block-origin-content (when (:block-origin-content op)
+      ::update-block
+      (let [block-origin-content (when (:block-origin-content (second op))
                                    (:block/content (d/entity db [:block/uuid block-uuid])))]
-        [:update-block
+        [::update-block
          (cond-> {:block-uuid block-uuid}
            block-origin-content (assoc :block-origin-content block-origin-content))]))))
 
-
 (def ^:private apply-conj-vec (partial apply (fnil conj [])))
 
 (defn- push-undo-ops
   [repo ops]
+  (assert (undo-ops-validator ops) ops)
   (swap! (:undo/repo->undo-stack @worker-state/*state) update repo apply-conj-vec ops))
 
-(defn- pop-undo-op
+(defn- pop-ops-helper
+  [stack]
+  (let [[ops i]
+        (loop [i (dec (count stack)) r []]
+          (let [peek-op (nth stack i nil)]
+            (cond
+              (neg? i)
+              [r 0]
+
+              (nil? peek-op)
+              [r i]
+
+              (= boundary peek-op)
+              [r i]
+
+              :else
+              (recur (dec i) (conj r peek-op)))))]
+    [ops (subvec (vec stack) 0 i)]))
+
+(defn- pop-undo-ops
+  [repo]
+  (let [repo->undo-stack (:undo/repo->undo-stack @worker-state/*state)
+        undo-stack (@repo->undo-stack repo)
+        [ops undo-stack*] (pop-ops-helper undo-stack)]
+    (swap! repo->undo-stack assoc repo undo-stack*)
+    ops))
+
+(defn- empty-undo-stack?
   [repo]
-  (let [repo->undo-stack (:undo/repo->undo-stack @worker-state/*state)]
-    (when-let [peek-op (peek (@repo->undo-stack repo))]
-      (swap! repo->undo-stack update repo pop)
-      peek-op)))
+  (empty? (@(:undo/repo->undo-stack @worker-state/*state) repo)))
+
+(defn- empty-redo-stack?
+  [repo]
+  (empty? (@(:undo/repo->redo-stack @worker-state/*state) repo)))
 
 (defn- push-redo-ops
   [repo ops]
+  (assert (undo-ops-validator ops) ops)
   (swap! (:undo/repo->redo-stack @worker-state/*state) update repo apply-conj-vec ops))
 
-(defn- pop-redo-op
+(defn- pop-redo-ops
   [repo]
-  (let [repo->redo-stack (:undo/repo->redo-stack @worker-state/*state)]
-    (when-let [peek-op (peek (@repo->redo-stack repo))]
-      (swap! repo->redo-stack update repo pop)
-      peek-op)))
+  (let [repo->redo-stack (:undo/repo->redo-stack @worker-state/*state)
+        redo-stack (@repo->redo-stack repo)
+        [ops redo-stack*] (pop-ops-helper redo-stack)]
+    (swap! repo->redo-stack assoc repo redo-stack*)
+    ops))
 
+(defn- normal-block?
+  [entity]
+  (and (:block/parent entity)
+       (:block/left entity)))
 
-(defmulti reverse-apply-op (fn [op _conn _repo] (first op)))
-(defmethod reverse-apply-op :remove-block
+(defmulti ^:private reverse-apply-op (fn [op _conn _repo] (first op)))
+(defmethod reverse-apply-op ::remove-block
   [op conn repo]
-  (let [[_ {:keys [block-uuid block-entity-map]}] op]
-    (when-let [left-entity (d/entity @conn [:block/uuid (:block/left block-entity-map)])]
-      (let [sibling? (not= (:block/left block-entity-map) (:block/parent block-entity-map))]
-        (outliner-tx/transact!
-         {:gen-undo-op? false
-          :outliner-op :insert-blocks
-          :transact-opts {:repo repo
-                          :conn conn}}
-         (outliner-core/insert-blocks! repo conn
-                                       [(cond-> {:block/uuid block-uuid
-                                                 :block/content (:block/content block-entity-map)
-                                                 :block/created-at (:block/created-at block-entity-map)
-                                                 :block/updated-at (:block/updated-at block-entity-map)
-                                                 :block/format :markdown}
-                                          (seq (:block/tags block-entity-map))
-                                          (assoc :block/tags (mapv (partial vector :block/uuid)
-                                                                   (:block/tags block-entity-map))))]
-                                       left-entity {:sibling? sibling? :keep-uuid? true}))
-        :push-undo-redo
-        ))))
-
-(defmethod reverse-apply-op :insert-block
+  (let [[_ {:keys [block-uuid block-entity-map]}] op
+        block-entity (d/entity @conn [:block/uuid block-uuid])]
+    (when-not block-entity ;; this block shouldn't exist now
+      (when-let [left-entity (d/entity @conn [:block/uuid (:block/left block-entity-map)])]
+        (let [sibling? (not= (:block/left block-entity-map) (:block/parent block-entity-map))]
+          (outliner-tx/transact!
+           {:gen-undo-op? false
+            :outliner-op :insert-blocks
+            :transact-opts {:repo repo
+                            :conn conn}}
+           (outliner-core/insert-blocks! repo conn
+                                         [(cond-> {:block/uuid block-uuid
+                                                   :block/content (:block/content block-entity-map)
+                                                   :block/format :markdown}
+                                            (:block/created-at block-entity-map)
+                                            (assoc :block/created-at (:block/created-at block-entity-map))
+
+                                            (:block/updated-at block-entity-map)
+                                            (assoc :block/updated-at (:block/updated-at block-entity-map))
+
+                                            (seq (:block/tags block-entity-map))
+                                            (assoc :block/tags (mapv (partial vector :block/uuid)
+                                                                     (:block/tags block-entity-map))))]
+                                         left-entity {:sibling? sibling? :keep-uuid? true}))
+          :push-undo-redo)))))
+
+(defmethod reverse-apply-op ::insert-block
   [op conn repo]
   (let [[_ {:keys [block-uuid]}] op]
     (when-let [block-entity (d/entity @conn [:block/uuid block-uuid])]
-      (when (empty? (seq (:block/_parent block-entity))) ;if have children, skip
+      (when (empty? (:block/_parent block-entity)) ;if have children, skip
         (outliner-tx/transact!
          {:gen-undo-op? false
           :outliner-op :delete-blocks
@@ -152,7 +238,7 @@
                                        {:children? false}))
         :push-undo-redo))))
 
-(defmethod reverse-apply-op :move-block
+(defmethod reverse-apply-op ::move-block
   [op conn repo]
   (let [[_ {:keys [block-uuid block-origin-left block-origin-parent]}] op]
     (when-let [block-entity (d/entity @conn [:block/uuid block-uuid])]
@@ -166,66 +252,58 @@
            (outliner-core/move-blocks! repo conn [block-entity] left-entity sibling?))
           :push-undo-redo)))))
 
-(defmethod reverse-apply-op :update-block
+(defmethod reverse-apply-op ::update-block
   [op conn repo]
   (let [[_ {:keys [block-uuid block-origin-content]}] op]
     (when-let [block-entity (d/entity @conn [:block/uuid block-uuid])]
-      (let [new-block (assoc block-entity :block/content block-origin-content)]
-        (outliner-tx/transact!
-         {:gen-undo-op? false
-          :outliner-op :save-block
-          :transact-opts {:repo repo
-                          :conn conn}}
-         (outliner-core/save-block! repo conn
-                                    (common-config/get-date-formatter (worker-state/get-config repo))
-                                    new-block))
-        :push-undo-redo))))
-
+      (when (normal-block? block-entity)
+        (let [new-block (assoc block-entity :block/content block-origin-content)]
+          (outliner-tx/transact!
+           {:gen-undo-op? false
+            :outliner-op :save-block
+            :transact-opts {:repo repo
+                            :conn conn}}
+           (outliner-core/save-block! repo conn
+                                      (common-config/get-date-formatter (worker-state/get-config repo))
+                                      new-block))
+          :push-undo-redo)))))
 
 (defn undo
-  [repo]
-  (when-let [op (pop-undo-op repo)]
-    (let [conn (worker-state/get-datascript-conn repo)
-          rev-op (reverse-op @conn op)]
-      (when (= :push-undo-redo (reverse-apply-op op conn repo))
-        (push-redo-ops repo [rev-op])))))
+  [repo conn]
+  (if-let [ops (not-empty (pop-undo-ops repo))]
+    (let [redo-ops-to-push (transient [])]
+      (batch-tx/with-batch-tx-mode conn
+        (doseq [op ops]
+          (let [rev-op (reverse-op @conn op)]
+            (when (= :push-undo-redo (reverse-apply-op op conn repo))
+              (conj! redo-ops-to-push rev-op)))))
+      (when-let [rev-ops (not-empty (persistent! redo-ops-to-push))]
+        (push-redo-ops repo (cons boundary rev-ops)))
+      nil)
+
+    (when (empty-undo-stack? repo)
+      (prn "No further undo information")
+      ::empty-undo-stack)))
 
 (defn redo
-  [repo]
-  (when-let [op (pop-redo-op repo)]
-    (let [conn (worker-state/get-datascript-conn repo)
-          rev-op (reverse-op @conn op)]
-      (when (= :push-undo-redo (reverse-apply-op op conn repo))
-        (push-undo-ops repo [rev-op])))))
-
+  [repo conn]
+  (if-let [ops (not-empty (pop-redo-ops repo))]
+    (let [undo-ops-to-push (transient [])]
+      (batch-tx/with-batch-tx-mode conn
+        (doseq [op ops]
+          (let [rev-op (reverse-op @conn op)]
+            (when (= :push-undo-redo (reverse-apply-op op conn repo))
+              (conj! undo-ops-to-push rev-op)))))
+      (when-let [rev-ops (not-empty (persistent! undo-ops-to-push))]
+        (push-undo-ops repo (cons boundary rev-ops)))
+      nil)
+
+    (when (empty-redo-stack? repo)
+      (prn "No further redo information")
+      ::empty-redo-stack)))
 
 ;;; listen db changes and push undo-ops
 
-(def ^:private entity-map-pull-pattern
-  [:block/uuid
-   {:block/left [:block/uuid]}
-   {:block/parent [:block/uuid]}
-   :block/content
-   :block/created-at
-   :block/updated-at
-   :block/format
-   {:block/tags [:block/uuid]}])
-
-(defn- ->block-entity-map
-  [db eid]
-  (let [m (-> (d/pull db entity-map-pull-pattern eid)
-              (update :block/left :block/uuid)
-              (update :block/parent :block/uuid))]
-    (if (seq (:block/tags m))
-      (update m :block/tags (partial mapv :block/uuid))
-      m)))
-
-(defn- normal-block?
-  [entity]
-  (and (:block/parent entity)
-       (:block/left entity)))
-
-
 (defn- entity-datoms=>ops
   [db-before db-after id->attr->datom entity-datoms]
   (when-let [e (ffirst entity-datoms)]
@@ -240,43 +318,80 @@
           (cond
             (and (not add1?) block-uuid
                  (normal-block? entity-before))
-            [[:remove-block
+            [[::remove-block
               {:block-uuid (:block/uuid entity-before)
                :block-entity-map (->block-entity-map db-before e)}]]
 
             (and add1? block-uuid
                  (normal-block? entity-after))
-            [[:insert-block {:block-uuid (:block/uuid entity-after)}]]
+            [[::insert-block {:block-uuid (:block/uuid entity-after)}]]
 
             (and (or add3? add4?)
                  (normal-block? entity-after))
-            (cond-> [[:move-block
-                      {:block-uuid (:block/uuid entity-after)
-                       :block-origin-left (:block/uuid (:block/left entity-before))
-                       :block-origin-parent (:block/uuid (:block/parent entity-before))}]]
-              (and add2? block-content)
-              (conj [:update-block
-                     {:block-uuid (:block/uuid entity-after)
-                      :block-origin-content (:block/content entity-before)}]))
+            (let [origin-left (:block/left entity-before)
+                  origin-parent (:block/parent entity-before)
+                  origin-left-in-db-after (d/entity db-after [:block/uuid (:block/uuid origin-left)])
+                  origin-parent-in-db-after (d/entity db-after [:block/uuid (:block/uuid origin-parent)])
+                  origin-left-and-parent-available-in-db-after?
+                  (and origin-left-in-db-after origin-parent-in-db-after
+                       (if (not= (:block/uuid origin-left) (:block/uuid origin-parent))
+                         (= (:block/uuid (:block/parent origin-left))
+                            (:block/uuid (:block/parent origin-left-in-db-after)))
+                         true))]
+              (cond-> []
+                origin-left-and-parent-available-in-db-after?
+                (conj [::move-block
+                       {:block-uuid (:block/uuid entity-after)
+                        :block-origin-left (:block/uuid (:block/left entity-before))
+                        :block-origin-parent (:block/uuid (:block/parent entity-before))}])
+
+                (and add2? block-content)
+                (conj [::update-block
+                       {:block-uuid (:block/uuid entity-after)
+                        :block-origin-content (:block/content entity-before)}])))
 
             (and add2? block-content
                  (normal-block? entity-after))
-            [[:update-block
+            [[::update-block
               {:block-uuid (:block/uuid entity-after)
                :block-origin-content (:block/content entity-before)}]]))))))
 
 (defn- generate-undo-ops
-  [repo db-before db-after same-entity-datoms-coll id->attr->datom]
+  [repo db-before db-after same-entity-datoms-coll id->attr->datom gen-boundary-op?]
   (let [ops (mapcat (partial entity-datoms=>ops db-before db-after id->attr->datom) same-entity-datoms-coll)]
-    (assert (undo-ops-validator ops) ops)
     (when (seq ops)
-      (push-undo-ops repo ops))))
-
+      (push-undo-ops repo (if gen-boundary-op? (cons boundary ops) ops)))))
 
 (defmethod db-listener/listen-db-changes :gen-undo-ops
   [_ {:keys [_tx-data tx-meta db-before db-after
              repo id->attr->datom same-entity-datoms-coll]}]
   (when (:gen-undo-op? tx-meta true)
-    (generate-undo-ops repo db-before db-after same-entity-datoms-coll id->attr->datom)))
+    (generate-undo-ops repo db-before db-after same-entity-datoms-coll id->attr->datom
+                       (:gen-undo-boundary-op? tx-meta true))))
 
 ;;; listen db changes and push undo-ops (ends)
+
+(defn clear-undo-redo-stack
+  []
+  (reset! (:undo/repo->undo-stack @worker-state/*state) {})
+  (reset! (:undo/repo->redo-stack @worker-state/*state) {}))
+
+(comment
+
+  (clear-undo-redo-stack)
+  (add-watch (:undo/repo->undo-stack @worker-state/*state)
+             :xxx
+             (fn [_ _ o n]
+               (cljs.pprint/pprint {:k :undo
+                                    :o o
+                                    :n n})))
+
+  (add-watch (:undo/repo->redo-stack @worker-state/*state)
+             :xxx
+             (fn [_ _ o n]
+               (cljs.pprint/pprint {:k :redo
+                                    :o o
+                                    :n n})))
+
+  (remove-watch (:undo/repo->undo-stack @worker-state/*state) :xxx)
+  (remove-watch (:undo/repo->redo-stack @worker-state/*state) :xxx))

src/main/logseq/api.cljs

@@ -86,6 +86,7 @@
                      (subs % 1)
                      (keyword %)))
              (get-in @state/state)
+             (#(if (util/atom? %) @% %))
              (sdk-utils/normalize-keyword-for-json)
              (bean/->js))))
 
@@ -545,6 +546,10 @@
                                                   (db-model/query-block-by-uuid)))))]
         (bean/->js (sdk-utils/normalize-keyword-for-json blocks))))))
 
+(def ^:export clear_selected_blocks
+  (fn []
+    (state/clear-selection!)))
+
 (def ^:export get_current_page
   (fn []
     (when-let [page (state/get-current-page)]

src/main/logseq/sdk/experiments.cljs

@@ -2,13 +2,34 @@
   (:require [frontend.state :as state]
             [frontend.components.page :as page]
             [frontend.util :as util]
+            [camel-snake-kebab.core :as csk]
+            [goog.object :as gobj]
             [frontend.handler.plugin :as plugin-handler]))
 
+(defn- jsx->clj
+  [^js obj]
+  (if (js/goog.isObject obj)
+    (-> (fn [result k]
+          (let [v (gobj/get obj k)
+                k (keyword (csk/->kebab-case k))]
+            (if (= "function" (goog/typeOf v))
+              (assoc result k v)
+              (assoc result k (jsx->clj v)))))
+      (reduce {} (gobj/getKeys obj)))
+    obj))
+
 (defn ^:export cp_page_editor
   [^js props]
-  (let [p (some-> props (aget "page"))]
-    (when-let [e (page/get-page-entity p)]
-      (page/page-blocks-cp (state/get-current-repo) e {}))))
+  (let [props1 (jsx->clj props)
+        page-name (some-> props1 :page)
+        config (some-> props1 (dissoc :page))]
+    (when-let [_entity (page/get-page-entity page-name)]
+      (page/page
+        {:repo (state/get-current-repo)
+         :page-name page-name
+         :preview? false
+         :sidebar? false
+         :config config}))))
 
 (defn ^:export register_fenced_code_renderer
   [pid type ^js opts]
@@ -34,10 +55,10 @@
   (when-let [^js _pl (plugin-handler/get-plugin-inst pid)]
     (plugin-handler/register-daemon-renderer
       (keyword pid) key (reduce #(assoc %1 %2 (aget opts (name %2))) {}
-                           [:before :subs :render]))))
+                          [:before :subs :render]))))
 
 (defn ^:export register_extensions_enhancer
   [pid type enhancer]
   (when-let [^js _pl (and (fn? enhancer) (plugin-handler/get-plugin-inst pid))]
     (plugin-handler/register-extensions-enhancer
-      (keyword pid) type {:enhancer enhancer})))
+      (keyword pid) type {:enhancer enhancer})))

src/test/frontend/test/generators.cljs

@@ -0,0 +1,30 @@
+(ns frontend.test.generators
+  "Generators for block-related data"
+  (:require [clojure.test.check.generators :as gen]
+            [datascript.core :as d]))
+
+
+(defn gen-available-block-uuid
+  [db]
+  (gen/elements
+   (->> (d/q '[:find ?block-uuid
+               :where
+               [?block :block/parent]
+               [?block :block/left]
+               [?block :block/uuid ?block-uuid]]
+             db)
+        (apply concat))))
+
+
+(defn gen-available-parent-left-pair
+  "generate [<parent-uuid> <left-uuid>]"
+  [db]
+  (gen/elements
+   (d/q '[:find ?parent-uuid ?left-uuid
+          :where
+          [?b :block/uuid]
+          [?b :block/parent ?parent]
+          [?b :block/left ?left]
+          [?parent :block/uuid ?parent-uuid]
+          [?left :block/uuid ?left-uuid]]
+        db)))

src/test/frontend/worker/rtc/db_listener_test.cljs

@@ -1,6 +1,7 @@
 (ns frontend.worker.rtc.db-listener-test
   (:require [cljs.test :as t :refer [deftest is testing]]
             [datascript.core :as d]
+            [frontend.worker.db-listener :as worker-db-listener]
             [frontend.worker.rtc.db-listener :as subject]
             [logseq.db.frontend.schema :as db-schema]))
 
@@ -8,6 +9,12 @@
 (def empty-db (d/empty-db db-schema/schema-for-db-based-graph))
 
 
+(defn- tx-data=>id->attr->datom
+  [tx-data]
+  (let [datom-vec-coll (map vec tx-data)
+        id->same-entity-datoms (group-by first datom-vec-coll)]
+    (update-vals id->same-entity-datoms #'worker-db-listener/entity-datoms=>attr->datom)))
+
 (deftest entity-datoms=>ops-test
   (testing "remove whiteboard page-block"
     (let [conn (d/conn-from-db empty-db)
@@ -22,4 +29,5 @@
       (is (= [["remove-page" {:block-uuid (str block-uuid)}]]
              (#'subject/entity-datoms=>ops (:db-before remove-whiteboard-page-block)
                                            (:db-after remove-whiteboard-page-block)
+                                           (tx-data=>id->attr->datom (:tx-data remove-whiteboard-page-block))
                                            (map vec (:tx-data remove-whiteboard-page-block))))))))

src/test/frontend/worker/rtc/fixture.cljs

@@ -1,16 +1,17 @@
 (ns frontend.worker.rtc.fixture
-  (:require [cljs.test :as t]
-            [cljs.core.async :as async :refer [<! >! chan go]]
-            [frontend.worker.rtc.mock :as rtc-mock]
-            [frontend.worker.rtc.core :as rtc-core]
-            [frontend.worker.rtc.asset-sync :as asset-sync]
-            [frontend.test.helper :as test-helper]
+  (:require [cljs.core.async :as async :refer [<! >! chan go]]
+            [cljs.test :as t]
             [datascript.core :as d]
+            [frontend.db :as db]
             [frontend.db.conn :as conn]
+            [frontend.state :as state]
+            [frontend.test.helper :as test-helper]
+            [frontend.worker.db-listener :as worker-db-listener]
+            [frontend.worker.rtc.asset-sync :as asset-sync]
+            [frontend.worker.rtc.core :as rtc-core]
             [frontend.worker.rtc.db-listener :as db-listener]
-            [frontend.worker.rtc.op-mem-layer :as op-mem-layer]
-            [frontend.db :as db]
-            [frontend.state :as state]))
+            [frontend.worker.rtc.mock :as rtc-mock]
+            [frontend.worker.rtc.op-mem-layer :as op-mem-layer]))
 
 (def *test-rtc-state (atom nil))
 (def *test-asset-sync-state (atom nil))
@@ -94,8 +95,16 @@
       (d/listen! test-db-conn
                  ::gen-ops
                  (fn [{:keys [tx-data tx-meta db-before db-after]}]
-                   (when (:persist-op? tx-meta true)
-                     (db-listener/generate-rtc-ops test-helper/test-db-name-db-version db-before db-after tx-data)))))
+                   (let [datom-vec-coll (map vec tx-data)
+                         id->same-entity-datoms (group-by first datom-vec-coll)
+                         id-order (distinct (map first datom-vec-coll))
+                         same-entity-datoms-coll (map id->same-entity-datoms id-order)
+                         id->attr->datom (update-vals
+                                          id->same-entity-datoms
+                                          #'worker-db-listener/entity-datoms=>attr->datom)]
+                     (when (:persist-op? tx-meta true)
+                       (db-listener/generate-rtc-ops test-helper/test-db-name-db-version db-before db-after
+                                                     same-entity-datoms-coll id->attr->datom))))))
    :after
    #(when-let [test-db-conn (conn/get-db test-helper/test-db-name-db-version false)]
       (d/unlisten! test-db-conn ::gen-ops))})

src/test/frontend/worker/undo_redo_test.cljs

@@ -1,13 +1,136 @@
 (ns frontend.worker.undo-redo-test
-  (:require [frontend.worker.undo-redo :as undo-redo]
-            [clojure.test :refer [deftest]]))
+  (:require [clojure.test :as t :refer [deftest is testing use-fixtures]]
+            [clojure.test.check.generators :as gen]
+            [datascript.core :as d]
+            [frontend.db :as db]
+            [frontend.test.generators :as t.gen]
+            [frontend.test.helper :as test-helper]
+            [frontend.worker.undo-redo :as undo-redo]))
 
+(def init-data (test-helper/initial-test-page-and-blocks))
+(defn start-and-destroy-db
+  [f]
+  (test-helper/db-based-start-and-destroy-db
+   f
+   {:init-data (fn [conn] (d/transact! conn init-data))}))
 
+(use-fixtures :each start-and-destroy-db)
 
-(deftest reverse-op-test
-  ;; TODO: add tests for undo-redo
-  undo-redo/undo-op-schema
-  undo-redo/reverse-op
-  undo-redo/undo
-  undo-redo/redo
-  )
+(def gen-non-exist-block-uuid gen/uuid)
+
+(defn- gen-block-uuid
+  [db & {:keys [non-exist-frequency] :or {non-exist-frequency 1}}]
+  (gen/frequency [[9 (t.gen/gen-available-block-uuid db)] [non-exist-frequency gen-non-exist-block-uuid]]))
+
+(defn- gen-parent-left-pair
+  [db]
+  (gen/frequency [[9 (t.gen/gen-available-parent-left-pair db)] [1 (gen/vector gen-non-exist-block-uuid 2)]]))
+
+(defn- gen-move-block-op
+  [db]
+  (gen/let [block-uuid (gen-block-uuid db)
+            [parent left] (gen-parent-left-pair db)]
+    [:frontend.worker.undo-redo/move-block
+     {:block-uuid block-uuid
+      :block-origin-left left
+      :block-origin-parent parent}]))
+
+(defn- gen-insert-block-op
+  [db]
+  (gen/let [block-uuid (gen-block-uuid db)]
+    [:frontend.worker.undo-redo/insert-block
+     {:block-uuid block-uuid}]))
+
+(defn- gen-remove-block-op
+  [db]
+  (gen/let [block-uuid (gen-block-uuid db {:non-exist-frequency 80})
+            [parent left] (gen-parent-left-pair db)
+            content gen/string-ascii]
+    [:frontend.worker.undo-redo/remove-block
+     {:block-uuid block-uuid
+      :block-entity-map
+      {:block/uuid block-uuid
+       :block/left left
+       :block/parent parent
+       :block/content content}}]))
+
+(defn- gen-update-block-op
+  [db]
+  (gen/let [block-uuid (gen-block-uuid db)
+            content gen/string-ascii]
+    [:frontend.worker.undo-redo/update-block
+     {:block-uuid block-uuid
+      :block-origin-content content}]))
+
+(def gen-boundary (gen/return [:frontend.worker.undo-redo/boundary]))
+
+(defn- gen-op
+  [db & {:keys [insert-block-op move-block-op remove-block-op update-block-op boundary-op]
+         :or {insert-block-op 2
+              move-block-op 2
+              remove-block-op 4
+              update-block-op 2
+              boundary-op 2}}]
+  (gen/frequency [[insert-block-op (gen-insert-block-op db)]
+                  [move-block-op (gen-move-block-op db)]
+                  [remove-block-op (gen-remove-block-op db)]
+                  [update-block-op (gen-update-block-op db)]
+                  [boundary-op gen-boundary]]))
+
+(defn- get-db-block-set
+  [db]
+  (set (d/q '[:find ?uuid ?parent-uuid ?left-uuid
+              :where
+              [?b :block/uuid ?uuid]
+              [?b :block/parent ?parent]
+              [?b :block/left ?left]
+              [?parent :block/uuid ?parent-uuid]
+              [?left :block/uuid ?left-uuid]]
+            db)))
+
+(defn- undo-all-then-redo-all
+  [conn]
+  (loop [i 0]
+    (if (not= :frontend.worker.undo-redo/empty-undo-stack
+              (undo-redo/undo test-helper/test-db-name-db-version conn))
+      (recur (inc i))
+      (prn :undo-count i)))
+
+  (loop []
+    (when (not= :frontend.worker.undo-redo/empty-redo-stack
+                (undo-redo/redo test-helper/test-db-name-db-version conn))
+      (recur))))
+
+(deftest undo-test
+  (let [conn (db/get-db false)
+        all-remove-ops (gen/generate (gen/vector (gen-op @conn {:remove-block-op 1000}) 100))]
+    (#'undo-redo/push-undo-ops test-helper/test-db-name-db-version all-remove-ops)
+    (loop [i 0]
+      (if (not= :frontend.worker.undo-redo/empty-undo-stack
+                (undo-redo/undo test-helper/test-db-name-db-version conn))
+        (recur (inc i))
+        (prn :undo-count i)))
+    (undo-redo/clear-undo-redo-stack)
+    (testing "move blocks"
+      (let [origin-graph-block-set (get-db-block-set @conn)
+            ops (gen/generate (gen/vector (gen-op @conn {:move-block-op 1000 :boundary-op 500}) 1000))]
+        (prn :ops (count ops))
+        (#'undo-redo/push-undo-ops test-helper/test-db-name-db-version ops)
+
+        (undo-all-then-redo-all conn)
+        (undo-all-then-redo-all conn)
+        (undo-all-then-redo-all conn)
+
+        (is (= origin-graph-block-set (get-db-block-set @conn)))))
+
+    (testing "random ops"
+      (let [origin-graph-block-set (get-db-block-set @conn)
+            ops (gen/generate (gen/vector (gen-op @conn) 1000))]
+        (prn :ops (count ops))
+        (#'undo-redo/push-undo-ops test-helper/test-db-name-db-version ops)
+
+        (undo-all-then-redo-all conn)
+        (undo-all-then-redo-all conn)
+        (undo-all-then-redo-all conn)
+
+        (is (= origin-graph-block-set (get-db-block-set @conn)))))))