Improve long page editing performance (#3855)

* Remove expensive parsing when saving files

* Add limit to page blocks query

* Don't collapse block's body to make it compatible with other tools

* Alert if there're unsaved changes when switching graphs

* DB schema migration for :block/collapsed? from it's property

Co-authored-by: Andelf <[email protected]>

.github/workflows/build.yml

@@ -139,6 +139,7 @@ jobs:
       - name: Run Playwright test
         run: xvfb-run -- yarn e2e-test
         env:
+          CI: true
           DEBUG: "pw:test"
 
       - name: Save test artifacts

e2e-tests/basic.spec.ts

@@ -85,7 +85,7 @@ test('create page and blocks', async ({ page }) => {
   await page.waitForTimeout(500)
   expect(await page.$$('.ls-block')).toHaveLength(5)
 
-  await page.waitForTimeout(500)
+  await page.waitForTimeout(1000)
 
   const contentOnDisk = await fs.readFile(
     path.join(graphDir, `pages/${pageTitle}.md`),

e2e-tests/basic.with-diacritics.spec.ts

@@ -10,7 +10,7 @@ test('create page and blocks (diacritics)', async ({ page }) => {
     hotkeyOpenLink = 'Meta+o'
     hotkeyBack = 'Meta+['
   }
-  
+
   const rand = randomString(20)
 
   // diacritic opening test
@@ -22,14 +22,9 @@ test('create page and blocks (diacritics)', async ({ page }) => {
   // build target Page with diacritics
   await activateNewPage(page)
   await page.type(':nth-match(textarea, 1)', 'Diacritic title test content')
-  await page.keyboard.press(hotkeyBack)
 
-  // visit target Page with diacritics (looks same but not same in Unicode)
-  await newBlock(page)
-  await page.type(':nth-match(textarea, 1)', '[[Einführung in die Allgemeine Sprachwissenschaft' + rand + ']] diacritic-block-2')
-  await page.keyboard.press(hotkeyOpenLink)
-  await lastInnerBlock(page)
-  expect(await page.inputValue(':nth-match(textarea, 1)')).toBe('Diacritic title test content')
+  await page.keyboard.press('Enter')
+  await page.fill(':nth-match(textarea, 1)', '[[Einführung in die Allgemeine Sprachwissenschaft' + rand + ']] diacritic-block-2')
   await page.keyboard.press(hotkeyBack)
 
   // check if diacritics are indexed
@@ -41,4 +36,4 @@ test('create page and blocks (diacritics)', async ({ page }) => {
   const results = await page.$$('#ui__ac-inner .block')
   expect(results.length).toEqual(3) // 2 blocks + 1 page
   await page.keyboard.press("Escape")
-})
+})

e2e-tests/code-editing.spec.ts

@@ -57,7 +57,7 @@ test('switch code editing mode', async ({ page }) => {
   await page.waitForSelector('.CodeMirror pre', { state: 'hidden' })
   expect(await page.inputValue('.block-editor textarea')).toBe('```clojure\n;; comment\n\n  \n(+ 1 1)\n```')
 
-  await page.waitForTimeout(500)
+  await page.waitForTimeout(1000)
 })
 
 
@@ -70,13 +70,13 @@ test('convert from block content to code', async ({ page }) => {
   await page.press('.block-editor textarea', 'Escape')
   await page.waitForSelector('.CodeMirror pre', { state: 'visible' })
 
-  await page.waitForTimeout(500)
+  await page.waitForTimeout(1000)
   await page.click('.CodeMirror pre')
-  await page.waitForTimeout(500)
+  await page.waitForTimeout(1000)
   expect(await page.locator('.CodeMirror-gutter-wrapper .CodeMirror-linenumber >> nth=-1').innerText()).toBe('1')
 
   await page.press('.CodeMirror textarea', 'Escape')
-  await page.waitForTimeout(500)
+  await page.waitForTimeout(1000)
 
   expect(await page.inputValue('.block-editor textarea')).toBe('```\n```')
 
@@ -113,13 +113,13 @@ test('code block mixed input source', async ({ page }) => {
   await createRandomPage(page)
 
   await page.fill('.block-editor textarea', '```\n  ABC\n```')
-  await page.waitForTimeout(100) // wait for fill
+  await page.waitForTimeout(500) // wait for fill
   await escapeToCodeEditor(page)
   await page.type('.CodeMirror textarea', '  DEF\nGHI')
 
-  await page.waitForTimeout(500)
+  await page.waitForTimeout(1000)
   await page.press('.CodeMirror textarea', 'Escape')
-  await page.waitForTimeout(500)
+  await page.waitForTimeout(1000)
   // NOTE: auto-indent is on
   expect(await page.inputValue('.block-editor textarea')).toBe('```\n  ABC  DEF\n  GHI\n```')
 })
@@ -128,13 +128,13 @@ test('code block with text around', async ({ page }) => {
   await createRandomPage(page)
 
   await page.fill('.block-editor textarea', 'Heading\n```\n```\nFooter')
-  await page.waitForTimeout(100)
+  await page.waitForTimeout(200)
   await escapeToCodeEditor(page)
   await page.type('.CodeMirror textarea', 'first\n  second')
 
-  await page.waitForTimeout(500)
+  await page.waitForTimeout(1000)
   await page.press('.CodeMirror textarea', 'Escape')
-  await page.waitForTimeout(500)
+  await page.waitForTimeout(1000)
   expect(await page.inputValue('.block-editor textarea')).toBe('Heading\n```\nfirst\n  second\n```\nFooter')
 })
 
@@ -149,15 +149,15 @@ test('multiple code block', async ({ page }) => {
   await page.waitForSelector('.CodeMirror pre', { state: 'visible' })
 
   // first
-  await page.waitForTimeout(500)
+  await page.waitForTimeout(1000)
   await page.click('.CodeMirror pre >> nth=0')
-  await page.waitForTimeout(500)
+  await page.waitForTimeout(1000)
 
   await page.type('.CodeMirror textarea >> nth=0', ':key-test\n', { strict: true })
-  await page.waitForTimeout(500)
+  await page.waitForTimeout(1000)
 
   await page.press('.CodeMirror textarea >> nth=0', 'Escape')
-  await page.waitForTimeout(500)
+  await page.waitForTimeout(1000)
   expect(await page.inputValue('.block-editor textarea'))
     .toBe('中文 Heading\n```clojure\n:key-test\n\n```\nMiddle 🚀\n```clojure\n```\nFooter')
 
@@ -165,15 +165,15 @@ test('multiple code block', async ({ page }) => {
   await page.press('.block-editor textarea', 'Escape')
   await page.waitForSelector('.CodeMirror pre', { state: 'visible' })
 
-  await page.waitForTimeout(500)
+  await page.waitForTimeout(1000)
   await page.click('.CodeMirror pre >> nth=1')
-  await page.waitForTimeout(500)
+  await page.waitForTimeout(1000)
 
   await page.type('.CodeMirror textarea >> nth=1', '\n  :key-test 日本語\n', { strict: true })
-  await page.waitForTimeout(500)
+  await page.waitForTimeout(1000)
 
   await page.press('.CodeMirror textarea >> nth=1', 'Escape')
-  await page.waitForTimeout(500)
+  await page.waitForTimeout(1000)
   expect(await page.inputValue('.block-editor textarea'))
     .toBe('中文 Heading\n```clojure\n:key-test\n\n```\nMiddle 🚀\n```clojure\n\n  :key-test 日本語\n\n```\nFooter')
 })
@@ -182,18 +182,18 @@ test('click outside to exit', async ({ page }) => {
   await createRandomPage(page)
 
   await page.fill('.block-editor textarea', 'Header ``Click``\n```\n  ABC\n```')
-  await page.waitForTimeout(100) // wait for fill
+  await page.waitForTimeout(200) // wait for fill
   await escapeToCodeEditor(page)
   await page.type('.CodeMirror textarea', '  DEF\nGHI')
 
-  await page.waitForTimeout(500)
+  await page.waitForTimeout(1000)
   await page.click('text=Click')
-  await page.waitForTimeout(500)
+  await page.waitForTimeout(1000)
   // NOTE: auto-indent is on
   expect(await page.inputValue('.block-editor textarea')).toBe('Header ``Click``\n```\n  ABC  DEF\n  GHI\n```')
 })
 
-test('click lanuage label to exit #3463', async ({ page }) => {
+test('click language label to exit #3463', async ({ page }) => {
   await createRandomPage(page)
 
   await page.press('.block-editor textarea', 'Enter')
@@ -204,9 +204,9 @@ test('click lanuage label to exit #3463', async ({ page }) => {
   await escapeToCodeEditor(page)
   await page.type('.CodeMirror textarea', '#include<iostream>')
 
-  await page.waitForTimeout(500)
+  await page.waitForTimeout(1000)
   await page.click('text=cpp') // the language label
-  await page.waitForTimeout(500)
+  await page.waitForTimeout(1000)
   expect(await page.inputValue('.block-editor textarea')).toBe('```cpp\n#include<iostream>\n```')
 })
 
@@ -227,12 +227,12 @@ test('multi properties with code', async ({ page }) => {
 
   // first character of code
   await page.click('.CodeMirror pre', { position: { x: 1, y: 5 } })
-  await page.waitForTimeout(500)
+  await page.waitForTimeout(1000)
   await page.type('.CodeMirror textarea', '// Returns nil\n')
 
-  await page.waitForTimeout(500)
+  await page.waitForTimeout(1000)
   await page.press('.CodeMirror textarea', 'Escape')
-  await page.waitForTimeout(500)
+  await page.waitForTimeout(1000)
   expect(await page.inputValue('.block-editor textarea')).toBe(
     'type:: code\n' +
     '类型:: 代码\n' +

e2e-tests/fixtures.ts

@@ -8,9 +8,14 @@ let electronApp: ElectronApplication
 let context: BrowserContext
 let page: Page
 
-// NOTE: Will test against a newly opened graph
-const repoName = 'Test' + randomString(6)
-export const graphDir = path.resolve(__dirname, '../tmp/e2e-graph', repoName)
+let repoName = randomString(10)
+let testTmpDir = path.resolve(__dirname, '../tmp')
+
+if (fs.existsSync(testTmpDir)) {
+    fs.rmdirSync(testTmpDir, { recursive: true })
+}
+
+export let graphDir = path.resolve(testTmpDir, "e2e-test", repoName)
 
 // NOTE: This is a console log watcher for error logs.
 const consoleLogWatcher = (msg: ConsoleMessage) => {

package.json

@@ -37,7 +37,7 @@
         "dev-electron-app": "gulp electron",
         "release-electron": "run-s gulp:build && gulp electronMaker",
         "debug-electron": "cd static/ && yarn electron:debug",
-        "e2e-test": "npx playwright test --reporter github",
+        "e2e-test": "cross-env CI=true npx playwright test --reporter github",
         "run-android-release": "yarn clean && yarn release-app && rm -rf ./public/static && rm -rf ./static/js/*.map && mv static ./public && npx cap sync android && npx cap run android",
         "run-ios-release": "yarn clean && yarn release-app && rm -rf ./public/static && rm -rf ./static/js/*.map && mv static ./public && npx cap sync ios && npx cap run ios",
         "clean": "gulp clean",

src/electron/electron/handler.cljs

@@ -167,16 +167,17 @@
 
 (defn- get-graphs-dir
   []
-  (let [dir (.join path (.homedir os) ".logseq" "graphs")]
+  (let [dir (if utils/ci?
+              (.resolve path js/__dirname "../tmp/graphs")
+              (.join path (.homedir os) ".logseq" "graphs"))]
     (fs-extra/ensureDirSync dir)
     dir))
 
 (defn- get-graphs
   []
-  (let [dir (get-graphs-dir)
-        graphs-path (.join path (.homedir os) ".logseq" "graphs")]
+  (let [dir (get-graphs-dir)]
     (->> (readdir dir)
-         (remove #{graphs-path})
+         (remove #{dir})
          (map #(path/basename % ".transit"))
          (map graph-name->path))))
 

src/electron/electron/utils.cljs

@@ -11,6 +11,11 @@
 (defonce linux? (= (.-platform js/process) "linux"))
 
 (defonce prod? (= js/process.env.NODE_ENV "production"))
+
+(defonce ci? (let [v js/process.env.CI]
+               (or (true? v)
+                   (= v "true"))))
+
 (defonce dev? (not prod?))
 (defonce logger (js/require "electron-log"))
 

src/main/frontend/components/block.cljs

@@ -1422,18 +1422,13 @@
    (every? #(= % ["Horizontal_Rule"]) body)))
 
 (rum/defcs block-control < rum/reactive
-  [state config block uuid block-id body children collapsed? *control-show? edit?]
+  [state config block uuid block-id children collapsed? *control-show? edit?]
   (let [doc-mode? (state/sub :document/mode?)
         has-children-blocks? (and (coll? children) (seq children))
         has-child? (and
                     (not (:pre-block? block))
-                    (or has-children-blocks?
-                        (and (seq (:block/title block)) (seq body))))
-        control-show? (and
-                       (or (and (seq (:block/title block))
-                                (seq body))
-                           has-children-blocks?)
-                       (util/react *control-show?))
+                    has-children-blocks?)
+        control-show? (util/react *control-show?)
         ref? (:ref? config)
         block? (:block? config)
         empty-content? (block-content-empty? block)]
@@ -1864,7 +1859,7 @@
   [config {:block/keys [uuid content children properties scheduled deadline format pre-block?] :as block} edit-input-id block-id slide?]
   (let [{:block/keys [title body] :as block} (if (:block/title block) block
                                                  (merge block (block/parse-title-and-body uuid format pre-block? content)))
-        collapsed? (get properties :collapsed)
+        collapsed? (util/collapsed? block)
         block-ref? (:block-ref? config)
         block-ref-with-title? (and block-ref? (seq title))
         block-type (or (:ls-type properties) :default)
@@ -2121,9 +2116,12 @@
   (editor-handler/unhighlight-blocks!))
 
 (defn- block-mouse-over
-  [e *control-show? block-id doc-mode?]
+  [uuid e *control-show? block-id doc-mode?]
   (util/stop e)
-  (reset! *control-show? true)
+  (when (or
+         (model/block-collapsed? uuid)
+         (editor-handler/collapsable? uuid))
+    (reset! *control-show? true))
   (when-let [parent (gdom/getElement block-id)]
     (let [node (.querySelector parent ".bullet-container")]
       (when doc-mode?
@@ -2178,26 +2176,27 @@
 (rum/defcs block-container < rum/reactive
   {:init (fn [state]
            (let [[config block] (:rum/args state)]
-             (when-not (some? (state/sub-collapsed (:block/uuid block)))
+             (when (and (not (some? (state/sub-collapsed (:block/uuid block))))
+                        (or (:ref? config) (:block? config)))
                (state/set-collapsed-block! (:block/uuid block)
                                            (editor-handler/block-default-collapsed? block config)))
-             (assoc state
-                    ::init-collapsed? (get-in block [:block/properties :collapsed])
-                    ::control-show? (atom false))))
+             (assoc state ::control-show? (atom false))))
    :should-update (fn [old-state new-state]
-                    (let [compare-keys [:block/uuid :block/properties
-                                        :block/parent :block/left
-                                        :block/children :block/content
+                    (let [compare-keys [:block/uuid :block/content :block/parent :block/collapsed? :block/children
+                                        :block/properties
                                         :block/_refs]
-                          config-compare-keys [:show-cloze?]]
-                      (or
-                       (not= (select-keys (second (:rum/args old-state)) compare-keys)
-                             (select-keys (second (:rum/args new-state)) compare-keys))
-                       (not= (select-keys (first (:rum/args old-state)) config-compare-keys)
-                             (select-keys (first (:rum/args new-state)) config-compare-keys)))))}
-  [state config {:block/keys [uuid repo children pre-block? top? properties refs heading-level level type format content] :as block}]
-  (let [block (merge block (block/parse-title-and-body uuid format pre-block? content))
-        body (:block/body block)
+                          config-compare-keys [:show-cloze?]
+                          b1 (second (:rum/args old-state))
+                          b2 (second (:rum/args new-state))
+                          result (or
+                                  (not= (select-keys b1 compare-keys)
+                                        (select-keys b2 compare-keys))
+                                  (not= (select-keys (first (:rum/args old-state)) config-compare-keys)
+                                        (select-keys (first (:rum/args new-state)) config-compare-keys)))]
+                      (boolean result)))}
+  [state config {:block/keys [uuid children pre-block? top? refs heading-level level type format content] :as block}]
+  (let [repo (state/get-current-repo)
+        block (merge block (block/parse-title-and-body uuid format pre-block? content))
         blocks-container-id (:blocks-container-id config)
         config (update config :block merge block)
         ;; Each block might have multiple queries, but we store only the first query's result
@@ -2210,7 +2209,7 @@
         block? (boolean (:block? config))
         collapsed? (if (or ref? block?)
                      (state/sub-collapsed uuid)
-                     (:collapsed properties))
+                     (util/collapsed? block))
         breadcrumb-show? (:breadcrumb-show? config)
         slide? (boolean (:slide? config))
         custom-query? (boolean (:custom-query? config))
@@ -2221,8 +2220,8 @@
         has-child? (boolean
                     (and
                      (not pre-block?)
-                     (or (and (coll? children) (seq children))
-                         (seq body))))
+                     (coll? children)
+                     (seq children)))
         attrs (on-drag-and-mouse-attrs block uuid top? block-id *move-to)
         children-refs (get-children-refs children)
         data-refs (build-refs-data-value children-refs)
@@ -2269,11 +2268,11 @@
      [:div.flex.flex-row.pr-2
       {:class (if heading? "items-baseline" "")
        :on-mouse-over (fn [e]
-                        (block-mouse-over e *control-show? block-id doc-mode?))
+                        (block-mouse-over uuid e *control-show? block-id doc-mode?))
        :on-mouse-leave (fn [e]
                          (block-mouse-leave e *control-show? block-id doc-mode?))}
       (when (not slide?)
-        (block-control config block uuid block-id body children collapsed? *control-show? edit?))
+        (block-control config block uuid block-id children collapsed? *control-show? edit?))
 
       (block-content-or-editor config block edit-input-id block-id heading-level edit?)]
 
@@ -2866,48 +2865,47 @@
 (def initial-blocks-length 200)
 (def step-loading-blocks 50)
 
-(defn- flat-blocks-tree
-  [vec-tree]
-  (->> (mapcat (fn [x] (tree-seq map? :block/children x)) vec-tree)
-       (map #(dissoc % :block/children))))
-
 (defn- get-segment
-  [_config flat-blocks idx blocks->vec-tree]
-  (let [new-idx (if-not (zero? idx)
-                  (+ idx step-loading-blocks)
-                  initial-blocks-length)
+  [flat-blocks idx blocks->vec-tree]
+  (let [new-idx (if (< idx initial-blocks-length)
+                  initial-blocks-length
+                  (+ idx step-loading-blocks))
         max-idx (count flat-blocks)
         idx (min max-idx new-idx)
         blocks (util/safe-subvec flat-blocks 0 idx)]
     [(blocks->vec-tree blocks)
      idx]))
 
-(rum/defcs lazy-blocks <
+(rum/defcs lazy-blocks < rum/reactive
   {:did-remount (fn [_old-state new-state]
                   ;; Loading more when pressing Enter or paste
-                  ;; FIXME: what if users paste too many blocks?
-                  ;; or, a template with a lot of blocks?
-                  (swap! (::last-idx new-state) + 100)
+                  (let [*last-idx (::last-idx new-state)
+                        new-idx (if (zero? *last-idx)
+                                  1
+                                  (inc @*last-idx))]
+                    (reset! *last-idx new-idx))
                   new-state)}
   (rum/local 0 ::last-idx)
   [state config flat-blocks blocks->vec-tree]
   (let [*last-idx (::last-idx state)
-        [segment idx] (get-segment config
-                                   flat-blocks
+        [segment idx] (get-segment flat-blocks
                                    @*last-idx
                                    blocks->vec-tree)
         bottom-reached (fn []
                          (reset! *last-idx idx)
                          (reset! ignore-scroll? false))
-        has-more? (>= (count flat-blocks) (inc idx))]
+        has-more? (and (>= (count flat-blocks) (inc idx))
+                       (not (and (:block? config)
+                                 (state/sub-collapsed (uuid (:id config))))))]
     [:div#lazy-blocks
      (ui/infinite-list
       "main-content-container"
       (block-list config segment)
       {:on-load bottom-reached
-       :threshold 1000
        :has-more has-more?
-       :more (if (:preview? config) "More" (ui/loading "Loading"))})]))
+       :more (if (or (:preview? config) (:sidebar? config))
+               "More"
+               (ui/loading "Loading"))})]))
 
 (rum/defcs blocks-container <
   {:init (fn [state]
@@ -2923,12 +2921,7 @@
         doc-mode? (:document/mode? config)]
     (when (seq blocks)
       (let [blocks->vec-tree #(if (custom-query-or-ref? config) % (tree/blocks->vec-tree % (:id config)))
-            blocks-tree (blocks->vec-tree blocks)
-            blocks-tree (if (seq blocks-tree) blocks-tree blocks)
-            flat-blocks (if (custom-query-or-ref? config)
-                          blocks-tree
-                          (flat-blocks-tree blocks-tree))
-            flat-blocks (vec flat-blocks)]
+            flat-blocks (vec blocks)]
         [:div.blocks-container.flex-1
          {:class (when doc-mode? "document-mode")}
          (lazy-blocks config flat-blocks blocks->vec-tree)]))))

src/main/frontend/components/page.cljs

@@ -62,9 +62,13 @@
   state)
 
 (rum/defc page-blocks-inner <
-  {:did-mount  open-first-block!
+  {:init (fn [state]
+           (when-let [block-id (last (:rum/args state))]
+             (state/set-collapsed-block! block-id false))
+           state)
+   :did-mount  open-first-block!
    :did-update open-first-block!}
-  [page-name _page-blocks hiccup sidebar? _preview? _block-uuid]
+  [page-name _blocks hiccup sidebar? _block-uuid]
   [:div.page-blocks-inner {:style {:margin-left (if sidebar? 0 -20)}}
    (rum/with-key
      (content/content page-name
@@ -111,7 +115,7 @@
 
 (rum/defc page-blocks-cp < rum/reactive
   db-mixins/query
-  [repo page-e {:keys [sidebar? preview?] :as config}]
+  [repo page-e {:keys [sidebar?] :as config}]
   (when page-e
     (let [page-name (or (:block/name page-e)
                         (str (:block/uuid page-e)))
@@ -131,7 +135,7 @@
               hiccup-config (common-handler/config-with-document-mode hiccup-config)
               hiccup (block/->hiccup page-blocks hiccup-config {})]
           [:div
-           (page-blocks-inner page-name page-blocks hiccup sidebar? preview? block-id)
+           (page-blocks-inner page-name page-blocks hiccup sidebar? block-id)
            (when-not config/publishing?
              (let [args (if block-id
                           {:block-uuid block-id}

src/main/frontend/components/repo.cljs

@@ -15,6 +15,7 @@
             [frontend.handler.route :as route-handler]
             [frontend.handler.ui :as ui-handler]
             [frontend.handler.web.nfs :as nfs-handler]
+            [frontend.handler.notification :as notification]
             [frontend.modules.shortcut.core :as shortcut]
             [frontend.state :as state]
             [frontend.ui :as ui]
@@ -22,6 +23,7 @@
             [frontend.fs :as fs]
             [frontend.version :as version]
             [reitit.frontend.easy :as rfe]
+            [frontend.modules.outliner.file :as outliner-file]
             [rum.core :as rum]
             [frontend.mobile.util :as mobile-util]
             [frontend.text :as text]
@@ -39,6 +41,14 @@
   (when-let [dir-name (config/get-repo-dir url)]
     (fs/watch-dir! dir-name)))
 
+(defn- switch-repo-if-writes-finished?
+  [url]
+  (if (outliner-file/writes-finished?)
+    (open-repo-url url)
+    (notification/show!
+     "Please wait seconds until all changes are saved for the current graph."
+     :warning)))
+
 (rum/defc add-repo
   [args]
   (if-let [graph-types (get-in args [:query-params :graph-types])]
@@ -82,7 +92,7 @@
                  (let [local-dir (config/get-local-dir url)
                        graph-name (text/get-graph-name-from-path local-dir)]
                    [:a {:title local-dir
-                        :on-click #(open-repo-url url)}
+                        :on-click #(switch-repo-if-writes-finished? url)}
                     graph-name])
                  [:a {:target "_blank"
                       :href url}
@@ -226,7 +236,7 @@
                               {:title short-repo-name
                                :hover-detail repo-path ;; show full path on hover
                                :options {:class "ml-1"
-                                         :on-click #(open-repo-url url)}}))
+                                         :on-click #(switch-repo-if-writes-finished? url)}}))
                           switch-repos)
               links (->>
                      (concat repo-links

src/main/frontend/db.cljs

@@ -10,6 +10,7 @@
             [frontend.db.react]
             [frontend.db.utils]
             [frontend.db.persist :as db-persist]
+            [frontend.db.migrate :as db-migrate]
             [frontend.namespaces :refer [import-vars]]
             [frontend.state :as state]
             [frontend.util :as util]
@@ -29,7 +30,7 @@
  [frontend.db.utils
   date->int db->json db->edn-str db->string get-max-tx-id get-tx-id
   group-by-page seq-flatten sort-by-pos
-  string->db with-repo
+  string->db
 
   entity pull pull-many transact! get-key-value]
 
@@ -48,7 +49,7 @@
   get-page-blocks-count get-page-blocks-no-cache get-page-file get-page-format get-page-properties
   get-page-referenced-blocks get-page-referenced-pages get-page-unlinked-references get-page-referenced-blocks-no-cache
   get-all-pages get-pages get-pages-relation get-pages-that-mentioned-page get-public-pages get-tag-pages
-  journal-page? local-native-fs? mark-repo-as-cloned! page-alias-set page-blocks-transform pull-block
+  journal-page? mark-repo-as-cloned! page-alias-set pull-block
   set-file-last-modified-at! transact-files-db! page-empty? page-empty-or-dummy? get-alias-source-page
   set-file-content! has-children? get-namespace-pages get-all-namespace-relation get-pages-by-name-partition]
 
@@ -66,6 +67,21 @@
 
  [frontend.db.default built-in-pages-names built-in-pages])
 
+(defn get-schema-version [db]
+  (d/q
+    '[:find ?v .
+      :where
+      [_ :schema/version ?v]]
+    db))
+
+(defn old-schema?
+  [db]
+  (let [v (get-schema-version db)]
+    (if (integer? v)
+      (> db-schema/version v)
+      ;; backward compatibility
+      true)))
+
 ;; persisting DBs between page reloads
 (defn persist! [repo]
   (let [key (datascript-db repo)
@@ -137,6 +153,8 @@
                 (assoc option
                        :listen-handler listen-and-persist!))))
 
+;; TODO: only restore the current graph instead of all the graphs to speedup and
+;; reduce memory usage.
 (defn restore!
   [{:keys [repos] :as me} _old-db-schema restore-config-handler]
   (let [logged? (:name me)]
@@ -145,17 +163,20 @@
        (let [repo url]
          (p/let [db-name (datascript-db repo)
                  db-conn (d/create-conn db-schema/schema)
-                 _ (d/transact! db-conn [{:schema/version db-schema/version}])
                  _ (swap! conns assoc db-name db-conn)
                  stored (db-persist/get-serialized-graph db-name)
                  _ (if stored
                      (let [stored-db (string->db stored)
                            attached-db (d/db-with stored-db (concat
                                                              [(me-tx stored-db me)]
-                                                             default-db/built-in-pages))]
-                       (conn/reset-conn! db-conn attached-db))
+                                                             default-db/built-in-pages))
+                           db (if (old-schema? attached-db)
+                                (db-migrate/migrate attached-db)
+                                attached-db)]
+                       (conn/reset-conn! db-conn db))
                      (when logged?
                        (d/transact! db-conn [(me-tx (d/db db-conn) me)])))]
+           (d/transact! db-conn [{:schema/version db-schema/version}])
            (restore-config-handler repo)
            (listen-and-persist! repo)))))))
 

src/main/frontend/db/migrate.cljs

@@ -1,28 +1,21 @@
 (ns frontend.db.migrate
-  (:require [datascript.core :as d]
-            [frontend.db-schema :as db-schema]
-            [frontend.state :as state]))
+  (:require [datascript.core :as d]))
 
-(defn- migrate-attribute
-  [f]
-  (if (and (keyword? f) (= "page" (namespace f)))
-    (let [k (keyword "block" (name f))]
-      (case k
-        :block/ref-pages
-        :block/refs
-        k))
-    f))
-
-(defn with-schema [db new-schema]
-  (let [datoms (->> (d/datoms db :eavt)
-                    (map (fn [d]
-                           (let [a (migrate-attribute (:a d))]
-                             (d/datom (:e d) a (:v d) (:tx d) (:added d))))))]
-    (-> (d/empty-db new-schema)
-       (with-meta (meta db))
-       (d/db-with datoms))))
+(defn get-collapsed-blocks
+  [db]
+  (d/q
+    '[:find [?b ...]
+      :where
+      [?b :block/properties ?properties]
+      [(get ?properties :collapsed) ?collapsed]
+      [(= true ?collapsed)]]
+    db))
 
 (defn migrate
-  [repo db]
-  (state/pub-event! [:graph/migrated repo])
-  (with-schema db db-schema/schema))
+  [db]
+  (when db
+    (let [collapsed-blocks (get-collapsed-blocks db)]
+      (when (seq collapsed-blocks)
+        (let [tx-data (map (fn [id] {:db/id id
+                                     :block/collapsed? true}) collapsed-blocks)]
+          (d/db-with db tx-data))))))

src/main/frontend/db/model.cljs

@@ -26,8 +26,9 @@
 (def block-attrs
   '[:db/id
     :block/uuid
-    :block/type
+    :block/parent
     :block/left
+    :block/collapsed?
     :block/format
     :block/refs
     :block/_refs
@@ -44,7 +45,6 @@
     :block/created-at
     :block/updated-at
     :block/file
-    :block/parent
     :block/heading-level
     {:block/page [:db/id :block/name :block/original-name :block/journal-day]}
     {:block/_parent ...}])
@@ -374,10 +374,6 @@
              (map (fn [m]
                     (or (:block/original-name m) (:block/name m)))))))))
 
-(defn page-blocks-transform
-  [repo-url result]
-  (db-utils/with-repo repo-url result))
-
 (defn with-pages
   [blocks]
   (let [pages-ids (->> (map (comp :db/id :block/page) blocks)
@@ -400,8 +396,8 @@
 ;; FIXME: alert
 (defn sort-by-left
   ([blocks parent]
-   (sort-by-left blocks parent true))
-  ([blocks parent check?]
+   (sort-by-left blocks parent {:check? true}))
+  ([blocks parent {:keys [check?]}]
    (when check?
      (when (not= (count blocks) (count (set (map :block/left blocks))))
        (let [duplicates (->> (map (comp :db/id :block/left) blocks)
@@ -437,24 +433,45 @@
                      f))
                  form))
 
-(defn flatten-blocks-sort-by-left
-  [blocks parent]
-  (let [ids->blocks (zipmap (map (fn [b] [(:db/id (:block/parent b))
-                                         (:db/id (:block/left b))]) blocks) blocks)]
+
+;; TODO: both zipmap and map lookup are slow in cljs
+;; zipmap 20k blocks takes 30ms on my M1 Air.
+(defn sort-blocks
+  [blocks parent limit]
+  (let [ids->blocks (zipmap (map
+                              (fn [b]
+                                [(:db/id (:block/parent b))
+                                 (:db/id (:block/left b))])
+                              blocks)
+                            blocks)]
     (loop [node parent
            next-siblings '()
            result []]
-      (let [id (:db/id node)
-            child-block (get ids->blocks [id id])
-            next-sibling (get ids->blocks [(:db/id (:block/parent node)) id])
-            next-siblings (if (and next-sibling child-block)
-                            (cons next-sibling next-siblings)
-                            next-siblings)]
-        (if-let [node (or child-block next-sibling)]
-          (recur node next-siblings (conj result node))
-          (if-let [sibling (first next-siblings)]
-            (recur sibling (rest next-siblings) (conj result sibling))
-            result))))))
+      (if (or (nil? node) (and limit (= (count result) limit)))
+        result
+        (let [id (:db/id node)
+              child-block (get ids->blocks [id id])
+              next-sibling (get ids->blocks [(:db/id (:block/parent node)) id])
+              next-siblings (if (and next-sibling child-block)
+                              (cons next-sibling next-siblings)
+                              next-siblings)]
+          (if-let [node (and
+                         (not (:block/collapsed? node))
+                         (or child-block next-sibling))]
+            (recur node next-siblings (conj result node))
+            (if-let [sibling (first next-siblings)]
+              (recur sibling (rest next-siblings) (conj result sibling))
+              result)))))))
+
+(comment
+  (let [page "Scripture (NASB 1995)"
+        page-entity (db-utils/pull [:block/name (string/lower-case page)])
+        blocks (->> (get-page-blocks (state/get-current-repo) (string/lower-case page) {:use-cache? false})
+                    (map (fn [b] (assoc b :block/content (:block/content (db-utils/entity (:db/id b)))))))]
+    (def page-entity page-entity)
+    (def blocks blocks)
+    (time (prn (count (sort-blocks blocks page-entity 1)))))
+  )
 
 (defn get-block-refs-count
   [block-id]
@@ -467,32 +484,65 @@
         nil)
       react))))
 
-;; FIXME: merge get-page-blocks and get-block-and-children to simplify the logic
+;; TODO: native sort and limit support in DB
+(defn- get-limited-blocks
+  [db page block-eids limit]
+  (let [lefts (d/datoms db :avet :block/left)
+        lefts (zipmap (map :e lefts) lefts)
+        collapsed (d/datoms db :avet :block/collapsed?)
+        collapsed (zipmap (map :e collapsed) collapsed)
+        parents (d/datoms db :avet :block/parent)
+        parents (zipmap (map :e parents) parents)
+        blocks (map (fn [id]
+                      (let [collapsed? (:v (get collapsed id))]
+                        (cond->
+                          {:db/id id
+                           :block/left {:db/id (:v (get lefts id))}
+                           :block/parent {:db/id (:v (get parents id))}}
+                          collapsed?
+                          (assoc :block/collapsed? true))))
+                 block-eids)
+        blocks (sort-blocks blocks page limit)]
+    (map :db/id blocks)))
+
+;; Use datoms index and provide limit support
 (defn get-page-blocks
   ([page]
    (get-page-blocks (state/get-current-repo) page nil))
   ([repo-url page]
    (get-page-blocks repo-url page nil))
-  ([repo-url page {:keys [use-cache? pull-keys]
+  ([repo-url page {:keys [use-cache? pull-keys limit]
                    :or {use-cache? true
                         pull-keys '[*]}}]
    (when page
-     (let [page (util/page-name-sanity-lc (string/trim page))
-           page-entity (or (db-utils/entity repo-url [:block/name page])
-                           (db-utils/entity repo-url [:block/original-name page]))
-           page-id (:db/id page-entity)]
+     (let [page-entity (if (integer? page)
+                         (db-utils/entity repo-url page)
+                         (let [page (util/page-name-sanity-lc (string/trim page))]
+                           (db-utils/entity repo-url [:block/name page])))
+           page-id (:db/id page-entity)
+           db (conn/get-conn repo-url)
+           bare-page-map {:db/id page-id
+                          :block/name (:block/name page-entity)
+                          :block/original-name (:block/original-name page-entity)
+                          :block/journal-day (:block/journal-day page-entity)}]
        (when page-id
          (some->
           (react/q repo-url [:page/blocks page-id]
             {:use-cache? use-cache?
-             :transform-fn #(page-blocks-transform repo-url %)
              :query-fn (fn [db]
                          (let [datoms (d/datoms db :avet :block/page page-id)
-                               block-eids (mapv :e datoms)]
-                           (db-utils/pull-many repo-url pull-keys block-eids)))}
+                               block-eids (mapv :e datoms)
+                               ;; TODO: needs benchmark
+                               long-page? (> (count datoms) 1000)
+                               block-eids (if long-page?
+                                            (get-limited-blocks db page-entity block-eids limit)
+                                            block-eids)
+                               blocks (db-utils/pull-many repo-url pull-keys block-eids)
+                               blocks (if long-page? blocks
+                                          (sort-blocks blocks page-entity nil))]
+                           (map (fn [b] (assoc b :block/page bare-page-map)) blocks)))}
             nil)
-          react
-          (flatten-blocks-sort-by-left page-entity)))))))
+          react))))))
 
 (defn get-page-blocks-no-cache
   ([page]
@@ -503,14 +553,12 @@
                    :or {pull-keys '[*]}}]
    (when page
      (let [page (util/page-name-sanity-lc page)
-           page-id (or (:db/id (db-utils/entity repo-url [:block/name page]))
-                       (:db/id (db-utils/entity repo-url [:block/original-name page])))
+           page-id (:db/id (db-utils/entity repo-url [:block/name page]))
            db (conn/get-conn repo-url)]
        (when page-id
          (let [datoms (d/datoms db :avet :block/page page-id)
                block-eids (mapv :e datoms)]
-           (some->> (db-utils/pull-many repo-url pull-keys block-eids)
-                    (page-blocks-transform repo-url))))))))
+           (db-utils/pull-many repo-url pull-keys block-eids)))))))
 
 (defn get-page-blocks-count
   [repo page-id]
@@ -572,15 +620,14 @@
   [repo block-id]
   (when-let [block (:block/parent (get-block-parents-v2 repo block-id))]
     (->> (tree-seq map? (fn [x] [(:block/parent x)]) block)
-         (map (comp :collapsed :block/properties))
-         (some true?))))
+         (some util/collapsed?))))
 
 (defn block-collapsed?
   ([block-id]
    (block-collapsed? (state/get-current-repo) block-id))
   ([repo block-id]
    (when-let [block (db-utils/entity repo [:block/uuid block-id])]
-     (get-in block [:block/properties :collapsed]))))
+     (util/collapsed? block))))
 
 (defn get-block-page
   [repo block-id]
@@ -603,10 +650,8 @@
                               ids))))))
 
 (defn block-and-children-transform
-  [result repo-url _block-uuid]
-  (some->> result
-           db-utils/seq-flatten
-           (db-utils/with-repo repo-url)))
+  [result _repo-url _block-uuid]
+  (db-utils/seq-flatten result))
 
 (defn get-block-children-ids
   [repo block-uuid]
@@ -638,8 +683,8 @@
         (sort-by-left (db-utils/entity [:block/uuid block-uuid])))))
 
 (defn get-blocks-by-page
-  [id-or-lookup-ref]
-  (when-let [conn (conn/get-conn)]
+  [repo id-or-lookup-ref]
+  (when-let [conn (conn/get-conn repo)]
     (->
      (d/q
       '[:find (pull ?block [*])
@@ -1182,9 +1227,10 @@
   [repo]
   (db-utils/get-key-value repo :db/type))
 
-(defn local-native-fs?
-  [repo]
-  (= :local-native-fs (get-db-type repo)))
+(defn db-graph?
+  "Is current graph a database graph instead of a graph on plain-text files?"
+  []
+  (= :database (get-db-type (state/get-current-repo))))
 
 (defn get-public-pages
   [db]
@@ -1358,12 +1404,12 @@
 (defn delete-page-blocks
   [repo-url page]
   (when page
-    (let [db (conn/get-conn repo-url)
-          page (db-utils/pull [:block/name (util/page-name-sanity-lc page)])]
-      (when page
-        (let [datoms (d/datoms db :avet :block/page (:db/id page))
-              block-eids (mapv :e datoms)]
-          (mapv (fn [eid] [:db.fn/retractEntity eid]) block-eids))))))
+    (when-let [db (conn/get-conn repo-url)]
+      (let [page (db-utils/pull [:block/name (util/page-name-sanity-lc page)])]
+        (when page
+          (let [datoms (d/datoms db :avet :block/page (:db/id page))
+                block-eids (mapv :e datoms)]
+            (mapv (fn [eid] [:db.fn/retractEntity eid]) block-eids)))))))
 
 (defn delete-file-pages!
   [repo-url path]

src/main/frontend/db/query_react.cljs

@@ -59,8 +59,7 @@
 (defn custom-query-result-transform
   [query-result remove-blocks q]
   (try
-    (let [repo (state/get-current-repo)
-          result (db-utils/seq-flatten query-result)
+    (let [result (db-utils/seq-flatten query-result)
           block? (:block/uuid (first result))
           result (if block?
                    (let [result (if (seq remove-blocks)
@@ -72,7 +71,6 @@
                      (some->> result
                               remove-nested-children-blocks
                               (model/sort-by-left-recursive)
-                              (db-utils/with-repo repo)
                               (model/with-pages)))
                    result)]
       (if-let [result-transform (:result-transform q)]

src/main/frontend/db/utils.cljs

@@ -54,12 +54,6 @@
   (util/parse-int
    (string/replace (date/ymd date) "/" "")))
 
-(defn with-repo
-  [repo blocks]
-  (map (fn [block]
-         (assoc block :block/repo repo))
-       blocks))
-
 (defn entity
   ([id-or-lookup-ref]
    (entity (state/get-current-repo) id-or-lookup-ref))

src/main/frontend/db_schema.cljs

@@ -1,7 +1,7 @@
 (ns frontend.db-schema)
 
-(defonce version "0.0.2")
-(defonce ast-version "0.0.1")
+(defonce version 1)
+(defonce ast-version 1)
 ;; A page is a special block, a page can corresponds to multiple files with the same ":block/name".
 (def schema
   {:schema/version  {}
@@ -24,8 +24,10 @@
 
    :block/type {}
    :block/uuid {:db/unique :db.unique/identity}
-   :block/parent {:db/valueType :db.type/ref}
+   :block/parent {:db/valueType :db.type/ref
+                  :db/index true}
    :block/left {:db/valueType :db.type/ref}
+   :block/collapsed? {:db/index true}
 
    ;; :markdown, :org
    :block/format {}

src/main/frontend/extensions/srs.cljs

@@ -20,7 +20,8 @@
             [cljs-time.coerce :as tc]
             [clojure.string :as string]
             [rum.core :as rum]
-            [frontend.modules.shortcut.core :as shortcut]))
+            [frontend.modules.shortcut.core :as shortcut]
+            [medley.core :as medley]))
 
 ;;; ================================================================
 ;;; Commentary
@@ -200,7 +201,10 @@
 (defn- clear-collapsed-property
   "Clear block's collapsed property if exists"
   [blocks]
-  (let [result (map (fn [block] (assoc-in block [:block/properties :collapsed] false)) blocks)]
+  (let [result (map (fn [block]
+                      (-> block
+                          (dissoc :block/collapsed?)
+                          (medley/dissoc-in [:block/properties :collapsed]))) blocks)]
     result))
 
 ;;; ================================================================

src/main/frontend/format/block.cljs

@@ -603,6 +603,9 @@
 
                                 (seq (:properties-order properties))
                                 (assoc :properties-order (:properties-order properties)))
+                        block (if (get-in block [:properties :collapsed])
+                                (assoc block :collapsed? true)
+                                block)
                         block (-> block
                                   (assoc-in [:meta :start-pos] start_pos)
                                   (assoc-in [:meta :end-pos] last-pos)

src/main/frontend/handler.cljs

@@ -130,10 +130,7 @@
                                  (js/console.error "Failed to request GitHub app tokens."))))
 
                             (watch-for-date!)
-                            (file-handler/watch-for-current-graph-dir!)
-                            ;; (when-not (state/logged?)
-                            ;;   (state/pub-event! [:after-db-restore repos]))
-                            ))
+                            (file-handler/watch-for-current-graph-dir!)))
                          (p/catch (fn [error]
                                     (log/error :db/restore-failed error))))))
         interval-id (js/setInterval inner-fn 50)]

src/main/frontend/handler/editor.cljs

@@ -481,7 +481,7 @@
                    (boolean? sibling?)
                    sibling?
 
-                   (:collapsed (:block/properties current-block))
+                   (util/collapsed? current-block)
                    true
 
                    :else
@@ -1189,7 +1189,7 @@
   [repo block-ids]
   (let [blocks (db-utils/pull-many repo '[*] (mapv (fn [id] [:block/uuid id]) block-ids))
         blocks* (flatten
-                 (mapv (fn [b] (if (:collapsed (:block/properties b))
+                 (mapv (fn [b] (if (util/collapsed? b)
                                  (vec (tree/sort-blocks (db/get-block-children repo (:block/uuid b)) b))
                                  [b])) blocks))
         block-ids* (mapv :block/uuid blocks*)
@@ -1280,7 +1280,7 @@
                ;; filter out blocks not belong to page with 'page-id'
                (remove (fn [block] (some-> (:db/id (:block/page block)) (not= page-id))))
                ;; expand collapsed blocks
-               (mapv (fn [b] (if (:collapsed (:block/properties b))
+               (mapv (fn [b] (if (util/collapsed? b)
                                (vec (tree/sort-blocks (db/get-block-children repo (:block/uuid b)) b))
                                [b])))
                (flatten))
@@ -2065,8 +2065,7 @@
 (defn edit-box-on-change!
   [e block id]
   (let [value (util/evalue e)
-        repo (or (:block/repo block)
-                 (state/get-current-repo))]
+        repo (state/get-current-repo)]
     (state/set-edit-content! id value false)
     (when @*auto-save-timeout
       (js/clearTimeout @*auto-save-timeout))
@@ -2157,7 +2156,7 @@
           block-self? (block-self-alone-when-insert? config block-id)
           has-children? (db/has-children? (state/get-current-repo)
                                           (:block/uuid editing-block))
-          collapsed? (:collapsed (:block/properties editing-block))]
+          collapsed? (util/collapsed? editing-block)]
       (conj (match (mapv boolean [(seq fst-block-text) (seq snd-block-text)
                                   block-self? has-children? (= parent left) collapsed?])
               ;; when zoom at editing-block
@@ -2779,7 +2778,7 @@
         repo (state/get-current-repo)
         right (outliner-core/get-right-node (outliner-core/block current-block))
         current-block-has-children? (db/has-children? repo (:block/uuid current-block))
-        collapsed? (:collapsed (:block/properties current-block))
+        collapsed? (util/collapsed? current-block)
         first-child (:data (tree/-get-down (outliner-core/block current-block)))
         next-block (if (or collapsed? (not current-block-has-children?))
                      (:data right)
@@ -3451,11 +3450,9 @@
 (defn collapsable? [block-id]
   (when block-id
     (if-let [block (db-model/query-block-by-uuid block-id)]
-      (let [block (block/parse-title-and-body block)]
-        (and
-         (nil? (-> block :block/properties :collapsed))
-         (or (not-empty (:block/body block))
-             (db-model/has-children? block-id))))
+      (and
+       (not (util/collapsed? block))
+       (db-model/has-children? block-id))
       false)))
 
 (defn all-blocks-with-level
@@ -3495,7 +3492,7 @@
          collapse?
          (w/postwalk
           (fn [b]
-            (if (and (map? b) (-> b :block/properties :collapsed))
+            (if (and (map? b) (util/collapsed? b))
               (assoc b :block/children []) b)))
 
          true
@@ -3513,15 +3510,44 @@
   (let [config (:config (state/get-editor-args))]
     (or (:ref? config) (:block? config))))
 
+(defn- set-blocks-collapsed!
+  [block-ids value]
+  (let [block-ids (map (fn [block-id] (if (string? block-id) (uuid block-id) block-id)) block-ids)
+        repo (state/get-current-repo)
+        value (boolean value)]
+    (when repo
+      (ds/auto-transact!
+       [txs-state (ds/new-outliner-txs-state)]
+       {:outliner-op :collapse-expand-blocks
+        :skip-transact? false}
+       (doseq [block-id block-ids]
+         (when-let [block (db/entity [:block/uuid block-id])]
+          (let [current-value (boolean (util/collapsed? block))]
+            (when-not (= current-value value)
+              (let [block (outliner-core/block {:block/uuid block-id
+                                                :block/collapsed? value})]
+                (outliner-core/save-node block {:txs-state txs-state})))))))
+      (let [block-id (first block-ids)
+            input-pos (or (state/get-edit-pos) :max)]
+        (db/refresh! (state/get-current-repo)
+                    {:key :block/change
+                     :data [(db/pull [:block/uuid block-id])]})
+        ;; update editing input content
+        (when-let [editing-block (state/get-edit-block)]
+          (when (= (:block/uuid editing-block) block-id)
+            (edit-block! editing-block
+                         input-pos
+                         (state/get-edit-input-id))))))))
+
 (defn collapse-block! [block-id]
   (when (collapsable? block-id)
     (when-not (skip-collapsing-in-db?)
-      (set-block-property! block-id :collapsed true)))
+      (set-blocks-collapsed! [block-id] true)))
   (state/set-collapsed-block! block-id true))
 
 (defn expand-block! [block-id]
   (when-not (skip-collapsing-in-db?)
-    (remove-block-property! block-id :collapsed))
+    (set-blocks-collapsed! [block-id] false))
   (state/set-collapsed-block! block-id false))
 
 (defn expand!
@@ -3552,8 +3578,7 @@
            nil
            (let [blocks-to-expand (->> blocks-with-level
                                        (filter (fn [b] (= (:block/level b) level)))
-                                       (filter (fn [{:block/keys [properties]}]
-                                                 (contains? properties :collapsed))))]
+                                       (filter util/collapsed?))]
              (if (empty? blocks-to-expand)
                (recur (inc level))
                (doseq [{:block/keys [uuid]} blocks-to-expand]
@@ -3599,17 +3624,15 @@
   ([]
    (collapse-all! nil))
   ([block-id]
-   (let [blocks-to-collapse (all-blocks-with-level {:expanded? true :root-block block-id})]
-     (doseq [{:block/keys [uuid]} blocks-to-collapse]
-       (collapse-block! uuid)))))
+   (let [blocks (all-blocks-with-level {:expanded? true :root-block block-id})]
+     (set-blocks-collapsed! (map :block/uuid blocks) true))))
 
 (defn expand-all!
   ([]
    (expand-all! nil))
   ([block-id]
-   (->> (all-blocks-with-level {:root-block block-id})
-        (map (comp expand-block! :block/uuid))
-        dorun)))
+   (let [blocks (all-blocks-with-level {:root-block block-id})]
+     (set-blocks-collapsed! (map :block/uuid blocks) false))))
 
 (defn- get-block-with-its-children
   [block-uuid]
@@ -3621,7 +3644,7 @@
 (defn expand-all?
   [block-uuid]
   (let [blocks (get-block-with-its-children block-uuid)]
-    (some #(get-in % [:block/properties :collapsed]) blocks)))
+    (some util/collapsed? blocks)))
 
 (defn collapse-all?
   [block-uuid]
@@ -3727,7 +3750,7 @@
                      false
 
                      (and (:block? config)
-                          (get-in block [:block/properties :collapsed]))
+                          (util/collapsed? block))
                      true
 
                      :else
@@ -3740,4 +3763,4 @@
                                     (state/get-ref-open-blocks-level)))))))]
     (if (or (:ref? config) (:block? config))
       collapsed?
-      (get-in block [:block/properties :collapsed]))))
+      (util/collapsed? block))))

src/main/frontend/modules/file/core.cljs

@@ -7,37 +7,21 @@
             [frontend.db.utils :as db-utils]
             [frontend.state :as state]
             [frontend.util :as util]
-            [frontend.debug :as debug]
-            [frontend.format.block :as block]))
+            [frontend.debug :as debug]))
 
 (defn- indented-block-content
   [content spaces-tabs]
   (let [lines (string/split-lines content)]
     (string/join (str "\n" spaces-tabs) lines)))
 
-(defn- allowed-block-as-title?
-  "Allowed to be in the first line of a block (a.k.a block title)"
-  [title body properties]
-  (and (not (seq title))
-       (or
-        (seq properties)
-        (contains?
-         #{"Quote" "Table" "Drawer" "Property_Drawer" "Footnote_Definition" "Custom" "Export" "Src" "Example" "Horizontal_Rule"}
-         (ffirst body)))))
-
 (defn transform-content
-  [{:block/keys [uuid format properties pre-block? unordered content heading-level left page parent]}
-   level
-   {:keys [heading-to-list?]}]
-  (let [{:block/keys [title body]} (block/parse-title-and-body uuid format pre-block? content)
-        content (or content "")
-        heading-with-title? (seq title)
-        allowed-block-as-title? (allowed-block-as-title? title body properties)
+  [{:block/keys [format pre-block? unordered content heading-level left page parent]} level {:keys [heading-to-list?]}]
+  (let [content (or content "")
         first-block? (= left page)
         pre-block? (and first-block? pre-block?)
         markdown? (= format :markdown)
         content (cond
-                  (and first-block? pre-block?)
+                  pre-block?
                   (let [content (string/trim content)]
                     (str content "\n"))
 
@@ -71,18 +55,10 @@
                                       (string/replace #"^\s?#+\s?$" ""))
                                   content)
                         new-content (indented-block-content (string/trim content) spaces-tabs)
-                        sep (cond
-                              markdown-top-heading?
+                        sep (if (or markdown-top-heading?
+                                    (string/blank? new-content))
                               ""
-
-                              (or heading-with-title? allowed-block-as-title?)
-                              " "
-
-                              (string/blank? new-content)
-                              ""
-
-                              :else
-                              (str "\n" spaces-tabs))]
+                              " ")]
                     (str prefix sep new-content)))]
     content))
 
@@ -92,16 +68,19 @@
   (loop [block-contents []
          [f & r] tree
          level init-level]
-    (if (nil? f)
-      (string/join "\n" block-contents)
-      (let [page? (nil? (:block/page f))
-            content (if page? nil (transform-content f level opts))
-            new-content
-            (->> (if-let [children (seq (:block/children f))]
-                   [content (tree->file-content children {:init-level (inc level)})]
-                   [content])
-                 (remove nil?))]
-        (recur (into block-contents new-content) r level)))))
+    (let [f (if (:block/collapsed? f)
+              (assoc-in f [:block/properties :collapsed] true)
+              f)]
+      (if (nil? f)
+        (string/join "\n" block-contents)
+        (let [page? (nil? (:block/page f))
+              content (if page? nil (transform-content f level opts))
+              new-content
+              (->> (if-let [children (seq (:block/children f))]
+                     [content (tree->file-content children {:init-level (inc level)})]
+                     [content])
+                   (remove nil?))]
+          (recur (into block-contents new-content) r level))))))
 
 (def init-level 1)
 

src/main/frontend/modules/outliner/core.cljs

@@ -229,13 +229,8 @@
                   sorted-children)))))))))
 
 (defn set-block-collapsed! [txs-state id collapsed?]
-  (let [e (db/entity id)
-        properties (:block/properties e)
-        properties (if collapsed?
-                     (assoc properties :collapsed true)
-                     (dissoc properties :collapsed))]
-    (swap! txs-state concat [{:db/id id
-                              :block/properties properties}])))
+  (swap! txs-state concat [{:db/id id
+                            :block/collapsed? collapsed?}]))
 
 (defn save-node
   ([node]

src/main/frontend/modules/outliner/file.cljs

@@ -9,39 +9,47 @@
             [frontend.modules.outliner.tree :as tree]
             [frontend.util :as util]
             [goog.object :as gobj]
-            [lambdaisland.glogi :as log]))
+            [lambdaisland.glogi :as log]
+            [frontend.state :as state]))
 
-(def write-chan (async/chan))
+(defonce write-chan (async/chan 100))
+(defonce write-chan-batch-buf (atom []))
 
 (def batch-write-interval 1000)
 
-;; FIXME name conflicts between multiple graphs
+(defn writes-finished?
+  []
+  (empty? @write-chan-batch-buf))
+
 (defn do-write-file!
-  [page-db-id]
-  (let [page-block (db/pull page-db-id)
+  [repo page-db-id]
+  (let [page-block (db/pull repo '[*] page-db-id)
         page-db-id (:db/id page-block)
-        blocks (model/get-blocks-by-page page-db-id)]
+        blocks (model/get-blocks-by-page repo page-db-id)]
     (when-not (and (= 1 (count blocks))
                    (string/blank? (:block/content (first blocks)))
                    (nil? (:block/file page-block)))
-      (let [tree (tree/blocks->vec-tree blocks (:block/name page-block))]
+      (let [tree (tree/blocks->vec-tree repo blocks (:block/name page-block))]
         (if page-block
           (file/save-tree page-block tree)
           (js/console.error (str "can't find page id: " page-db-id)))))))
 
 (defn write-files!
-  [page-db-ids]
-  (when (seq page-db-ids)
+  [pages]
+  (when (seq pages)
     (when-not config/publishing?
-      (doseq [page-db-id (set page-db-ids)]
-        (try (do-write-file! page-db-id)
-             (catch js/Error e
-               (notification/show!
-                [:div
-                 [:p "Write file failed, please copy the changes to other editors in case of losing data."]
-                 "Error: " (str (gobj/get e "stack"))]
-                :error)
-               (log/error :file/write-file-error {:error e})))))))
+      (if (state/input-idle? (state/get-current-repo))
+        (doseq [[repo page-id] (set pages)]
+          (try (do-write-file! repo page-id)
+               (catch js/Error e
+                 (notification/show!
+                  [:div
+                   [:p "Write file failed, please copy the changes to other editors in case of losing data."]
+                   "Error: " (str (gobj/get e "stack"))]
+                  :error)
+                 (log/error :file/write-file-error {:error e}))))
+        (doseq [page pages]
+          (async/put! write-chan page))))))
 
 (defn sync-to-file
   [{page-db-id :db/id}]
@@ -49,8 +57,10 @@
     (notification/show!
      "Write file failed, can't find the current page!"
      :error)
-    (async/put! write-chan page-db-id)))
+    (when-let [repo (state/get-current-repo)]
+      (async/put! write-chan [repo page-db-id]))))
 
 (util/batch write-chan
             batch-write-interval
-            write-files!)
+            write-files!
+            write-chan-batch-buf)

src/main/frontend/modules/outliner/tree.cljs

@@ -1,6 +1,8 @@
 (ns frontend.modules.outliner.tree
   (:require [frontend.db :as db]
-            [frontend.util :as util]))
+            [frontend.util :as util]
+            [clojure.string :as string]
+            [frontend.state :as state]))
 
 (defprotocol INode
   (-get-id [this])
@@ -41,25 +43,27 @@
     (block-children root 1)))
 
 (defn- get-root-and-page
-  [page-name-or-block-id]
-  (if (string? page-name-or-block-id)
-    (if (util/uuid-string? page-name-or-block-id)
-      [false (db/entity [:block/uuid (uuid page-name-or-block-id)])]
-      [true (db/entity [:block/name (util/page-name-sanity-lc page-name-or-block-id)])])
-    [false page-name-or-block-id]))
+  [repo root-id]
+  (if (string? root-id)
+    (if (util/uuid-string? root-id)
+      [false (db/entity repo [:block/uuid (uuid root-id)])]
+      [true (db/entity repo [:block/name (string/lower-case root-id)])])
+    [false root-id]))
 
 (defn blocks->vec-tree
-  [blocks page-name-or-block-id]
-  (let [[page? root] (get-root-and-page (str page-name-or-block-id))]
-    (if-not root ; custom query
-      blocks
-      (let [result (blocks->vec-tree-aux blocks root)]
-        (if page?
-          result
-          ;; include root block
-          (let [root-block (some #(when (= (:db/id %) (:db/id root)) %) blocks)
-                root-block (assoc root-block :block/children result)]
-            [root-block]))))))
+  ([blocks root-id]
+   (blocks->vec-tree (state/get-current-repo) blocks root-id))
+  ([repo blocks root-id]
+   (let [[page? root] (get-root-and-page repo (str root-id))]
+     (if-not root ; custom query
+       blocks
+       (let [result (blocks->vec-tree-aux blocks root)]
+         (if page?
+           result
+           ;; include root block
+           (let [root-block (some #(when (= (:db/id %) (:db/id root)) %) blocks)
+                 root-block (assoc root-block :block/children result)]
+             [root-block])))))))
 
 (defn- sort-blocks-aux
   [parents parent-groups]

src/main/frontend/state.cljs

@@ -15,8 +15,7 @@
             [lambdaisland.glogi :as log]
             [promesa.core :as p]
             [rum.core :as rum]
-            [frontend.mobile.util :as mobile-util]
-            [cljs.cache :as cache]))
+            [frontend.mobile.util :as mobile-util]))
 
 (defonce state
   (let [document-mode? (or (storage/get :document/mode?) false)
@@ -207,33 +206,24 @@
 
       :srs/mode?                             false
 
-      :srs/cards-due-count                   nil})))
+      :srs/cards-due-count                   nil
+      })))
 
 ;; block uuid -> {content(String) -> ast}
-(def blocks-ast-cache (atom (cache/lru-cache-factory {} :threshold 5000)))
+(def blocks-ast-cache (atom {}))
 (defn add-block-ast-cache!
   [block-uuid content ast]
   (when (and block-uuid content ast)
-    (let [k block-uuid
-          add-cache! (fn []
-                       (reset! blocks-ast-cache (cache/evict @blocks-ast-cache block-uuid))
-                       (reset! blocks-ast-cache (cache/miss @blocks-ast-cache k {content ast})))]
-      (if (cache/has? @blocks-ast-cache k)
-        (let [m (cache/lookup @blocks-ast-cache k)]
-          (if (and (map? m) (get m content))
-            (reset! blocks-ast-cache (cache/hit @blocks-ast-cache k))
-            (add-cache!)))
-        (add-cache!)))))
+    (let [new-value (assoc-in @blocks-ast-cache [block-uuid content] ast)
+          new-value (if (> (count new-value) 10000)
+                      (into {} (take 5000 new-value))
+                      new-value)]
+      (reset! blocks-ast-cache new-value))))
 
 (defn get-block-ast
   [block-uuid content]
   (when (and block-uuid content)
-    (let [k block-uuid]
-      (when (cache/has? @blocks-ast-cache k)
-        (let [m (cache/lookup @blocks-ast-cache k)]
-          (when-let [result (and (map? m) (get m content))]
-            (reset! blocks-ast-cache (cache/hit @blocks-ast-cache k))
-            result))))))
+    (get-in @blocks-ast-cache [block-uuid content])))
 
 (defn sub
   [ks]

src/main/frontend/util.cljc

@@ -1383,17 +1383,19 @@
 
 (defn keyname [key] (str (namespace key) "/" (name key)))
 
-(defn batch [in max-time handler]
-  (async/go-loop [buf [] t (async/timeout max-time)]
+(defn batch [in max-time handler buf-atom]
+  (async/go-loop [buf buf-atom t (async/timeout max-time)]
     (let [[v p] (async/alts! [in t])]
       (cond
         (or (= p t) (nil? v))
         (let [timeout (async/timeout max-time)]
-          (handler buf)
-          (recur [] timeout))
+          (handler @buf)
+          (reset! buf [])
+          (recur buf timeout))
 
         :else
-        (recur (conj buf v) t)))))
+        (do (swap! buf conj v)
+          (recur buf t))))))
 
 #?(:cljs
    (defn trace!
@@ -1638,3 +1640,9 @@
        (if (and (not route?) (electron?))
          (js/window.apis.openExternal url)
          (set! (.-href js/window.location) url)))))
+
+(defn collapsed?
+  [block]
+  (or (:block/collapsed? block)
+      ;; for backward compatiblity
+      (get-in block [:properties :collapsed])))

src/main/logseq/api.cljs

@@ -533,7 +533,7 @@
     (when-let [block (db-model/get-block-by-uuid uuid)]
       (let [{:keys [flag]} (bean/->clj opts)
             flag (if (= "toggle" flag)
-                   (not (-> block :block/properties :collapsed))
+                   (not (util/collapsed? block))
                    (boolean flag))]
         (if flag (editor-handler/collapse-block! uuid)
                  (editor-handler/expand-block! uuid))))))