chore: remove :block/title, :block/body and :block/unordered

For two reasons:

1. Reducing memory usage, both block/title and block/body are AST parsed
by Mldoc. After removing, logseq is able to handle 10k notes.

2. Simplify the db schema

src/main/frontend/components/block.cljs

@@ -48,7 +48,7 @@
             [frontend.template :as template]
             [frontend.text :as text]
             [frontend.ui :as ui]
-            [frontend.util :as util]
+            [frontend.util :as util :refer [profile]]
             [frontend.util.clock :as clock]
             [frontend.util.property :as property]
             [frontend.util.drawer :as drawer]
@@ -643,13 +643,10 @@
           hl-type (get-in block [:block/properties :hl-type])
           repo (state/get-current-repo)]
       (if block
-        (let [title (let [title (:block/title block)
-                          block-content (block-content (assoc config :block-ref? true)
-                                                       block nil (:block/uuid block)
-                                                       (:slide? config))
-                          class (if (seq title) "block-ref" "block-ref-no-title")]
-                      [:span {:class class}
-                       block-content])
+        (let [title [:span {:class "block-ref"}
+                     (block-content (assoc config :block-ref? true)
+                                    block nil (:block/uuid block)
+                                    (:slide? config))]
               inner (if label
                       (->elem
                        :span.block-ref
@@ -1369,7 +1366,7 @@
                (rum/with-key (block-container config child)
                  (:block/uuid child)))))]))))
 
-(defn block-content-empty?
+(defn- block-content-empty?
   [{:block/keys [properties title body]}]
   (and
    (or
@@ -1813,8 +1810,10 @@
                      summary]]))))])
 
 (rum/defc block-content < rum/reactive
-  [config {:block/keys [uuid title body content children properties scheduled deadline] :as block} edit-input-id block-id slide?]
-  (let [collapsed? (get properties :collapsed)
+  [config {:block/keys [uuid content children properties scheduled deadline format pre-block?] :as block} edit-input-id block-id slide?]
+  (let [{:block/keys [title body] :as block} (if (:block/title block) block
+                                                 (merge block (block/parse-title-and-body format pre-block? content)))
+        collapsed? (get properties :collapsed)
         block-ref? (:block-ref? config)
         block-ref-with-title? (and block-ref? (seq title))
         block-type (or (:ls-type properties) :default)
@@ -1999,12 +1998,13 @@
                               [:page
                                (or page-original-name page-name)])
             parents-props (doall
-                           (for [{:block/keys [uuid title body name] :as block} parents]
+                           (for [{:block/keys [uuid name content] :as block} parents]
                              (when-not name ; not page
-                               [block
-                                (if (seq title)
-                                  (->elem :span (map-inline config title))
-                                  (->elem :div (markup-elements-cp config body)))])))
+                               (let [{:block/keys [title body]} (block/parse-title-and-body (:block/format block) (:block/pre-block? block) content)]
+                                 [block
+                                 (if (seq title)
+                                   (->elem :span (map-inline config title))
+                                   (->elem :div (markup-elements-cp config body)))]))))
             breadcrumb (->> (into [] parents-props)
                             (concat [page-name-props] (when more? [:more]))
                             (filterv identity)
@@ -2159,8 +2159,10 @@
                              (select-keys (second (:rum/args new-state)) compare-keys))
                        (not= (select-keys (first (:rum/args old-state)) config-compare-keys)
                              (select-keys (first (:rum/args new-state)) config-compare-keys)))))}
-  [state config {:block/keys [uuid body repo children pre-block? top? properties refs heading-level level type] :as block}]
-  (let [blocks-container-id (:blocks-container-id config)
+  [state config {:block/keys [uuid repo children pre-block? top? properties refs heading-level level type format content] :as block}]
+  (let [block (merge block (block/parse-title-and-body format pre-block? content))
+        body (:block/body block)
+        blocks-container-id (:blocks-container-id config)
         config (update config :block merge block)
         ;; Each block might have multiple queries, but we store only the first query's result
         config (if (nil? (:query-result config))
@@ -2384,7 +2386,7 @@
                                clocks))]
         [:div.overflow-x-scroll.sm:overflow-auto
          (->elem
-          :table.m-0 
+          :table.m-0
           {:class "logbook-table"
            :border 0
            :style {:width "max-content"}

src/main/frontend/components/query_table.cljs

@@ -9,6 +9,7 @@
             [frontend.util :as util]
             [frontend.util.clock :as clock]
             [frontend.util.property :as property]
+            [frontend.format.block :as block]
             [medley.core :as medley]
             [rum.core :as rum]))
 
@@ -47,7 +48,8 @@
   [result]
   (let [ks [:block/properties :clock-time]
         result (map (fn [b]
-                      (assoc-in b ks (or (clock/clock-summary (:block/body b) false) 0)))
+                      (let [b (block/parse-title-and-body b)]
+                        (assoc-in b ks (or (clock/clock-summary (:block/body b) false) 0))))
                  result)]
     (if (every? #(zero? (get-in % ks)) result)
       (map #(medley/dissoc-in % ks) result)
@@ -125,10 +127,11 @@
                                           (:block/name item))]
 
                              :block       ; block title
-                             (let [title (:block/title item)]
+                             (let [content (:block/content item)
+                                   {:block/keys [title]} (block/parse-title-and-body (:block/format item) (:block/pre-block? item) content)]
                                (if (seq title)
                                  [:element (->elem :div (map-inline config title))]
-                                 [:string (:block/content item)]))
+                                 [:string content]))
 
                              :created-at
                              [:string (when-let [created-at (:block/created-at item)]

src/main/frontend/components/sidebar.cljs

@@ -65,15 +65,6 @@
      [:a.more svg/arrow-down-v2]]]
    [:div.bd child]])
 
-;; TODO: enhance
-(defn- pick-one-ast-page-ref
-  [block]
-  (when-let [title-ast (and block (:block/title block))]
-    (when-let [link-ref (and (= (ffirst title-ast) "Link")
-                             (:url (second (first title-ast))))]
-      (when (= "Page_ref" (first link-ref))
-        (second link-ref)))))
-
 (defn- delta-y
   [e]
   (let [rect (.. (.. e -target) getBoundingClientRect)]

src/main/frontend/db/model.cljs

@@ -27,7 +27,6 @@
     :block/type
     :block/left
     :block/format
-    :block/title
     :block/refs
     :block/_refs
     :block/path-refs
@@ -36,7 +35,6 @@
     :block/marker
     :block/priority
     :block/properties
-    :block/body
     :block/pre-block?
     :block/scheduled
     :block/deadline
@@ -45,7 +43,6 @@
     :block/updated-at
     :block/file
     :block/parent
-    :block/unordered
     :block/heading-level
     {:block/page [:db/id :block/name :block/original-name :block/journal-day]}
     {:block/_parent ...}])

src/main/frontend/db_schema.cljs

@@ -30,9 +30,6 @@
    ;; :markdown, :org
    :block/format {}
 
-   ;; mldoc parsed ast
-   :block/title {}
-
    ;; belongs to which page
    :block/page {:db/valueType :db.type/ref
                 :db/index true}
@@ -71,9 +68,6 @@
    ;; vector
    :block/properties-order {}
 
-   ;; parsed ast
-   :block/body {}
-
    ;; first block that's not a heading or unordered list
    :block/pre-block? {}
 
@@ -140,8 +134,6 @@
     :block/level
     :block/heading-level
     :block/type
-    :block/title
-    :block/body
     :block/properties
     :block/created-at
     :block/updated-at
@@ -160,5 +152,4 @@
     :block/content
     :block/properties
     :block/alias
-    :block/tags
-    :block/unordered})
+    :block/tags})

src/main/frontend/format/block.cljs

@@ -716,24 +716,29 @@
                     new-block
                     {:block/path-refs path-ref-pages})
                    (> (count blocks) 1)
-                   (assoc :block/warning :multiple-blocks))]
+                   (assoc :block/warning :multiple-blocks))
+           block (dissoc block :block/title :block/body :block/level)]
        (if uuid (assoc block :block/uuid uuid) block)))))
 
 (defn parse-title-and-body
-  [format pre-block? content]
-  (def content content)
-  (let [ast (format/to-edn content format nil)
-        content (if pre-block? content
-                    (str (config/get-block-pattern format) " " (string/triml content)))
-        content (property/remove-properties format content)
-        ast (->> (format/to-edn content format nil)
-                 (map first))
-        title (when (heading-block? (first ast))
-                (:title (second (first ast))))]
-    (cond->
-      {:block/body (vec (if title (rest ast) ast))}
-      title
-      (assoc :block/title title))))
+  ([block]
+   (when (map? block)
+     (merge block
+            (parse-title-and-body (:block/format block)
+                                  (:block/pre-block? block)
+                                  (:block/content block)))))
+  ([format pre-block? content]
+   (let [content (if pre-block? content
+                     (str (config/get-block-pattern format) " " (string/triml content)))
+         content (property/remove-properties format content)
+         ast (->> (format/to-edn content format nil)
+                  (map first))
+         title (when (heading-block? (first ast))
+                 (:title (second (first ast))))]
+     (cond->
+       {:block/body (vec (if title (rest ast) ast))}
+       title
+       (assoc :block/title title)))))
 
 (defn macro-subs
   [macro-content arguments]

src/main/frontend/handler/block.cljs

@@ -68,7 +68,7 @@
                (remove nil?)))))))
 
 ;; TODO: reduced version
-(defn walk-block
+(defn- walk-block
   [block check? transform]
   (let [result (atom nil)]
     (walk/postwalk

src/main/frontend/handler/editor.cljs

@@ -347,8 +347,10 @@
     value))
 
 (defn wrap-parse-block
-  [{:block/keys [content format left page uuid level] :as block}]
-  (let [block (or (and (:db/id block) (db/pull (:db/id block))) block)
+  [{:block/keys [content format left page uuid level pre-block?] :as block}]
+  (let [block (merge
+               (or (and (:db/id block) (db/pull (:db/id block))) block)
+               (block/parse-title-and-body format pre-block? content))
         properties (:block/properties block)
         real-content (:block/content block)
         content (if (and (seq properties) real-content (not= real-content content))
@@ -747,7 +749,6 @@
      :block/format format
      :block/content content
      :block/parent page
-     :block/unordered true
      :block/page page}))
 
 (defn default-properties-block
@@ -767,7 +768,6 @@
       :block/format format
       :block/content content
       :block/parent page
-      :block/unordered true
       :block/page page})))
 
 (defn add-default-title-property-if-needed!
@@ -2151,22 +2151,10 @@
   [uuid page exclude-properties format content-update-fn]
   (fn [block]
     (outliner-core/block
-     (let [[new-content new-title]
+     (let [new-content
            (if content-update-fn
-             (let [new-content (content-update-fn (:block/content block))
-                   new-title (or (->> (mldoc/->edn
-                                       (str (case format
-                                              :markdown "- "
-                                              :org "* ")
-                                            (if (seq (:block/title block)) "" "\n")
-                                            new-content)
-                                       (mldoc/default-config format))
-                                      (ffirst)
-                                      (second)
-                                      (:title))
-                                 (:block/title block))]
-               [new-content new-title])
-             [(:block/content block) (:block/title block)])
+             (content-update-fn (:block/content block))
+             (:block/content block))
            new-content
            (->> new-content
                 (property/remove-property format "id")
@@ -2185,7 +2173,6 @@
                                                  exclude-properties))
                      :block/meta (dissoc (:block/meta block) :start-pos :end-pos)
                      :block/content new-content
-                     :block/title new-title
                      :block/path-refs (->> (cons (:db/id page) (:block/path-refs block))
                                            (remove nil?))})]
        m))))
@@ -2441,7 +2428,7 @@
                   (when (thingatpt/get-setting :properties?)
                     (thingatpt/properties-at-point input))
                   (when (thingatpt/get-setting :list?)
-                    (and (cursor/end-of-line? input) ;; only apply DWIM when cursor at EOL 
+                    (and (cursor/end-of-line? input) ;; only apply DWIM when cursor at EOL
                          (thingatpt/list-item-at-point input))))]
           (cond
             thing-at-point
@@ -3313,10 +3300,11 @@
 
 (defn collapsable? [block-id]
   (if-let [block (db-model/get-block-by-uuid block-id)]
-    (and
-     (nil? (-> block :block/properties :collapsed))
-     (or (not-empty (:block/body block))
-         (db-model/has-children? block-id)))
+    (let [block (block/parse-title-and-body block)]
+      (and
+       (nil? (-> block :block/properties :collapsed))
+       (or (not-empty (:block/body block))
+           (db-model/has-children? block-id))))
     false))
 
 (defn collapse-block! [block-id]

src/main/frontend/handler/export.cljs

@@ -547,8 +547,6 @@
           :block/heading-level
           :block/format
           :block/children
-          :block/title
-          :block/body
           :block/content]))})
 
 (defn- file-name [repo extension]

src/main/frontend/handler/extract.cljs

@@ -137,9 +137,10 @@
                            (remove nil? blocks))
                      (remove nil?))
           pages (remove nil? pages)
-          pages (map (fn [page] (assoc page :block/uuid (db/new-block-id))) pages)]
-      [pages
-       (remove nil? blocks)])
+          pages (map (fn [page] (assoc page :block/uuid (db/new-block-id))) pages)
+          blocks (->> (remove nil? blocks)
+                      (map (fn [b] (dissoc b :block/title :block/body :block/level))))]
+      [pages blocks])
     (catch js/Error e
       (log/error :exception e))))
 
@@ -232,8 +233,11 @@
                             pages (with-ref-pages pages blocks)
                             blocks (map (fn [block]
                                           (let [id (:block/uuid block)
-                                                properties (get-in metadata [:block/properties id])]
-                                            (update block :block/properties merge properties)))
+                                                properties (merge (get-in metadata [:block/properties id])
+                                                                  (:block/properties block))]
+                                            (if (seq properties)
+                                              (assoc block :block/properties properties)
+                                              (dissoc block :block/properties))))
                                      blocks)
                             ;; To prevent "unique constraint" on datascript
                             pages-index (map #(select-keys % [:block/name]) pages)

src/main/frontend/handler/page.cljs

@@ -126,31 +126,6 @@
            (route-handler/redirect-to-page! page))
          page)))))
 
-
-
-(defn get-plugins
-  [blocks]
-  (let [plugins (atom {})
-        add-plugin #(swap! plugins assoc % true)]
-    (walk/postwalk
-     (fn [x]
-       (if (and (vector? x)
-                (>= (count x) 2))
-         (let [[type option] x]
-           (case type
-             "Src" (when (:language option)
-                     (add-plugin "highlight"))
-             "Export" (when (= option "latex")
-                        (add-plugin "latex"))
-             "Latex_Fragment" (add-plugin "latex")
-             "Math" (add-plugin "latex")
-             "Latex_Environment" (add-plugin "latex")
-             nil)
-           x)
-         x))
-     (map :block/body blocks))
-    @plugins))
-
 (defn delete-file!
   [repo page-name]
   (let [file (db/get-page-file page-name)
@@ -350,22 +325,20 @@
         page-ids (->> (map :block/page blocks)
                       (remove nil?)
                       (set))
-        tx       (->> (map (fn [{:block/keys [uuid title content properties format pre-block?] :as block}]
+        tx       (->> (map (fn [{:block/keys [uuid content properties format pre-block?] :as block}]
                              (let [content    (let [content' (replace-old-page! content old-original-name new-name)]
                                                 (when-not (= content' content)
                                                   content'))
                                    properties (let [properties' (walk-replace-old-page! properties old-original-name new-name)]
                                                 (when-not (= properties' properties)
                                                   properties'))]
-                               (when (or title content properties)
+                               (when (or content properties)
                                  (util/remove-nils-non-nested
-                                  (merge
-                                   {:block/uuid       uuid
-                                    :block/content    content
-                                    :block/properties properties
-                                    :block/refs (rename-update-block-refs! (:block/refs block) (:db/id page) (:db/id to-page))
-                                    :block/path-refs (rename-update-block-refs! (:block/path-refs block) (:db/id page) (:db/id to-page))}
-                                   (block/parse-title-and-body format pre-block? content)))))) blocks)
+                                  {:block/uuid       uuid
+                                   :block/content    content
+                                   :block/properties properties
+                                   :block/refs (rename-update-block-refs! (:block/refs block) (:db/id page) (:db/id to-page))
+                                   :block/path-refs (rename-update-block-refs! (:block/path-refs block) (:db/id page) (:db/id to-page))})))) blocks)
                       (remove nil?))]
     (db/transact! repo tx)
     (doseq [page-id page-ids]

src/main/frontend/modules/file/core.cljs

@@ -6,8 +6,9 @@
             [frontend.db :as db]
             [frontend.db.utils :as db-utils]
             [frontend.state :as state]
-            [frontend.util :as util]
-            [frontend.debug :as debug]))
+            [frontend.util :as util :refer [profile]]
+            [frontend.debug :as debug]
+            [frontend.format.block :as block]))
 
 (defn- indented-block-content
   [content spaces-tabs]
@@ -23,8 +24,9 @@
         (ffirst body))))
 
 (defn transform-content
-  [{:block/keys [format pre-block? title content unordered body heading-level left page scheduled deadline parent] :as block} level {:keys [heading-to-list?]}]
-  (let [content (or content "")
+  [{:block/keys [format pre-block? unordered content heading-level left page scheduled deadline parent] :as block} level {:keys [heading-to-list?]}]
+  (let [{:block/keys [title body]} (block/parse-title-and-body format pre-block? content)
+        content (or content "")
         heading-with-title? (seq title)
         allowed-block-as-title? (allowed-block-as-title? title body)
         first-block? (= left page)

src/main/frontend/modules/outliner/core.cljs

@@ -60,15 +60,6 @@
      (outliner-state/get-by-parent-id repo [:block/uuid id])
      (mapv block))))
 
-(defn- update-block-unordered
-  [block]
-  (let [parent (:block/parent block)
-        page (:block/page block)
-        type (:block/type block)]
-    (if (and parent page type (= parent page) (= type :heading))
-      (assoc block :block/unordered false)
-      (assoc block :block/unordered true))))
-
 (defn- block-with-timestamps
   [block]
   (let [updated-at (util/time-ms)
@@ -155,8 +146,7 @@
   (-save [this txs-state]
     (assert (ds/outliner-txs-state? txs-state)
             "db should be satisfied outliner-tx-state?")
-    (let [this (block (update-block-unordered (:data this)))
-          m (-> (:data this)
+    (let [m (-> (:data this)
                 (dissoc :block/children :block/meta :block/top? :block/bottom?)
                 (util/remove-nils))
           m (if (state/enable-block-timestamps?) (block-with-timestamps m) m)

src/main/frontend/tools/html_export.cljs

@@ -5,7 +5,8 @@
             [frontend.components.block :as block]
             [frontend.db :as db]
             [frontend.extensions.slide :as slide]
-            [medley.core :as medley]))
+            [medley.core :as medley]
+            [frontend.format.block :as block]))
 
 ;; Consider generate a db index so that search can still works
 
@@ -16,7 +17,8 @@
 
 (defn- build-block
   [config block]
-  (let [body (:block/body block)
+  (let [block (block/parse-title-and-body block)
+        body (:block/body block)
         block (block/build-block-title config block)]
     [:div.block
      block

src/main/frontend/util/drawer.cljs

@@ -76,7 +76,7 @@
                             lines (concat [title] scheduled deadline before
                                           [(drawer-start typ)] middle [drawer-end] after)]
                         (string/join "\n" lines))
-                      
+
                       :else
                       content)]
         (string/trimr result))

src/main/frontend/util/page_property.cljs

@@ -109,7 +109,6 @@
                      :block/left page-id
                      :block/parent page-id
                      :block/page page-id
-                     :block/title []
                      :block/content (if org?
                                       (str "#+" (string/upper-case (name key)) ": " value)
                                       (str (name key) ":: " value))
@@ -124,4 +123,3 @@
                              :data [block]})
           (ui-handler/re-render-root!)))
       (outliner-file/sync-to-file page-id))))
-

src/test/frontend/handler/block_test.cljs

@@ -5,5 +5,5 @@
 (comment
   (defn clip-block [x]
     (map #(select-keys % [:block/parent :block/left :block/pre-block? :block/uuid :block/level
-                          :block/title :db/id])
+                          :db/id])
       x)))

src/test/frontend/modules/outliner/ds_test.cljs

@@ -15,12 +15,10 @@
                                   :block/refs (),
                                   :block/anchor "level_2123123",
                                   :block/repo "logseq_local_test_navtive_fs",
-                                  :block/body [],
                                   :block/meta {:timestamps [], :properties [], :start-pos 0, :end-pos 15},
                                   :block/format :markdown,
                                   :block/level 1,
                                   :block/tags [],
-                                  :block/title [["Plain" "level 2123123"]],
                                   :block/refs-with-children (),
                                   :block/content "level test",
                                   :db/id 72,
@@ -29,14 +27,12 @@
         rt [[72 :block/uuid #uuid "606c1962-ad7f-424e-b120-0dc7fcb25415" 536870913 true]
             [72 :block/anchor "level_2123123" 536870913 true]
             [72 :block/repo "logseq_local_test_navtive_fs" 536870913 true]
-            [72 :block/body [] 536870913 true]
             [72 :block/meta {:timestamps [], :properties [], :start-pos 0, :end-pos 15} 536870913 true]
             [72 :block/format :markdown 536870913 true]
             [72 :block/level 1 536870913 true]
-            [72 :block/title [["Plain" "level 2123123"]] 536870913 true]
             [72 :block/refs-with-children () 536870913 true]
             [72 :block/content "level test" 536870913 true]]]
     (is (= rt (mapv vec (:tx-data db-report))))))
 
 (comment
-  (run-test test-with-db-macro))
+  (run-test test-with-db-macro))