enhance: cut memory usage

package.json

@@ -94,7 +94,7 @@
         "ignore": "5.1.8",
         "is-svg": "4.3.0",
         "jszip": "3.5.0",
-        "mldoc": "1.2.7",
+        "mldoc": "1.2.9",
         "remove-accents": "0.4.2",
         "path": "0.12.7",
         "pixi-graph-fork": "0.2.0",

src/main/frontend/db.cljs

@@ -98,7 +98,7 @@
   (when-let [old-job (get @persistent-jobs repo)]
     (js/clearTimeout old-job)))
 
-(defn- persist-if-idle!
+(defn persist-if-idle!
   [repo]
   (clear-repo-persistent-job! repo)
   (let [job (js/setTimeout
@@ -120,24 +120,25 @@
   [repo conn]
   (d/listen! conn :persistence
              (fn [tx-report]
-               (if (util/electron?)
-                 (when-not (:dbsync? (:tx-meta tx-report))
-                   ;; sync with other windows if needed
-                   (p/let [graph-has-other-window? (ipc/ipc "graphHasOtherWindow" repo)]
-                    (when graph-has-other-window?
-                      (ipc/ipc "dbsync" repo {:data (db->string (:tx-data tx-report))}))))
-                 (do
-                   (state/set-last-transact-time! repo (util/time-ms))
-                   (persist-if-idle! repo)))
-
-               ;; rebuild search indices
-               (let [data (:tx-data tx-report)
-                     datoms (filter
-                             (fn [datom]
-                               (contains? #{:block/name :block/content} (:a datom)))
-                             data)]
-                 (when-let [f @*sync-search-indice-f]
-                   (f datoms))))))
+               (when-not (:new-graph? (:tx-meta tx-report)) ; skip initial txs
+                 (if (util/electron?)
+                   (when-not (:dbsync? (:tx-meta tx-report))
+                     ;; sync with other windows if needed
+                     (p/let [graph-has-other-window? (ipc/ipc "graphHasOtherWindow" repo)]
+                       (when graph-has-other-window?
+                         (ipc/ipc "dbsync" repo {:data (db->string (:tx-data tx-report))}))))
+                   (do
+                     (state/set-last-transact-time! repo (util/time-ms))
+                     (persist-if-idle! repo)))
+
+                 ;; rebuild search indices
+                 (let [data (:tx-data tx-report)
+                       datoms (filter
+                               (fn [datom]
+                                 (contains? #{:block/name :block/content} (:a datom)))
+                               data)]
+                   (when-let [f @*sync-search-indice-f]
+                     (f datoms)))))))
 
 (defn- listen-and-persist!
   [repo]

src/main/frontend/db/outliner.cljs

@@ -1,5 +1,6 @@
 (ns frontend.db.outliner
-  (:require [datascript.core :as d]))
+  (:require [datascript.core :as d]
+            [clojure.set :as set]))
 
 (defn get-by-id
   [conn id]
@@ -9,13 +10,12 @@
 
 (defn get-by-parent-&-left
   [conn parent-id left-id]
-  (let [r (d/q '[:find (pull ?a [*])
-                 :in $ ?p ?l
-                 :where
-                 [?a :block/left ?l]
-                 [?a :block/parent ?p]]
-            @conn parent-id left-id)]
-    (ffirst r)))
+  (when (and parent-id left-id)
+    (let [lefts (:block/_left (d/entity @conn left-id))
+          children (:block/_parent (d/entity @conn parent-id))
+          ids (set/intersection lefts children)
+          id (:db/id (first ids))]
+      (when id (d/pull @conn '[*] id)))))
 
 ;; key [:block/children parent-id]
 

src/main/frontend/db/utils.cljs

@@ -91,12 +91,16 @@
   ([tx-data]
    (transact! (state/get-current-repo) tx-data))
   ([repo-url tx-data]
+   (transact! (state/get-current-repo) tx-data nil))
+  ([repo-url tx-data tx-meta]
    (when-not config/publishing?
      (let [tx-data (->> (util/remove-nils tx-data)
                         (remove nil?))]
        (when (seq tx-data)
          (when-let [conn (conn/get-conn repo-url false)]
-           (d/transact! conn (vec tx-data))))))))
+           (if tx-meta
+             (d/transact! conn (vec tx-data) tx-meta)
+             (d/transact! conn (vec tx-data)))))))))
 
 (defn get-key-value
   ([key]

src/main/frontend/format/block.cljs

@@ -471,11 +471,11 @@
           (update :refs (fn [col] (remove nil? col)))))
 
 (defn extract-blocks*
-  [blocks pre-block-body pre-block-properties encoded-content]
+  [blocks pre-block-properties encoded-content]
   (let [first-block (first blocks)
         first-block-start-pos (get-in first-block [:block/meta :start-pos])
-        blocks (if (or (seq @pre-block-body)
-                       (seq @pre-block-properties))
+        blocks (if (or (> first-block-start-pos 0)
+                       (empty? blocks))
                  (cons
                   (merge
                    (let [content (utf8/substring encoded-content 0 first-block-start-pos)
@@ -486,7 +486,6 @@
                                 :meta {:start-pos 0
                                        :end-pos (or first-block-start-pos
                                                     (utf8/length encoded-content))}
-                                :body @pre-block-body
                                 :properties @pre-block-properties
                                 :properties-order (keys @pre-block-properties)
                                 :refs (->> (get-page-refs-from-properties @pre-block-properties)
@@ -506,11 +505,9 @@
   (try
     (let [encoded-content (utf8/encode content)
           last-pos (utf8/length encoded-content)
-          pre-block-body (atom nil)
           pre-block-properties (atom nil)
           blocks
           (loop [headings []
-                 block-body []
                  blocks (reverse blocks)
                  timestamps {}
                  properties {}
@@ -527,17 +524,12 @@
                   (paragraph-timestamp-block? block)
                   (let [timestamps (extract-timestamps block)
                         timestamps' (merge timestamps timestamps)
-                        [timestamps others] (split-with #(= "Timestamp" (first %)) (second block))
-                        other-body (->>
-                                    (concat
-                                     timestamps
-                                     (drop-while #(contains? #{"Hard_Break_Line" "Break_Line"} (first %)) others))
-                                    (remove nil?))]
-                    (recur headings (conj block-body ["Paragraph" other-body]) (rest blocks) timestamps' properties last-pos last-level children (conj block-all-content block-content)))
+                        [timestamps others] (split-with #(= "Timestamp" (first %)) (second block))]
+                    (recur headings (rest blocks) timestamps' properties last-pos last-level children (conj block-all-content block-content)))
 
                   (property/properties-ast? block)
                   (let [properties (extract-properties (second block))]
-                    (recur headings block-body (rest blocks) timestamps properties last-pos last-level children (conj block-all-content block-content)))
+                    (recur headings (rest blocks) timestamps properties last-pos last-level children (conj block-all-content block-content)))
 
                   (heading-block? block)
                   (let [id (get-custom-id-or-new-id properties)
@@ -568,9 +560,6 @@
                         block (cond->
                                 (assoc block
                                       :uuid id
-                                      :body (vec
-                                             (->> (reverse block-body)
-                                                  (map #(remove-indentations format (:level block) %))))
                                       :refs ref-pages-in-properties
                                       :children (or current-block-children [])
                                       :format format)
@@ -600,20 +589,17 @@
 
                                 (and updated-at (integer? updated-at))
                                 (assoc :block/updated-at updated-at))]
-                    (recur (conj headings block) [] (rest blocks) {} {} last-pos' (:level block) children []))
+                    (recur (conj headings block) (rest blocks) {} {} last-pos' (:level block) children []))
 
                   :else
-                  (let [block-body' (conj block-body block)]
-                    (recur headings block-body' (rest blocks) timestamps properties last-pos last-level children (conj block-all-content block-content)))))
+                  (recur headings (rest blocks) timestamps properties last-pos last-level children (conj block-all-content block-content))))
               (do
-                (when (seq block-body)
-                  (reset! pre-block-body (reverse block-body)))
                 (when (seq properties)
                   (let [properties (:properties properties)]
                     (reset! pre-block-properties properties)))
                 (-> (reverse headings)
                     safe-blocks))))]
-      (extract-blocks* blocks pre-block-body pre-block-properties encoded-content))
+      (extract-blocks* blocks pre-block-properties encoded-content))
     (catch js/Error e
       (js/console.error "extract-blocks-failed")
       (log/error :exception e))))

src/main/frontend/format/mldoc.cljs

@@ -36,9 +36,10 @@
 (defn default-config
   ([format]
    (default-config format {:export-heading-to-list? false}))
-  ([format {:keys [export-heading-to-list? export-keep-properties? export-md-indent-style export-md-remove-options]}]
+  ([format {:keys [export-heading-to-list? export-keep-properties? export-md-indent-style export-md-remove-options parse_outline_only?]}]
    (let [format (string/capitalize (name (or format :markdown)))]
      (->> {:toc false
+           :parse_outline_only (or parse_outline_only? false)
            :heading_number false
            :keep_line_break true
            :format format
@@ -198,28 +199,28 @@
         []))
     (log/error :edn/wrong-content-type content)))
 
-(defn ->edn-async
-  ([content config]
-   (->edn-async nil content config))
-  ([file content config]
-   (if util/node-test?
-     (p/resolved (->edn content config))
-     (try
-       (if (string/blank? content)
-         (p/resolved [])
-         (p/let [v (pool/add-parse-job! content config)]
-           (try
-             (-> v
-                 (util/json->clj)
-                 (update-src-full-content content)
-                 (collect-page-properties))
-             (catch js/Error e
-               (println :parser/failed file)
-               (js/console.error e)
-               (p/resolved [])))))
-       (catch js/Error e
-         (log/error :parser/failed e)
-         (p/resolved []))))))
+;; (defn ->edn-async
+;;   ([content config]
+;;    (->edn-async nil content config))
+;;   ([file content config]
+;;    (if util/node-test?
+;;      (p/resolved (->edn content config))
+;;      (try
+;;        (if (string/blank? content)
+;;          (p/resolved [])
+;;          (p/let [v (pool/add-parse-job! content config)]
+;;            (try
+;;              (-> v
+;;                  (util/json->clj)
+;;                  (update-src-full-content content)
+;;                  (collect-page-properties))
+;;              (catch js/Error e
+;;                (println :parser/failed file)
+;;                (js/console.error e)
+;;                (p/resolved [])))))
+;;        (catch js/Error e
+;;          (log/error :parser/failed e)
+;;          (p/resolved []))))))
 
 (defn opml->edn
   [content]

src/main/frontend/handler/events.cljs

@@ -20,6 +20,7 @@
             [frontend.handler.editor :as editor-handler]
             [frontend.handler.notification :as notification]
             [frontend.handler.page :as page-handler]
+            [frontend.handler.search :as search-handler]
             [frontend.handler.ui :as ui-handler]
             [frontend.handler.repo :as repo-handler]
             [frontend.handler.route :as route-handler]
@@ -74,12 +75,8 @@
     close-fn)))
 
 (defmethod handle :graph/added [[_ repo]]
-  ;; TODO: add ast/version to db
-  (let [_conn (conn/get-conn repo false)
-        ; ast-version (d/datoms @conn :aevt :ast/version)
-        ]
-    (db/set-key-value repo :ast/version db-schema/ast-version)
-    (srs/update-cards-due-count!)))
+  (db/set-key-value repo :ast/version db-schema/ast-version)
+  (search-handler/rebuild-indices!))
 
 (defn- graph-switch [graph]
   (repo-handler/push-if-auto-enabled! (state/get-current-repo))

src/main/frontend/handler/extract.cljs

@@ -45,7 +45,7 @@
 ;; TODO: performance improvement
 (defn- extract-pages-and-blocks
   #_:clj-kondo/ignore
-  [repo-url format ast properties file content _utf8-content _journal?]
+  [repo-url format ast properties file content]
   (try
     (let [page (get-page-name file ast)
           [_original-page-name page-name _journal-day] (block/convert-page-if-journal page)
@@ -138,39 +138,31 @@
       (log/error :exception e))))
 
 (defn extract-blocks-pages
-  ([repo-url file content]
-   (extract-blocks-pages repo-url file content (utf8/encode content)))
-  ([repo-url file content utf8-content]
-   (if (string/blank? content)
-     (p/resolved [])
-     (p/let [format (format/get-format file)
-             _ (println "Parsing start : " file)
-             parse-f (if (and (mobile/is-native-platform?) config/dev?)
-                       mldoc/->edn
-                       (fn [content config]
-                         (mldoc/->edn-async file content config)))
-             ast (parse-f content (mldoc/default-config format))]
-       _ (println "Parsing finished : " file)
-       (let [journal? (config/journal? file)
-             first-block (ffirst ast)
-             properties (let [properties (and (property/properties-ast? first-block)
-                                              (->> (last first-block)
-                                                   (map (fn [[x y]]
-                                                          [x (if (string? y)
-                                                               (text/parse-property x y)
-                                                               y)]))
-                                                   (into {})
-                                                   (walk/keywordize-keys)))]
-                          (when (and properties (seq properties))
-                            (if (:filters properties)
-                              (update properties :filters
-                                      (fn [v]
-                                        (string/replace (or v "") "\\" "")))
-                              properties)))]
-         (extract-pages-and-blocks
-          repo-url
-          format ast properties
-          file content utf8-content journal?))))))
+  [repo-url file content]
+  (if (string/blank? content)
+    []
+    (let [format (format/get-format file)
+          ast (mldoc/->edn content (mldoc/default-config format {:parse_outline_only? true}))]
+      (println "Parsing finished : " file)
+      (let [first-block (ffirst ast)
+            properties (let [properties (and (property/properties-ast? first-block)
+                                             (->> (last first-block)
+                                                  (map (fn [[x y]]
+                                                         [x (if (string? y)
+                                                              (text/parse-property x y)
+                                                              y)]))
+                                                  (into {})
+                                                  (walk/keywordize-keys)))]
+                         (when (and properties (seq properties))
+                           (if (:filters properties)
+                             (update properties :filters
+                                     (fn [v]
+                                       (string/replace (or v "") "\\" "")))
+                             properties)))]
+        (extract-pages-and-blocks
+         repo-url
+         format ast properties
+         file content)))))
 
 (defn with-block-uuid
   [pages]
@@ -190,57 +182,26 @@
          (map (partial apply merge))
          (with-block-uuid))))
 
-(defn remove-illegal-refs
-  [block block-ids-set refresh?]
-  (let [aux-fn (fn [refs]
-                 (let [block-refs (if refresh? (set refs)
-                                      (set/intersection (set refs) block-ids-set))]
-                   (set/union
-                    (set (filter :block/name refs))
-                    block-refs)))]
-    (-> block
-        (update :block/refs aux-fn)
-        (update :block/path-refs aux-fn))))
-
-;; TODO: refactor with reset-file!
-(defn extract-all-blocks-pages
-  [repo-url files metadata refresh?]
-  (when (seq files)
-    (-> (p/all (map
-                 (fn [{:file/keys [path content]}]
-                   (when content
-                     (let [org? (= "org" (string/lower-case (util/get-file-ext path)))
-                           content (if org?
-                                     content
-                                     (text/scheduled-deadline-dash->star content))
-                           utf8-content (utf8/encode content)]
-                       (extract-blocks-pages repo-url path content utf8-content))))
-                 files))
-        (p/then (fn [result]
-                  (let [result (remove empty? result)]
-                    (when (seq result)
-                      (let [result (util/distinct-by (fn [[pages _blocks]]
-                                                       (let [page (first pages)]
-                                                         (:block/name page))) result)
-                            [pages blocks] (apply map concat result)
-                            block-ids (->> (map :block/uuid blocks)
-                                           (remove nil?))
-                            pages (with-ref-pages pages blocks)
-                            blocks (map (fn [block]
-                                          (let [id (:block/uuid block)
-                                                properties (merge (get-in metadata [:block/properties id])
-                                                                  (:block/properties block))]
-                                            (if (seq properties)
-                                              (assoc block :block/properties properties)
-                                              (dissoc block :block/properties))))
-                                     blocks)
-                            ;; To prevent "unique constraint" on datascript
-                            pages-index (map #(select-keys % [:block/name]) pages)
-                            block-ids-set (set (map (fn [uuid] [:block/uuid uuid]) block-ids))
-                            blocks (map #(remove-illegal-refs % block-ids-set refresh?) blocks)
-                            block-ids (map (fn [uuid] {:block/uuid uuid}) block-ids)]
-                        (apply concat [pages-index pages block-ids blocks])))))))))
-
 (defn extract-all-block-refs
   [content]
   (map second (re-seq #"\(\(([a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12})\)\)" content)))
+
+(comment
+  (def page (frontend.db/entity [:block/name (string/lower-case "Scripture (NASB 1995)")]))
+  (def page (frontend.db/entity [:block/name (string/lower-case "test")]))
+  (def file (:block/file page))
+  (def content (:file/content (db/pull (:db/id file))))
+
+  (do
+    (prn "Full parse: ")
+    (dotimes [_ 5] (time (do (mldoc/->edn content (mldoc/default-config :markdown)) nil)))
+    (prn "Parse outline-only")
+    (dotimes [_ 5] (time (do (mldoc/->edn content (mldoc/default-config :markdown {:parse_outline_only? true})) nil))))
+  (simple-benchmark []
+                    (mldoc/->edn content (mldoc/default-config :markdown))
+                    5)
+  (simple-benchmark []
+                    (mldoc/->edn content (mldoc/default-config :markdown {:parse_outline_only? true}))
+                    5)
+
+  )

src/main/frontend/handler/file.cljs

@@ -18,12 +18,12 @@
             [frontend.handler.extract :as extract-handler]
             [frontend.handler.ui :as ui-handler]
             [frontend.state :as state]
-            [frontend.utf8 :as utf8]
             [frontend.util :as util]
             [lambdaisland.glogi :as log]
             [promesa.core :as p]
             [frontend.debug :as debug]
-            [frontend.mobile.util :as mobile]))
+            [frontend.mobile.util :as mobile]
+            [clojure.set :as set]))
 
 ;; TODO: extract all git ops using a channel
 
@@ -131,74 +131,73 @@
        current-file))))
 
 (defn reset-file!
-  [repo-url file content]
-  (let [electron-local-repo? (and (util/electron?)
-                                  (config/local-db? repo-url))
-        file (cond
-               (and electron-local-repo?
-                    util/win32?
-                    (utils/win32 file))
-               file
+  ([repo-url file content]
+   (reset-file! repo-url file content false))
+  ([repo-url file content new-graph?]
+   (let [electron-local-repo? (and (util/electron?)
+                                   (config/local-db? repo-url))
+         file (cond
+                (and electron-local-repo?
+                     util/win32?
+                     (utils/win32 file))
+                file
 
-               (and electron-local-repo? (or
-                                          util/win32?
-                                          (not= "/" (first file))))
-               (str (config/get-repo-dir repo-url) "/" file)
+                (and electron-local-repo? (or
+                                           util/win32?
+                                           (not= "/" (first file))))
+                (str (config/get-repo-dir repo-url) "/" file)
 
-               (and (mobile/native-android?) (not= "/" (first file)))
-               (str (config/get-repo-dir repo-url) "/" file)
+                (and (mobile/native-android?) (not= "/" (first file)))
+                (str (config/get-repo-dir repo-url) "/" file)
 
-               (and (mobile/native-ios?) (not= "/" (first file)))
-               file
+                (and (mobile/native-ios?) (not= "/" (first file)))
+                file
 
-               :else
-               file)
-        file (util/path-normalize file)
-        new? (nil? (db/entity [:file/path file]))]
-    ;; TODO: refactor with reset-file!
-    ;; TODO: missing text/scheduled-deadline-dash->star
-    (db/set-file-content! repo-url file content)
-    (let [format (format/get-format file)
-          utf8-content (utf8/encode content)
-          file-content [{:file/path file}]]
-      (p/let [tx (if (contains? config/mldoc-support-formats format)
-                   ;; TODO: compare the logic with extract-all-blocks-pages
-                   (p/let [[pages blocks] (extract-handler/extract-blocks-pages repo-url file content utf8-content)
-                           first-page (first pages)
-                           delete-blocks (->
-                                          (concat
-                                           (db/delete-file-blocks! repo-url file)
-                                           (when first-page (db/delete-page-blocks repo-url (:block/name first-page))))
-                                          (distinct))
-                           _ (when-let [current-file (page-exists-in-another-file repo-url first-page file)]
-                               (when (not= file current-file)
-                                 (let [error (str "Page already exists with another file: " current-file ", current file: " file)]
-                                   (state/pub-event! [:notification/show
-                                                      {:content error
-                                                       :status :error
-                                                       :clear? false}]))))
-                           ;; TODO: merge with the validation logic in extract-all-blocks-pages
-                           blocks (remove-non-exists-refs! blocks)
-                           block-ids (map (fn [block] {:block/uuid (:block/uuid block)}) blocks)
-                           pages (extract-handler/with-ref-pages pages blocks)
-                           ;; To prevent "unique constraint" on datascript
-                           pages-index (map #(select-keys % [:block/name]) pages)
-                           block-ids-set (set (map (fn [uuid] [:block/uuid uuid]) block-ids))
-                           blocks (map #(extract-handler/remove-illegal-refs % block-ids-set true) blocks) ;; refresh should be true or false?
-                           block-ids (map (fn [uuid] {:block/uuid uuid}) block-ids)]
-                     ;; does order matter?
-                     (concat file-content pages-index delete-blocks pages block-ids blocks))
-                   file-content)]
-        (let [tx (concat tx [(let [t (tc/to-long (t/now))] ;; TODO: use file system timestamp?
-                               (cond->
-                                 {:file/path file}
-                                 new?
-                                 (assoc :file/created-at t)))])]
-          (db/transact! repo-url tx))))))
+                :else
+                file)
+         file (util/path-normalize file)
+         new? (nil? (db/entity [:file/path file]))]
+     (db/set-file-content! repo-url file content)
+     (let [format (format/get-format file)
+           file-content [{:file/path file}]]
+       (let [tx (if (contains? config/mldoc-support-formats format)
+                  (let [[pages blocks] (extract-handler/extract-blocks-pages repo-url file content)
+                        first-page (first pages)
+                        delete-blocks (->
+                                       (concat
+                                        (db/delete-file-blocks! repo-url file)
+                                        (when first-page (db/delete-page-blocks repo-url (:block/name first-page))))
+                                       (distinct))
+                        _ (when-let [current-file (page-exists-in-another-file repo-url first-page file)]
+                            (when (not= file current-file)
+                              (let [error (str "Page already exists with another file: " current-file ", current file: " file)]
+                                (state/pub-event! [:notification/show
+                                                   {:content error
+                                                    :status :error
+                                                    :clear? false}]))))
+                        block-ids (map (fn [block] {:block/uuid (:block/uuid block)}) blocks)
+                        block-refs-ids (->> (mapcat :block/refs blocks)
+                                            (filter (fn [ref] (and (vector? ref)
+                                                                   (= :block/uuid (first ref)))))
+                                            (map (fn [ref] {:block/uuid (second ref)}))
+                                            (seq))
+                        ;; To prevent "unique constraint" on datascript
+                        block-ids (set/union (set block-ids) (set block-refs-ids))
+                        pages (extract-handler/with-ref-pages pages blocks)
+                        pages-index (map #(select-keys % [:block/name]) pages)]
+                    ;; does order matter?
+                    (concat file-content pages-index delete-blocks pages block-ids blocks))
+                  file-content)]
+         (let [tx (concat tx [(let [t (tc/to-long (t/now))] ;; TODO: use file system timestamp?
+                                (cond->
+                                  {:file/path file}
+                                  new?
+                                  (assoc :file/created-at t)))])]
+           (db/transact! repo-url tx (when new-graph? {:new-graph? true}))))))))
 
 ;; TODO: Remove this function in favor of `alter-files`
 (defn alter-file
-  [repo path content {:keys [reset? re-render-root? from-disk? skip-compare?]
+  [repo path content {:keys [reset? re-render-root? from-disk? skip-compare? new-graph?]
                       :or {reset? true
                            re-render-root? false
                            from-disk? false
@@ -209,14 +208,14 @@
                       #(fs/write-file! repo (config/get-repo-dir repo) path content
                                        (assoc (when original-content {:old-content original-content})
                                               :skip-compare? skip-compare?)))]
-    (p/let [_ (if reset?
-                (do
-                  (when-let [page-id (db/get-file-page-id path)]
-                    (db/transact! repo
-                      [[:db/retract page-id :block/alias]
-                       [:db/retract page-id :block/tags]]))
-                  (reset-file! repo path content))
-                (db/set-file-content! repo path content))]
+    (let [_ (if reset?
+              (do
+                (when-let [page-id (db/get-file-page-id path)]
+                  (db/transact! repo
+                    [[:db/retract page-id :block/alias]
+                     [:db/retract page-id :block/tags]]))
+                (reset-file! repo path content new-graph?))
+              (db/set-file-content! repo path content))]
       (util/p-handle (write-file!)
                      (fn [_]
                        (when (= path (config/get-config-path repo))

src/main/frontend/handler/repo.cljs

@@ -13,7 +13,6 @@
             [frontend.fs.nfs :as nfs]
             [frontend.git :as git]
             [frontend.handler.common :as common-handler]
-            [frontend.handler.extract :as extract-handler]
             [frontend.handler.file :as file-handler]
             [frontend.handler.git :as git-handler]
             [frontend.handler.notification :as notification]
@@ -202,21 +201,22 @@
 
 (defn- parse-files-and-create-default-files-inner!
   [repo-url files delete-files delete-blocks file-paths first-clone? db-encrypted? re-render? re-render-opts metadata opts]
-  (p/let [refresh? (:refresh? opts)
-          parsed-files (filter
-                        (fn [file]
-                          (let [format (format/get-format (:file/path file))]
-                            (contains? config/mldoc-support-formats format)))
-                        files)
-          blocks-pages (if (seq parsed-files)
-                         (extract-handler/extract-all-blocks-pages repo-url parsed-files metadata refresh?)
-                         [])]
-    (let [config-file (config/get-config-path)]
-      (when (contains? (set file-paths) config-file)
-        (when-let [content (some #(when (= (:file/path %) config-file)
-                                    (:file/content %)) files)]
-          (file-handler/restore-config! repo-url content true))))
-    (reset-contents-and-blocks! repo-url files blocks-pages delete-files delete-blocks refresh?)
+  (let [refresh? (:refresh? opts)
+        parsed-files (filter
+                      (fn [file]
+                        (let [format (format/get-format (:file/path file))]
+                          (contains? config/mldoc-support-formats format)))
+                      files)
+        parsed-files (sort-by :file/path parsed-files)
+        new-graph? (:new-graph? opts)]
+    (doseq [file parsed-files]
+      (file-handler/alter-file repo-url
+                               (:file/path file)
+                               (:file/content file)
+                               {:new-graph? new-graph?
+                                :re-render-root? false
+                                :from-disk? true
+                                :metadata metadata}))
     (load-pages-metadata! repo-url file-paths files)
     (when first-clone?
       (if (and (not db-encrypted?) (state/enable-encryption? repo-url))
@@ -261,7 +261,8 @@
       (parse-files-and-create-default-files! repo-url files delete-files delete-blocks file-paths first-clone? db-encrypted? re-render? re-render-opts metadata opts))))
 
 (defn load-repo-to-db!
-  [repo-url {:keys [first-clone? diffs nfs-files refresh?]}]
+  [repo-url {:keys [first-clone? diffs nfs-files refresh? new-graph?]
+             :as opts}]
   (spec/validate :repos/url repo-url)
   (when (= :repos (state/get-current-route))
     (route-handler/redirect-to-home!))
@@ -282,7 +283,8 @@
                            (parse-files-and-load-to-db! repo-url files-contents (assoc option :refresh? refresh?)))))]
     (cond
       (and (not (seq diffs)) nfs-files)
-      (parse-files-and-load-to-db! repo-url nfs-files {:first-clone? true})
+      (parse-files-and-load-to-db! repo-url nfs-files {:first-clone? true
+                                                       :new-graph? new-graph?})
 
       (and first-clone? (not nfs-files))
       (->

src/main/frontend/handler/web/nfs.cljs

@@ -174,14 +174,14 @@
                            (repo-handler/start-repo-db-if-not-exists! repo {:db-type :local-native-fs})
                            (p/let [_ (repo-handler/load-repo-to-db! repo
                                                                     {:first-clone? true
+                                                                     :new-graph?   true
                                                                      :nfs-files    files})]
                              (state/add-repo! {:url repo :nfs? true})
                              (state/set-loading-files! repo false)
-                             (and ok-handler (ok-handler))
+                             (when ok-handler (ok-handler))
                              (when (util/electron?)
                                (fs/watch-dir! dir-name))
-                             (state/pub-event! [:graph/added repo])
-                             (db/persist! repo)))))
+                             (db/persist-if-idle! repo)))))
                (p/catch (fn [error]
                           (log/error :nfs/load-files-error repo)
                           (log/error :exception error)))))))

src/main/frontend/modules/outliner/core.cljs

@@ -7,7 +7,6 @@
             [frontend.db.conn :as conn]
             [frontend.db.outliner :as db-outliner]
             [frontend.modules.outliner.datascript :as ds]
-            [frontend.modules.outliner.state :as outliner-state]
             [frontend.modules.outliner.tree :as tree]
             [frontend.modules.outliner.utils :as outliner-u]
             [frontend.state :as state]
@@ -53,13 +52,6 @@
     {}
     blocks))
 
-(defn get-children
-  [id]
-  (let [repo (state/get-current-repo)]
-   (some->>
-     (outliner-state/get-by-parent-id repo [:block/uuid id])
-     (mapv block))))
-
 (defn- block-with-timestamps
   [block]
   (let [updated-at (util/time-ms)
@@ -209,24 +201,10 @@
       block-id))
 
   (-get-children [this]
-    (let [children (get-children (tree/-get-id this))]
-      (when (seq children)
-        (let [left-id->block (index-blocks-by-left-id children)]
-          (loop [sorted-children []
-                 current-node this]
-            (let [id (tree/-get-id current-node)]
-              (if-let [right (get left-id->block id)]
-                (recur (conj sorted-children right) right)
-                (do
-                  (let [should-equal
-                        (=
-                          (count children)
-                          (count sorted-children))]
-                    (when-not should-equal
-                      (prn "children: " (mapv #(get-in % [:data :block/uuid]) children))
-                      (prn "sorted-children: " (mapv #(get-in % [:data :block/uuid]) sorted-children))
-                      (throw (js/Error. "Number of children and sorted-children are not equal."))))
-                  sorted-children)))))))))
+    (let [parent-id (tree/-get-id this)
+          parent-data (get-data this)
+          children (db-model/get-block-immediate-children (state/get-current-repo) parent-id)]
+      (map block children))))
 
 (defn set-block-collapsed! [txs-state id collapsed?]
   (swap! txs-state concat [{:db/id id

src/main/frontend/modules/outliner/state.cljs

@@ -1,16 +0,0 @@
-(ns frontend.modules.outliner.state
-  (:require [frontend.db.outliner :as db-outliner]
-            [frontend.db.react :as react]
-            [frontend.modules.outliner.utils :as u-outliner]
-            [frontend.util :as util]))
-
-(defn get-by-parent-id
-  [repo id]
-  (->
-    (react/q repo [:block/children (str (u-outliner/->block-id id))]
-      {:use-cache? false}
-      db-outliner/get-by-parent-id
-      (u-outliner/->block-lookup-ref id))
-    (util/react)
-    (flatten)
-    (seq)))

yarn.lock

@@ -5201,10 +5201,10 @@ mkdirp@^1.0.3:
   resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e"
   integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==
 
[email protected].7:
-  version "1.2.7"
-  resolved "https://registry.yarnpkg.com/mldoc/-/mldoc-1.2.7.tgz#3eb8b07cf33cc975e5306c1e6f81bfd72f74ee5c"
-  integrity sha512-AUB8cCNN+wHT5ktUDfR0Jqt+nw/rNybmkhKP/BEmI/kw8e6jNRHg7HIQSJEv94lrcCbkivEDbluzU6/YtKI6AA==
[email protected].8:
+  version "1.2.8"
+  resolved "https://registry.yarnpkg.com/mldoc/-/mldoc-1.2.8.tgz#e352471ca08c614d0cc2eab860b1b2749a791ebf"
+  integrity sha512-bzRYfeBzBSuenlwdzPb1AUohRrcmwg6t3o+y2YPXBPF2HpxuaAoBn3oMCBQ36IvVaLugyYqrq4dSrFbkK7VUPg==
   dependencies:
     yargs "^12.0.2"
 
@@ -6711,8 +6711,6 @@ [email protected]:
   version "2.2.7"
   resolved "https://registry.yarnpkg.com/react-icons/-/react-icons-2.2.7.tgz#d7860826b258557510dac10680abea5ca23cf650"
   integrity sha512-0n4lcGqzJFcIQLoQytLdJCE0DKSA9dkwEZRYoGrIDJZFvIT6Hbajx5mv9geqhqFiNjUgtxg8kPyDfjlhymbGFg==
-  dependencies:
-    react-icon-base "2.1.0"
 
 react-is@^16.13.1, react-is@^16.3.1:
   version "16.13.1"