wip: file worker

deps/db/src/logseq/db.cljs

@@ -10,6 +10,7 @@
             [logseq.db.sqlite.util :as sqlite-util]
             [clojure.string :as string]
             [logseq.graph-parser.util :as gp-util]
+            [logseq.graph-parser.config :as gp-config]
             [logseq.db.frontend.content :as db-content]))
 
 ;; Use it as an input argument for datalog queries
@@ -133,3 +134,20 @@
           first
           flatten-tree
           (->> (map #(db-content/update-block-content repo db % (:db/id %))))))
+
+(defn whiteboard-page?
+  "Given a page name or a page object, check if it is a whiteboard page"
+  [db page]
+  (cond
+    (string? page)
+    (let [page (d/entity db [:block/name (gp-util/page-name-sanity-lc page)])]
+      (or
+       (contains? (set (:block/type page)) "whiteboard")
+       (when-let [file (:block/file page)]
+         (when-let [path (:file/path (d/entity db (:db/id file)))]
+           (gp-config/whiteboard? path)))))
+
+    (seq page)
+    (contains? (set (:block/type page)) "whiteboard")
+
+    :else false))

deps/outliner/src/logseq/outliner/tree.cljs

@@ -129,7 +129,7 @@
 (defn get-sorted-block-and-children
   [repo db db-id]
   (when db-id
-    (when-let [root-block (d/pull db db-id)]
+    (when-let [root-block (d/pull db '[*]  db-id)]
       (let [blocks (ldb/get-block-and-children repo db (:block/uuid root-block))
             blocks-exclude-root (remove (fn [b] (= (:db/id b) db-id)) blocks)]
         (sort-blocks blocks-exclude-root root-block)))))

scripts/src/logseq/tasks/dev/db_and_file_graphs.clj

@@ -25,8 +25,7 @@
          "frontend.db.file-based"
          "frontend.fs"
          "frontend.components.conversion" "frontend.components.file-sync"
-         "frontend.util.fs"
-         "frontend.modules.outliner.file"]))
+         "frontend.util.fs"]))
 
 (def db-graph-paths
   "Paths _only_ for DB graphs"

src/bench/frontend/benchmark_test_runner.cljs

@@ -2,10 +2,10 @@
   "Runs a benchmark"
   (:require [clojure.edn :as edn]
             [frontend.macros :refer [slurped]]
-            [frontend.modules.file.uprint :as up]
             [clojure.pprint :as pprint]
             [clojure.test :refer [deftest testing]]
-            [fipp.edn :as fipp]))
+            [fipp.edn :as fipp]
+            [frontend.worker.file.util :as wfu]))
 
 (def onboarding
   (edn/read-string (slurped "resources/whiteboard/onboarding.edn")))
@@ -19,10 +19,10 @@
                       (with-out-str (fipp/pprint onboarding))
                       10)
     (simple-benchmark []
-                      (up/ugly-pr-str onboarding)
+                      (wfu/ugly-pr-str onboarding)
                       10)
     (simple-benchmark []
                       (pr-str onboarding)
                       10)
     ;; uncomment to see the output
-    #_(println (up/ugly-pr-str onboarding))))
+    #_(println (wfu/ugly-pr-str onboarding))))

src/main/frontend/date.cljs

@@ -7,10 +7,10 @@
             [cljs-time.format :as tf]
             [cljs-time.local :as tl]
             [frontend.state :as state]
-            [logseq.graph-parser.util :as gp-util]
             [logseq.graph-parser.date-time-util :as date-time-util]
             [goog.object :as gobj]
-            [lambdaisland.glogi :as log]))
+            [lambdaisland.glogi :as log]
+            [frontend.worker.date :as worker-date]))
 
 (defn nld-parse
   [s]
@@ -21,39 +21,7 @@
 
 (defn journal-title-formatters
   []
-  (->
-   (cons
-    (state/get-date-formatter)
-    (list
-     "do MMM yyyy"
-     "do MMMM yyyy"
-     "MMM do, yyyy"
-     "MMMM do, yyyy"
-     "E, dd-MM-yyyy"
-     "E, dd.MM.yyyy"
-     "E, MM/dd/yyyy"
-     "E, yyyy/MM/dd"
-     "EEE, dd-MM-yyyy"
-     "EEE, dd.MM.yyyy"
-     "EEE, MM/dd/yyyy"
-     "EEE, yyyy/MM/dd"
-     "EEEE, dd-MM-yyyy"
-     "EEEE, dd.MM.yyyy"
-     "EEEE, MM/dd/yyyy"
-     "EEEE, yyyy/MM/dd"
-     "dd-MM-yyyy"
-     ;; This tyle will mess up other date formats like "2022-08" "2022Q4" "2022/10"
-     ;;  "dd.MM.yyyy"
-     "MM/dd/yyyy"
-     "MM-dd-yyyy"
-     "MM_dd_yyyy"
-     "yyyy/MM/dd"
-     "yyyy-MM-dd"
-     "yyyy-MM-dd EEEE"
-     "yyyy_MM_dd"
-     "yyyyMMdd"
-     "yyyy年MM月dd日"))
-   (distinct)))
+  (worker-date/journal-title-formatters (state/get-date-formatter)))
 
 (defn get-date-time-string
   ([]
@@ -142,25 +110,12 @@
                  :hourCycle "h23"}))))
 
 (defn normalize-date
-  "Given raw date string, return a normalized date string at best effort.
-   Warning: this is a function with heavy cost (likely 50ms). Use with caution"
   [s]
-  (some
-   (fn [formatter]
-     (try
-       (tf/parse (tf/formatter formatter) s)
-       (catch :default _e
-         false)))
-   (journal-title-formatters)))
+  (worker-date/normalize-date s (state/get-date-formatter)))
 
 (defn normalize-journal-title
-  "Normalize journal title at best effort. Return nil if title is not a valid date.
-   Return goog.date.Date.
-
-   Return format: 20220812T000000"
   [title]
-  (and title
-       (normalize-date (gp-util/capitalize-all title))))
+  (worker-date/normalize-journal-title title (state/get-date-formatter)))
 
 (defn valid-journal-title?
   "This is a loose rule, requires double check by journal-title->custom-format.

src/main/frontend/db/model.cljs

@@ -1275,19 +1275,7 @@ independent of format as format specific heading characters are stripped"
 (defn whiteboard-page?
   "Given a page name or a page object, check if it is a whiteboard page"
   [page]
-  (cond
-    (string? page)
-    (let [page (db-utils/entity [:block/name (util/safe-page-name-sanity-lc page)])]
-      (or
-       (contains? (set (:block/type page)) "whiteboard")
-       (when-let [file (:block/file page)]
-         (when-let [path (:file/path (db-utils/entity (:db/id file)))]
-           (gp-config/whiteboard? path)))))
-
-    (seq page)
-    (contains? (set (:block/type page)) "whiteboard")
-
-    :else false))
+  (ldb/whiteboard-page? (conn/get-db) page))
 
 (defn get-orphaned-pages
   [{:keys [repo pages empty-ref-f]

src/main/frontend/extensions/srs.cljs

@@ -25,7 +25,7 @@
             [frontend.template :as template]
             [frontend.ui :as ui]
             [frontend.util :as util]
-            [frontend.util.block-content :as content]
+            [frontend.format.mldoc :as mldoc]
             [frontend.util.drawer :as drawer]
             [frontend.util.persist-var :as persist-var]
             [logseq.graph-parser.property :as gp-property]
@@ -787,7 +787,7 @@
             content (-> (property-file/remove-built-in-properties-when-file-based
                          (state/get-current-repo) (:block/format block) content)
                         (drawer/remove-logbook))
-            [title body] (content/get-title&body content format)]
+            [title body] (mldoc/get-title&body content format)]
         [block (str title " #" card-hash-tag "\n" body)])))
 
 (defn make-block-a-card!

src/main/frontend/format/mldoc.cljs

@@ -13,7 +13,9 @@
             [logseq.graph-parser.text :as text]
             [logseq.graph-parser.block :as gp-block]
             [clojure.walk :as walk]
-            [cljs-bean.core :as bean]))
+            [cljs-bean.core :as bean]
+            [frontend.worker.mldoc :as mldoc-worker]
+            [frontend.worker.mldoc :as worker-mldoc]))
 
 (defonce anchorLink (gobj/get Mldoc "anchorLink"))
 (defonce parseOPML (gobj/get Mldoc "parseOPML"))
@@ -38,12 +40,7 @@
                       title
                       (or references gp-mldoc/default-references)))
 
-(defn block-with-title?
-  [type]
-  (contains? #{"Paragraph"
-               "Raw_Html"
-               "Hiccup"
-               "Heading"} type))
+(def block-with-title? worker-mldoc/block-with-title?)
 
 (defn opml->edn
   [config content]
@@ -57,20 +54,12 @@
       [])))
 
 (defn get-default-config
-  "Gets a mldoc default config for the given format. Works for DB and file graphs"
   [format]
-  (let [db-based? (config/db-based-graph? (state/get-current-repo))]
-    (->>
-     (cond-> (gp-mldoc/default-config-map format)
-       db-based?
-       (assoc :enable_drawers false))
-     bean/->js
-     js/JSON.stringify)))
+  (mldoc-worker/get-default-config (state/get-current-repo) format))
 
 (defn ->edn
-  "Wrapper around gp-mldoc/->edn that builds mldoc config given a format"
   [content format]
-  (gp-mldoc/->edn content (get-default-config format)))
+  (mldoc-worker/->edn (state/get-current-repo) content format))
 
 (defrecord MldocMode []
   protocol/Format
@@ -87,9 +76,7 @@
   [plains]
   (string/join (map last plains)))
 
-(defn properties?
-  [ast]
-  (contains? #{"Properties" "Property_Drawer"} (ffirst ast)))
+(def properties? worker-mldoc/properties?)
 
 (defn typ-drawer?
   [ast typ]
@@ -167,4 +154,11 @@
      ast)
     (->> @*result
          (remove string/blank?)
-         (distinct))))
+         (distinct))))
+
+(defn get-title&body
+  "parses content and returns [title body]
+   returns nil if no title"
+  [content format]
+  (when-let [repo (state/get-current-repo)]
+    (worker-mldoc/get-title&body repo content format)))

src/main/frontend/handler.cljs

@@ -32,7 +32,6 @@
             [frontend.idb :as idb]
             [frontend.mobile.util :as mobile-util]
             [frontend.modules.instrumentation.core :as instrument]
-            [frontend.modules.outliner.file :as file]
             [frontend.modules.shortcut.core :as shortcut]
             [frontend.state :as state]
             [frontend.util :as util]
@@ -223,7 +222,6 @@
        (p/finally (fn []
                     (state/set-db-restoring! false))))
 
-   (file/<ratelimit-file-writes!)
    (util/<app-wake-up-from-sleep-loop (atom false))
 
    (when config/dev?

src/main/frontend/handler/events.cljs

@@ -61,7 +61,6 @@
             [frontend.mobile.util :as mobile-util]
             [frontend.modules.instrumentation.posthog :as posthog]
             [frontend.modules.instrumentation.sentry :as sentry-event]
-            [frontend.modules.outliner.file :as outliner-file]
             [frontend.modules.shortcut.core :as st]
             [frontend.quick-capture :as quick-capture]
             [frontend.search :as search]
@@ -183,12 +182,14 @@
    state/set-state! :sync-graph/init? false))
 
 (defmethod handle :graph/switch [[_ graph opts]]
-  (if (or (not (false? (get @outliner-file/*writes-finished? graph)))
-          (:sync-graph/init? @state/state))
-    (graph-switch-on-persisted graph opts)
-    (notification/show!
-     "Please wait seconds until all changes are saved for the current graph."
-     :warning)))
+  ;; FIXME: wait for writes finished
+  ;; (if (or (not (false? (get @outliner-file/*writes-finished? graph)))
+  ;;         (:sync-graph/init? @state/state))
+  ;;   (graph-switch-on-persisted graph opts)
+  ;;   (notification/show!
+  ;;    "Please wait seconds until all changes are saved for the current graph."
+  ;;    :warning))
+  )
 
 (defmethod handle :graph/pull-down-remote-graph [[_ graph dir-name]]
   (if (mobile-util/native-ios?)

src/main/frontend/handler/file_based/page.cljs

@@ -17,7 +17,6 @@
             [frontend.util :as util]
             [frontend.util.fs :as fs-util]
             [frontend.modules.outliner.core :as outliner-core]
-            [frontend.modules.outliner.file :as outliner-file]
             [logseq.outliner.tree :as otree]
             [frontend.fs :as fs]
             [logseq.graph-parser.property :as gp-property]
@@ -148,9 +147,7 @@
                                                     (map :db/id)
                                                     (set))})))) blocks)
                       (remove nil?))]
-    (db/transact! repo tx)
-    (doseq [page-id page-ids]
-      (outliner-file/sync-to-file page-id))))
+    (db/transact! repo tx)))
 
 (defn- compute-new-file-path
   "Construct the full path given old full path and the file sanitized body.
@@ -237,9 +234,7 @@
 
         (rename-update-refs! page old-original-name new-name)
 
-        (page-common-handler/rename-update-namespace! page old-original-name new-name)
-
-        (outliner-file/sync-to-file page))
+        (page-common-handler/rename-update-namespace! page old-original-name new-name))
 
       ;; Redirect to the newly renamed page
       (when redirect?
@@ -341,7 +336,6 @@
                              (= (:block/parent block) {:db/id from-id})
                              (assoc :block/parent {:db/id to-id})))) blocks)]
       (db/transact! repo tx-data)
-      (outliner-file/sync-to-file {:db/id to-id})
 
       (rename-update-refs! from-page
                            (util/get-page-original-name from-page)

src/main/frontend/handler/file_based/page_property.cljs

@@ -3,7 +3,6 @@
   (:require [clojure.string :as string]
             [frontend.db :as db]
             [frontend.modules.outliner.core :as outliner-core]
-            [frontend.modules.outliner.file :as outliner-file]
             [frontend.modules.outliner.transaction :as outliner-tx]
             [frontend.state :as state]
             [frontend.util :as util]))
@@ -87,5 +86,4 @@
             (outliner-tx/transact!
              {:outliner-op :insert-blocks
               :additional-tx page-properties-tx}
-             (outliner-core/insert-blocks! block page {:sibling? false}))))
-        (outliner-file/sync-to-file page-id)))))
+             (outliner-core/insert-blocks! block page {:sibling? false}))))))))

src/main/frontend/handler/file_based/property/util.cljs

@@ -4,7 +4,6 @@
             [frontend.util :as util]
             [clojure.set :as set]
             [frontend.config :as config]
-            [logseq.graph-parser.util :as gp-util]
             [logseq.graph-parser.property :as gp-property :refer [properties-start properties-end]]
             [logseq.graph-parser.util.page-ref :as page-ref]
             [frontend.format.mldoc :as mldoc]
@@ -12,7 +11,7 @@
             [frontend.db :as db]
             [frontend.state :as state]
             [frontend.util.cursor :as cursor]
-            [frontend.util.block-content :as content]))
+            [frontend.worker.file.property-util :as wpu]))
 
 (defn hidden-properties
   "These are properties hidden from user including built-in ones and ones
@@ -36,17 +35,7 @@
                     "")
     content))
 
-(defn- simplified-property?
-  [line]
-  (boolean
-   (and (string? line)
-        (re-find (re-pattern (str "^\\s?[^ ]+" gp-property/colons)) line))))
-
-(defn- front-matter-property?
-  [line]
-  (boolean
-   (and (string? line)
-        (util/safe-re-find #"^\s*[^ ]+:" line))))
+(def simplified-property? wpu/simplified-property?)
 
 (defn- get-property-key
   [line format]
@@ -158,16 +147,6 @@
     :else
     content))
 
-(defn- build-properties-str
-  [format properties]
-  (when (seq properties)
-    (let [org? (= format :org)
-          kv-format (if org? ":%s: %s" (str "%s" gp-property/colons " %s"))
-          full-format (if org? ":PROPERTIES:\n%s\n:END:" "%s\n")
-          properties-content (->> (map (fn [[k v]] (util/format kv-format (name k) v)) properties)
-                                  (string/join "\n"))]
-      (util/format full-format properties-content))))
-
 ;; title properties body
 (defn with-built-in-properties
   [properties content format]
@@ -220,92 +199,8 @@
   ([format content key value]
    (insert-property format content key value false))
   ([format content key value front-matter?]
-   (when (string? content)
-     (let [ast (mldoc/->edn content format)
-           title? (mldoc/block-with-title? (ffirst (map first ast)))
-           has-properties? (or (and title?
-                                    (or (mldoc/properties? (second ast))
-                                        (mldoc/properties? (second
-                                                            (remove
-                                                             (fn [[x _]]
-                                                               (contains? #{"Hiccup" "Raw_Html"} (first x)))
-                                                             ast)))))
-                               (mldoc/properties? (first ast)))
-           lines (string/split-lines content)
-           [title body] (content/get-title&body content format)
-           scheduled (filter #(string/starts-with? % "SCHEDULED") lines)
-           deadline (filter #(string/starts-with? % "DEADLINE") lines)
-           body-without-timestamps (filter
-                                    #(not (or (string/starts-with? % "SCHEDULED")
-                                              (string/starts-with? % "DEADLINE")))
-                                    (string/split-lines body))
-           org? (= :org format)
-           key (string/lower-case (name key))
-           value (string/trim (str value))
-           start-idx (.indexOf lines properties-start)
-           end-idx (.indexOf lines properties-end)
-           result (cond
-                    (and org? (not has-properties?))
-                    (let [properties (build-properties-str format {key value})]
-                      (if title
-                        (string/join "\n" (concat [title] scheduled deadline [properties] body-without-timestamps))
-                        (str properties "\n" content)))
-
-                    (and has-properties? (>= start-idx 0) (> end-idx 0) (> end-idx start-idx))
-                    (let [exists? (atom false)
-                          before (subvec lines 0 start-idx)
-                          middle (doall
-                                  (->> (subvec lines (inc start-idx) end-idx)
-                                       (mapv (fn [text]
-                                               (let [[k v] (gp-util/split-first ":" (subs text 1))]
-                                                 (if (and k v)
-                                                   (let [key-exists? (= k key)
-                                                         _ (when key-exists? (reset! exists? true))
-                                                         v (if key-exists? value v)]
-                                                     (str ":" k ": "  (string/trim v)))
-                                                   text))))))
-                          middle (if @exists? middle (conj middle (str ":" key ": "  value)))
-                          after (subvec lines (inc end-idx))
-                          lines (concat before [properties-start] middle [properties-end] after)]
-                      (string/join "\n" lines))
-
-                    (not org?)
-                    (let [exists? (atom false)
-                          sym (if front-matter? ": " (str gp-property/colons " "))
-                          new-property-s (str key sym value)
-                          property-f (if front-matter? front-matter-property? simplified-property?)
-                          groups (partition-by property-f lines)
-                          compose-lines (fn []
-                                          (mapcat (fn [lines]
-                                                    (if (property-f (first lines))
-                                                      (let [lines (doall
-                                                                   (mapv (fn [text]
-                                                                           (let [[k v] (gp-util/split-first sym text)]
-                                                                             (if (and k v)
-                                                                               (let [key-exists? (= k key)
-                                                                                     _ (when key-exists? (reset! exists? true))
-                                                                                     v (if key-exists? value v)]
-                                                                                 (str k sym  (string/trim v)))
-                                                                               text)))
-                                                                         lines))
-                                                            lines (if @exists? lines (conj lines new-property-s))]
-                                                        lines)
-                                                      lines))
-                                                  groups))
-                          lines (cond
-                                  has-properties?
-                                  (compose-lines)
-
-                                  title?
-                                  (cons (first lines) (cons new-property-s (rest lines)))
-
-                                  :else
-                                  (cons new-property-s lines))]
-                      (string/join "\n" lines))
-
-                    :else
-                    content)]
-       (string/trimr result)))))
+   (let [repo (state/get-current-repo)]
+     (wpu/insert-property repo format content key value front-matter?))))
 
 (defn insert-properties
   [format content kvs]
@@ -325,22 +220,7 @@
        (insert-property format content k v)))
    content kvs))
 
-(defn remove-property
-  ([format key content]
-   (remove-property format key content true))
-  ([format key content first?]
-   (when (not (string/blank? (name key)))
-     (let [format (or format :markdown)
-           key (string/lower-case (name key))
-           remove-f (if first? util/remove-first remove)]
-       (if (and (= format :org) (not (gp-property/contains-properties? content)))
-         content
-         (let [lines (->> (string/split-lines content)
-                          (remove-f (fn [line]
-                                      (let [s (string/triml (string/lower-case line))]
-                                        (or (string/starts-with? s (str ":" key ":"))
-                                            (string/starts-with? s (str key gp-property/colons " ")))))))]
-           (string/join "\n" lines)))))))
+(def remove-property wpu/remove-property)
 
 (defn remove-id-property
   [format content]

src/main/frontend/handler/whiteboard.cljs

@@ -9,7 +9,6 @@
             [frontend.handler.route :as route-handler]
             [frontend.handler.property.util :as pu]
             [frontend.modules.editor.undo-redo :as history]
-            [frontend.modules.outliner.file :as outliner-file]
             [frontend.state :as state]
             [frontend.config :as config]
             [frontend.storage :as storage]
@@ -210,11 +209,6 @@
    (let [uuid (or (and name (parse-uuid name)) (d/squuid))
          name (or name (str uuid))]
      (db/transact! (get-default-new-whiteboard-tx name uuid))
-     (let [page-entity (get-whiteboard-entity name)]
-       (when (and page-entity
-                  (nil? (:block/file page-entity))
-                  (not (config/db-based-graph? (state/get-current-repo))))
-         (outliner-file/sync-to-file page-entity)))
      name)))
 
 (defn create-new-whiteboard-and-redirect!

src/main/frontend/modules/file/core.cljs

@@ -6,7 +6,6 @@
             [frontend.db.model :as model]
             [frontend.db.utils :as db-utils]
             [frontend.handler.file :as file-handler]
-            [frontend.modules.file.uprint :as up]
             [frontend.state :as state]
             [frontend.util.fs :as fs-util]
             [frontend.handler.file-based.property.util :as property-util]
@@ -152,7 +151,7 @@
     (if (and (string? file-path) (not-empty file-path))
       (let [new-content (if (contains? (:block/type page-block) "whiteboard")
                           (->
-                           (up/ugly-pr-str {:blocks tree
+                           (wfu/ugly-pr-str {:blocks tree
                                             :pages (list (remove-transit-ids page-block))})
                            (string/triml))
                           (tree->file-content tree {:init-level init-level}))]

src/main/frontend/modules/file/uprint.cljs

@@ -1,18 +0,0 @@
-(ns frontend.modules.file.uprint
-  "A fast pprint alternative.")
-
-(defn print-prefix-map* [prefix m print-one writer opts]
-  (pr-sequential-writer
-    writer
-    (fn [e w opts]
-      (print-one (key e) w opts)
-      (-write w \space)
-      (print-one (val e) w opts))
-    (str prefix "\n{") \newline "}"
-    opts (seq m)))
-
-(defn ugly-pr-str
-  "Ugly printing fast, with newlines so that git diffs are smaller"
-  [x]
-  (with-redefs [print-prefix-map print-prefix-map*]
-    (pr-str x)))

src/main/frontend/modules/outliner/file.cljs

@@ -1,117 +0,0 @@
-(ns frontend.modules.outliner.file
-  (:require [clojure.core.async :as async]
-            [clojure.string :as string]
-            [frontend.config :as config]
-            [frontend.db :as db]
-            [frontend.db.model :as model]
-            [frontend.handler.notification :as notification]
-            [frontend.modules.file.core :as file]
-            [frontend.modules.outliner.tree :as tree]
-            [frontend.util :as util]
-            [goog.object :as gobj]
-            [lambdaisland.glogi :as log]
-            [frontend.state :as state]
-            [cljs-time.core :as t]
-            [cljs-time.coerce :as tc]))
-
-(def batch-write-interval 1000)
-
-(def whiteboard-blocks-pull-keys-with-persisted-ids
-  '[:block/properties
-    :block/uuid
-    :block/content
-    :block/format
-    :block/created-at
-    :block/updated-at
-    :block/collapsed?
-    {:block/page      [:block/uuid]}
-    {:block/left      [:block/uuid]}
-    {:block/parent    [:block/uuid]}])
-
-(defn- cleanup-whiteboard-block
-  [block]
-  (if (get-in block [:block/properties :ls-type] false)
-    (dissoc block
-            :db/id
-            :block/uuid ;; shape block uuid is read from properties
-            :block/collapsed?
-            :block/content
-            :block/format
-            :block/left
-            :block/page
-            :block/parent) ;; these are auto-generated for whiteboard shapes
-    (dissoc block :db/id :block/page)))
-
-
-(defn do-write-file!
-  [repo page-db-id outliner-op]
-  (let [page-block (db/pull repo '[*] page-db-id)
-        page-db-id (:db/id page-block)
-        whiteboard? (contains? (:block/type page-block) "whiteboard")
-        blocks-count (model/get-page-blocks-count repo page-db-id)
-        blocks-just-deleted? (and (zero? blocks-count)
-                                  (contains? #{:delete-blocks :move-blocks} outliner-op))]
-    (when (or (>= blocks-count 1) blocks-just-deleted?)
-      (if (or (and (> blocks-count 500)
-                   (not (state/input-idle? repo {:diff 3000}))) ;; long page
-              ;; when this whiteboard page is just being updated
-              (and whiteboard? (not (state/whiteboard-idle? repo))))
-        (async/put! (state/get-file-write-chan) [repo page-db-id outliner-op (tc/to-long (t/now))])
-        (let [pull-keys (if whiteboard? whiteboard-blocks-pull-keys-with-persisted-ids '[*])
-              blocks (model/get-page-blocks-no-cache repo (:block/name page-block) {:pull-keys pull-keys})
-              blocks (if whiteboard? (map cleanup-whiteboard-block blocks) blocks)]
-          (when-not (and (= 1 (count blocks))
-                         (string/blank? (:block/content (first blocks)))
-                         (nil? (:block/file page-block)))
-            (let [tree-or-blocks (if whiteboard? blocks
-                                     (tree/blocks->vec-tree repo blocks (:block/name page-block)))]
-              (if page-block
-                (file/save-tree! page-block tree-or-blocks blocks-just-deleted?)
-                (js/console.error (str "can't find page id: " page-db-id))))))))))
-
-(defn write-files!
-  [pages]
-  (when (seq pages)
-    (when-not config/publishing?
-      (doseq [[repo page-id outliner-op] (set (map #(take 3 %) pages))] ; remove time to dedupe pages to write
-        (try (do-write-file! repo page-id outliner-op)
-             (catch :default e
-               (notification/show!
-                [:div
-                 [:p "Write file failed, please copy the changes to other editors in case of losing data."]
-                 "Error: " (str (gobj/get e "stack"))]
-                :error)
-               (log/error :file/write-file-error {:error e})))))))
-
-(defn sync-to-file
-  ([page]
-   (sync-to-file page nil))
-  ([{page-db-id :db/id} outliner-op]
-   (if (nil? page-db-id)
-     (notification/show!
-      "Write file failed, can't find the current page!"
-      :error)
-     (when-let [repo (state/get-current-repo)]
-       (if (:graph/importing @state/state) ; write immediately
-         (write-files! [[repo page-db-id outliner-op]])
-         (async/put! (state/get-file-write-chan) [repo page-db-id outliner-op (tc/to-long (t/now))]))))))
-
-(def *writes-finished? (atom {}))
-
-(defn <ratelimit-file-writes!
-  []
-  (util/<ratelimit (state/get-file-write-chan) batch-write-interval
-                   :filter-fn
-                   (fn [[repo _ _ time]]
-                     (swap! *writes-finished? assoc repo {:time time
-                                                          :value false})
-                     true)
-                   :flush-fn
-                   (fn [col]
-                     (let [start-time (tc/to-long (t/now))
-                           repos (distinct (map first col))]
-                       (write-files! col)
-                       (doseq [repo repos]
-                         (let [last-write-time (get-in @*writes-finished? [repo :time])]
-                           (when (> start-time last-write-time)
-                             (swap! *writes-finished? assoc repo {:value true}))))))))

src/main/frontend/modules/outliner/pipeline.cljs

@@ -4,18 +4,10 @@
             [frontend.db :as db]
             [frontend.db.react :as react]
             [frontend.handler.file-based.property.util :as property-util]
-            [frontend.modules.outliner.file :as file]
             [frontend.state :as state]
             [frontend.util.cursor :as cursor]
             [frontend.util.drawer :as drawer]))
 
-(defn updated-page-hook
-  [tx-meta page]
-  (when (and
-         (not (config/db-based-graph? (state/get-current-repo)))
-         (not (:created-from-journal-template? tx-meta)))
-    (file/sync-to-file page (:outliner-op tx-meta))))
-
 (defn- reset-editing-block-content!
   [tx-data tx-meta]
   (let [repo (state/get-current-repo)
@@ -41,7 +33,7 @@
                   (when pos (cursor/move-cursor-to input pos)))))))))))
 
 (defn invoke-hooks
-  [{:keys [tx-meta tx-data deleted-block-uuids affected-keys pages blocks]}]
+  [{:keys [tx-meta tx-data deleted-block-uuids affected-keys blocks]}]
   (let [{:keys [from-disk? new-graph?]} tx-meta
         repo (state/get-current-repo)
         tx-report {:tx-meta tx-meta
@@ -56,10 +48,6 @@
         (when-not importing?
           (react/refresh! repo tx-report affected-keys))
 
-        (when (not (:delete-files? tx-meta))
-          (doseq [p (seq pages)]
-            (updated-page-hook tx-meta p)))
-
         (when (and state/lsp-enabled?
                    (seq blocks)
                    (not importing?)

src/main/frontend/persist_db/browser.cljs

@@ -83,7 +83,15 @@
       (when-not (:pipeline-replace? tx-meta) ; from db worker
         (let [tx-meta' (pr-str tx-meta)
               tx-data' (pr-str tx-data)
-              context {:importing? (:graph/importing @state/state)}]
+              context (if (config/db-based-graph? repo)
+                        {}
+                        {:importing? (:graph/importing @state/state)
+                         :date-formatter (state/get-date-formatter)
+                         :export-bullet-indentation (state/get-export-bullet-indentation)
+                         :preferred-format (state/get-preferred-format)
+                         :journals-directory (config/get-journals-directory)
+                         :whiteboards-directory (config/get-whiteboards-directory)
+                         :pages-directory (config/get-pages-directory)})]
           (if sqlite
             (p/let [result (.transact sqlite repo tx-data' tx-meta'
                                       (pr-str context))

src/main/frontend/util.cljc

@@ -156,10 +156,8 @@
      []
      (string/starts-with? js/window.location.href "file://")))
 
-(defn format
-  [fmt & args]
-  #?(:cljs (apply gstring/format fmt args)
-     :clj (apply clojure.core/format fmt args)))
+#?(:cljs
+   (def format worker-util/format))
 
 #?(:cljs
    (defn evalue
@@ -1197,14 +1195,7 @@
      []
      (js/console.trace)))
 
-(defn remove-first [pred coll]
-  ((fn inner [coll]
-     (lazy-seq
-      (when-let [[x & xs] (seq coll)]
-        (if (pred x)
-          xs
-          (cons x (inner xs))))))
-   coll))
+(def remove-first worker-util/remove-first)
 
 (def pprint clojure.pprint/pprint)
 

src/main/frontend/util/block_content.cljs

@@ -1,18 +0,0 @@
-(ns frontend.util.block-content
-  "utils for text content residing in a block"
-  (:require [clojure.string :as string]
-            [frontend.format.mldoc :as mldoc]))
-
-(defn- has-title?
-  [content format]
-  (let [ast (mldoc/->edn content format)]
-    (mldoc/block-with-title? (ffirst (map first ast)))))
-
-(defn get-title&body
-  "parses content and returns [title body]
-   returns nil if no title"
-  [content format]
-  (let [lines (string/split-lines content)]
-    (if (has-title? content format)
-      [(first lines) (string/join "\n" (rest lines))]
-      [nil (string/join "\n" lines)])))

src/main/frontend/util/fs.cljs

@@ -10,7 +10,8 @@
             [frontend.fs :as fs]
             [frontend.config :as config]
             [promesa.core :as p]
-            [cljs.reader :as reader]))
+            [cljs.reader :as reader]
+            [frontend.worker.file.util :as wfu]))
 
 ;; NOTE: This is not the same ignored-path? as src/electron/electron/utils.cljs.
 ;;       The assets directory is ignored.
@@ -73,127 +74,9 @@
   (when-let [repo-dir (config/get-repo-dir repo-url)]
     (fs/read-file repo-dir file-rpath)))
 
-;; Update repo/invalid-graph-name-warning if characters change
-(def multiplatform-reserved-chars ":\\*\\?\"<>|\\#\\\\")
-
-(def reserved-chars-pattern
-  (re-pattern (str "[" multiplatform-reserved-chars "]+")))
-
-(defn include-reserved-chars?
-  "Includes reserved characters that would broken FS"
-  [s]
-  (util/safe-re-find reserved-chars-pattern s))
-
-(defn- encode-url-lowbar
-  [input]
-  (string/replace input "_" "%5F"))
-
-(defn- encode-url-percent
-  [input]
-  (string/replace input "%" "%25"))
-
-(defn- escape-namespace-slashes-and-multilowbars
-  "Encode slashes / as triple lowbars ___
-   Don't encode _ in most cases, except causing ambiguation"
-  [string]
-  (-> string
-      ;; The ambiguation is caused by the unbounded _ (possible continuation of `_`s)
-      (string/replace "___" encode-url-lowbar)
-      (string/replace "_/" encode-url-lowbar)
-      (string/replace "/_" encode-url-lowbar)
-      ;; After ambiguaous _ encoded, encode the slash
-      (string/replace "/" "___")))
-
-(def windows-reserved-filebodies
-  (set '("CON" "PRN" "AUX" "NUL" "COM1" "COM2" "COM3" "COM4" "COM5" "COM6"
-               "COM7" "COM8" "COM9" "LPT1" "LPT2" "LPT3" "LPT4" "LPT5" "LPT6" "LPT7"
-               "LPT8" "LPT9")))
-
-(defn- escape-windows-reserved-filebodies
-  "Encode reserved file names in Windows"
-  [file-body]
-  (str file-body (when (or (contains? windows-reserved-filebodies file-body)
-                           (string/ends-with? file-body "."))
-                   "/"))) ;; "___" would not break the title, but follow the Windows ruling
-
-(defn- url-encode-file-name
-  [file-name]
-  (-> file-name
-      js/encodeURIComponent
-      (string/replace "*" "%2A") ;; extra token that not involved in URI encoding
-      ))
-
-(defn- tri-lb-file-name-sanity
-  "Sanitize page-name for file name (strict), for file name in file writing.
-   Use triple lowbar as namespace separator"
-  [title]
-  (some-> title
-          gp-util/page-name-sanity ;; we want to preserve the case sensitive nature of most file systems, don't lowercase
-          (string/replace gp-util/url-encoded-pattern encode-url-percent) ;; pre-encode % in title on demand
-          (string/replace reserved-chars-pattern url-encode-file-name)
-          (escape-windows-reserved-filebodies) ;; do this before the lowbar encoding to avoid ambiguity
-          (escape-namespace-slashes-and-multilowbars)))
-
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;;     Keep for backward compatibility     ;;
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-;; Rule of dir-ver 0 (before 2022 May)
-;; Source: https://github.com/logseq/logseq/blob/1519e35e0c8308d8db90b2525bfe7a716c4cdf04/src/main/frontend/util.cljc#L930
-(defn legacy-dot-file-name-sanity
-  [page-name]
-  (when (string? page-name)
-    ;; Bug in original code, but doesn't affect the result
-    ;; https://github.com/logseq/logseq/blob/1519e35e0c8308d8db90b2525bfe7a716c4cdf04/src/main/frontend/util.cljc#L892
-    #_{:clj-kondo/ignore [:regex-checks/double-escaped-regex]}
-    (let [normalize (fn [s] (.normalize s "NFC"))
-          remove-boundary-slashes (fn [s] (when (string? s)
-                                            (let [s (if (= \/ (first s))
-                                                      (subs s 1)
-                                                      s)]
-                                              (if (= \/ (last s))
-                                                (subs s 0 (dec (count s)))
-                                                s))))
-          page (some-> page-name
-                       (remove-boundary-slashes)
-                       ;; Windows reserved path characters
-                       (string/replace #"[:\\*\\?\"<>|]+" "_")
-                       ;; for android filesystem compatibility
-                       (string/replace #"[\\#|%]+" "_")
-                       (normalize))]
-      (string/replace page #"/" "."))))
-
-;; Rule of dir-ver 0 (after 2022 May)
-;; Source: https://github.com/logseq/logseq/blob/e7110eea6790eda5861fdedb6b02c2a78b504cd9/src/main/frontend/util.cljc#L927
-(defn legacy-url-file-name-sanity
-  [page-name]
-  (let [url-encode #(some-> % str (js/encodeURIComponent) (.replace "+" "%20"))]
-    ;; Bug in original code, but doesn't affect the result
-    ;; https://github.com/logseq/logseq/blob/1519e35e0c8308d8db90b2525bfe7a716c4cdf04/src/main/frontend/util.cljc#L892
-    #_{:clj-kondo/ignore [:regex-checks/double-escaped-regex]}
-    (some-> page-name
-            gp-util/page-name-sanity
-            ;; for android filesystem compatibility
-            (string/replace #"[\\#|%]+" url-encode)
-             ;; Windows reserved path characters
-            (string/replace #"[:\\*\\?\"<>|]+" url-encode)
-            (string/replace #"/" url-encode)
-            (string/replace "*" "%2A"))))
-
-;; Register sanitization / parsing fns in:
-;; logseq.graph-parser.util (parsing only)
-;; frontend.util.fs         (sanitization only)
-;; frontend.handler.conversion (both)
-(defn file-name-sanity
-  ([title]
-   (file-name-sanity title (state/get-filename-format)))
-  ([title file-name-format]
-   (when (string? title)
-     (case file-name-format
-       :triple-lowbar (tri-lb-file-name-sanity title)
-       ;; The earliest file name rule (before May 2022). For file name check in the conversion logic only. Don't allow users to use this or show up in config, as it's not handled.
-       :legacy-dot    (legacy-dot-file-name-sanity title)
-       (legacy-url-file-name-sanity title)))))
+(def include-reserved-chars? wfu/include-reserved-chars?)
+(def windows-reserved-filebodies wfu/windows-reserved-filebodies)
+(def file-name-sanity wfu/file-name-sanity)
 
 (defn create-title-property?
   [page-name]

src/main/frontend/worker/date.cljs

@@ -0,0 +1,60 @@
+(ns frontend.worker.date
+  (:require [cljs-time.format :as tf]
+            [logseq.graph-parser.util :as gp-util]))
+
+(defn journal-title-formatters
+  [date-formatter]
+  (->
+   (cons
+    date-formatter
+    (list
+     "do MMM yyyy"
+     "do MMMM yyyy"
+     "MMM do, yyyy"
+     "MMMM do, yyyy"
+     "E, dd-MM-yyyy"
+     "E, dd.MM.yyyy"
+     "E, MM/dd/yyyy"
+     "E, yyyy/MM/dd"
+     "EEE, dd-MM-yyyy"
+     "EEE, dd.MM.yyyy"
+     "EEE, MM/dd/yyyy"
+     "EEE, yyyy/MM/dd"
+     "EEEE, dd-MM-yyyy"
+     "EEEE, dd.MM.yyyy"
+     "EEEE, MM/dd/yyyy"
+     "EEEE, yyyy/MM/dd"
+     "dd-MM-yyyy"
+     ;; This tyle will mess up other date formats like "2022-08" "2022Q4" "2022/10"
+     ;;  "dd.MM.yyyy"
+     "MM/dd/yyyy"
+     "MM-dd-yyyy"
+     "MM_dd_yyyy"
+     "yyyy/MM/dd"
+     "yyyy-MM-dd"
+     "yyyy-MM-dd EEEE"
+     "yyyy_MM_dd"
+     "yyyyMMdd"
+     "yyyy年MM月dd日"))
+   (distinct)))
+
+(defn normalize-date
+  "Given raw date string, return a normalized date string at best effort.
+   Warning: this is a function with heavy cost (likely 50ms). Use with caution"
+  [s date-formatter]
+  (some
+   (fn [formatter]
+     (try
+       (tf/parse (tf/formatter formatter) s)
+       (catch :default _e
+         false)))
+   (journal-title-formatters date-formatter)))
+
+(defn normalize-journal-title
+  "Normalize journal title at best effort. Return nil if title is not a valid date.
+   Return goog.date.Date.
+
+   Return format: 20220812T000000"
+  [title date-formatter]
+  (and title
+       (normalize-date (gp-util/capitalize-all title) date-formatter)))

src/main/frontend/worker/file.cljs

@@ -2,14 +2,11 @@
   "Save pages to files for file-based graphs"
   (:require [clojure.core.async :as async]
             [clojure.string :as string]
-            [frontend.config :as config]
             [frontend.db :as db]
             [frontend.db.model :as model]
-            [frontend.handler.notification :as notification]
-            [frontend.modules.file.core :as file]
-            [frontend.modules.outliner.tree :as tree]
+            [frontend.worker.file.core :as file]
+            [logseq.outliner.tree :as otree]
             [frontend.util :as util]
-            [goog.object :as gobj]
             [lambdaisland.glogi :as log]
             [frontend.state :as state]
             [cljs-time.core :as t]
@@ -45,7 +42,7 @@
 
 
 (defn do-write-file!
-  [repo page-db-id outliner-op]
+  [repo conn page-db-id outliner-op context]
   (let [page-block (db/pull repo '[*] page-db-id)
         page-db-id (:db/id page-block)
         whiteboard? (contains? (:block/type page-block) "whiteboard")
@@ -65,42 +62,36 @@
                          (string/blank? (:block/content (first blocks)))
                          (nil? (:block/file page-block)))
             (let [tree-or-blocks (if whiteboard? blocks
-                                     (tree/blocks->vec-tree repo blocks (:block/name page-block)))]
+                                     (otree/blocks->vec-tree repo @conn blocks (:block/name page-block)))]
               (if page-block
-                (file/save-tree! page-block tree-or-blocks blocks-just-deleted?)
+                (file/save-tree! repo conn page-block tree-or-blocks blocks-just-deleted? context)
                 (js/console.error (str "can't find page id: " page-db-id))))))))))
 
 (defn write-files!
-  [pages]
+  [conn pages context]
   (when (seq pages)
-    (when-not config/publishing?
-      (doseq [[repo page-id outliner-op] (set (map #(take 3 %) pages))] ; remove time to dedupe pages to write
-        (try (do-write-file! repo page-id outliner-op)
-             (catch :default e
-               (notification/show!
-                [:div
-                 [:p "Write file failed, please copy the changes to other editors in case of losing data."]
-                 "Error: " (str (gobj/get e "stack"))]
-                :error)
-               (log/error :file/write-file-error {:error e})))))))
+    (doseq [[repo page-id outliner-op] (set (map #(take 3 %) pages))] ; remove time to dedupe pages to write
+      (try (do-write-file! repo conn page-id outliner-op context)
+           (catch :default e
+             ;; FIXME: notification
+             ;; (notification/show!
+             ;;  [:div
+             ;;   [:p "Write file failed, please copy the changes to other editors in case of losing data."]
+             ;;   "Error: " (str (gobj/get e "stack"))]
+             ;;  :error)
+             (log/error :file/write-file-error {:error e}))))))
 
 (defn sync-to-file
-  ([page]
-   (sync-to-file page nil))
-  ([{page-db-id :db/id} outliner-op]
-   (if (nil? page-db-id)
-     (notification/show!
-      "Write file failed, can't find the current page!"
-      :error)
-     (when-let [repo (state/get-current-repo)]
-       (if (:graph/importing @state/state) ; write immediately
-         (write-files! [[repo page-db-id outliner-op]])
-         (async/put! (state/get-file-write-chan) [repo page-db-id outliner-op (tc/to-long (t/now))]))))))
+  [repo {page-db-id :db/id} outliner-op tx-meta]
+  (when (and repo page-db-id
+             (not (:created-from-journal-template? tx-meta))
+             (not (:delete-files? tx-meta)))
+    (async/put! (state/get-file-write-chan) [repo page-db-id outliner-op (tc/to-long (t/now))])))
 
 (def *writes-finished? (atom {}))
 
 (defn <ratelimit-file-writes!
-  []
+  [conn context]
   (util/<ratelimit (state/get-file-write-chan) batch-write-interval
                    :filter-fn
                    (fn [[repo _ _ time]]
@@ -111,7 +102,7 @@
                    (fn [col]
                      (let [start-time (tc/to-long (t/now))
                            repos (distinct (map first col))]
-                       (write-files! col)
+                       (write-files! conn col context)
                        (doseq [repo repos]
                          (let [last-write-time (get-in @*writes-finished? [repo :time])]
                            (when (> start-time last-write-time)

src/main/frontend/worker/file/core.cljs

@@ -0,0 +1,168 @@
+(ns frontend.worker.file.core
+  (:require [clojure.string :as string]
+            [frontend.worker.file.util :as wfu]
+            [frontend.worker.file.property-util :as property-util]
+            [logseq.common.path :as path]
+            [datascript.core :as d]
+            [logseq.db :as ldb]
+            [logseq.worker.date :as worker-date]))
+
+(defn- indented-block-content
+  [content spaces-tabs]
+  (let [lines (string/split-lines content)]
+    (string/join (str "\n" spaces-tabs) lines)))
+
+(defn- content-with-collapsed-state
+  "Only accept nake content (without any indentation)"
+  [repo format content collapsed?]
+  (cond
+    collapsed?
+    (property-util/insert-property repo format content :collapsed true)
+
+    ;; Don't check properties. Collapsed is an internal state log as property in file, but not counted into properties
+    (false? collapsed?)
+    (property-util/remove-property format :collapsed content)
+
+    :else
+    content))
+
+(defn transform-content
+  [repo db {:block/keys [collapsed? format pre-block? content left page parent properties] :as b} level {:keys [heading-to-list?]} context]
+  (let [block-ref-not-saved? (and (seq (:block/_refs (d/entity db (:db/id b))))
+                                  (not (string/includes? content (str (:block/uuid b)))))
+        heading (:heading properties)
+        markdown? (= :markdown format)
+        content (or content "")
+        pre-block? (or pre-block?
+                       (and (= page parent left) ; first block
+                            markdown?
+                            (string/includes? (first (string/split-lines content)) ":: ")))
+        content (cond
+                  pre-block?
+                  (let [content (string/trim content)]
+                    (str content "\n"))
+
+                  :else
+                  (let [
+                        ;; first block is a heading, Markdown users prefer to remove the `-` before the content
+                        markdown-top-heading? (and markdown?
+                                                   (= parent page left)
+                                                   heading)
+                        [prefix spaces-tabs]
+                        (cond
+                          (= format :org)
+                          [(->>
+                            (repeat level "*")
+                            (apply str)) ""]
+
+                          markdown-top-heading?
+                          ["" ""]
+
+                          :else
+                          (let [level (if (and heading-to-list? heading)
+                                        (if (> heading 1)
+                                          (dec heading)
+                                          heading)
+                                        level)
+                                spaces-tabs (->>
+                                             (repeat (dec level) (:export-bullet-indentation context))
+                                             (apply str))]
+                            [(str spaces-tabs "-") (str spaces-tabs "  ")]))
+                        content (if heading-to-list?
+                                  (-> (string/replace content #"^\s?#+\s+" "")
+                                      (string/replace #"^\s?#+\s?$" ""))
+                                  content)
+                        content (content-with-collapsed-state repo format content collapsed?)
+                        new-content (indented-block-content (string/trim content) spaces-tabs)
+                        sep (if (or markdown-top-heading?
+                                    (string/blank? new-content))
+                              ""
+                              " ")]
+                    (str prefix sep new-content)))
+        content (if block-ref-not-saved?
+                  (property-util/insert-property repo format content :id (str (:block/uuid b)))
+                  content)]
+    content))
+
+(defn- tree->file-content-aux
+  [repo db tree {:keys [init-level] :as opts} context]
+  (let [block-contents (transient [])]
+    (loop [[f & r] tree level init-level]
+      (if (nil? f)
+        (->> block-contents persistent! flatten (remove nil?))
+        (let [page? (nil? (:block/page f))
+              content (if page? nil (transform-content repo db f level opts context))
+              new-content
+              (if-let [children (seq (:block/children f))]
+                     (cons content (tree->file-content-aux repo db children {:init-level (inc level)} context))
+                     [content])]
+          (conj! block-contents new-content)
+          (recur r level))))))
+
+(defn tree->file-content
+  [repo db tree opts context]
+  (->> (tree->file-content-aux repo db tree opts context) (string/join "\n")))
+
+(def init-level 1)
+
+(defn- transact-file-tx-if-not-exists!
+  [conn page-block ok-handler context]
+  (when (:block/name page-block)
+    (let [format (name (get page-block :block/format (:preferred-format context)))
+          date-formatter (:date-formatter context)
+          title (string/capitalize (:block/name page-block))
+          whiteboard-page? (ldb/whiteboard-page? @conn page-block)
+          format (if whiteboard-page? "edn" format)
+          journal-page? (worker-date/valid-journal-title? title date-formatter)
+          journal-title (worker-date/normalize-journal-title title date-formatter)
+          journal-page? (and journal-page? (not (string/blank? journal-title)))
+          filename (if journal-page?
+                     (worker-date/date->file-name journal-title date-formatter)
+                     (-> (or (:block/original-name page-block) (:block/name page-block))
+                         (wfu/file-name-sanity nil)))
+          sub-dir (cond
+                    journal-page?    (:journals-directory context)
+                    whiteboard-page? (:whiteboards-directory context)
+                    :else            (:pages-directory context))
+          ext (if (= format "markdown") "md" format)
+          file-rpath (path/path-join sub-dir (str filename "." ext))
+          file {:file/path file-rpath}
+          tx [{:file/path file-rpath}
+              {:block/name (:block/name page-block)
+               :block/file file}]]
+      (d/transact! conn tx)
+      (when ok-handler (ok-handler)))))
+
+(defn- remove-transit-ids [block] (dissoc block :db/id :block/file))
+
+(defn save-tree-aux!
+  [repo db page-block tree blocks-just-deleted? context]
+  (let [page-block (d/pull db '[*] (:db/id page-block))
+        file-db-id (-> page-block :block/file :db/id)
+        file-path (-> (d/entity db file-db-id) :file/path)]
+    (if (and (string? file-path) (not-empty file-path))
+      (let [new-content (if (contains? (:block/type page-block) "whiteboard")
+                          (->
+                           (wfu/ugly-pr-str {:blocks tree
+                                             :pages (list (remove-transit-ids page-block))})
+                           (string/triml))
+                          (tree->file-content repo db tree {:init-level init-level} context))]
+        (when-not (and (string/blank? new-content) (not blocks-just-deleted?))
+          (let [files [[file-path new-content]]]
+            ;; TODO: send files to main thread to save
+            ;; (file-handler/alter-files-handler! repo files {} {})
+            )))
+      ;; In e2e tests, "card" page in db has no :file/path
+      (js/console.error "File path from page-block is not valid" page-block tree))))
+
+(defn save-tree!
+  [repo conn page-block tree blocks-just-deleted? context]
+  {:pre [(map? page-block)]}
+  (when repo
+    (let [ok-handler #(save-tree-aux! repo @conn page-block tree blocks-just-deleted? context)
+          file (or (:block/file page-block)
+                   (when-let [page-id (:db/id (:block/page page-block))]
+                     (:block/file (d/entity @conn page-id))))]
+      (if file
+        (ok-handler)
+        (transact-file-tx-if-not-exists! conn page-block ok-handler context)))))

src/main/frontend/worker/file/property_util.cljs

@@ -0,0 +1,138 @@
+(ns frontend.worker.file.property-util
+  "Property fns needed by the rest of the app and not graph-parser"
+  (:require [clojure.string :as string]
+            [logseq.graph-parser.util :as gp-util]
+            [logseq.graph-parser.property :as gp-property :refer [properties-start properties-end]]
+            [frontend.worker.mldoc :as worker-mldoc]
+            [frontend.worker.util :as util]))
+
+(defn- build-properties-str
+  [format properties]
+  (when (seq properties)
+    (let [org? (= format :org)
+          kv-format (if org? ":%s: %s" (str "%s" gp-property/colons " %s"))
+          full-format (if org? ":PROPERTIES:\n%s\n:END:" "%s\n")
+          properties-content (->> (map (fn [[k v]] (util/format kv-format (name k) v)) properties)
+                                  (string/join "\n"))]
+      (util/format full-format properties-content))))
+
+(defn simplified-property?
+  [line]
+  (boolean
+   (and (string? line)
+        (re-find (re-pattern (str "^\\s?[^ ]+" gp-property/colons)) line))))
+
+(defn- front-matter-property?
+  [line]
+  (boolean
+   (and (string? line)
+        (util/safe-re-find #"^\s*[^ ]+:" line))))
+
+(defn insert-property
+  "Only accept nake content (without any indentation)"
+  ([repo format content key value]
+   (insert-property repo format content key value false))
+  ([repo format content key value front-matter?]
+   (when (string? content)
+     (let [ast (worker-mldoc/->edn repo content format)
+           title? (worker-mldoc/block-with-title? (ffirst (map first ast)))
+           has-properties? (or (and title?
+                                    (or (worker-mldoc/properties? (second ast))
+                                        (worker-mldoc/properties? (second
+                                                            (remove
+                                                             (fn [[x _]]
+                                                               (contains? #{"Hiccup" "Raw_Html"} (first x)))
+                                                             ast)))))
+                               (worker-mldoc/properties? (first ast)))
+           lines (string/split-lines content)
+           [title body] (worker-mldoc/get-title&body repo content format)
+           scheduled (filter #(string/starts-with? % "SCHEDULED") lines)
+           deadline (filter #(string/starts-with? % "DEADLINE") lines)
+           body-without-timestamps (filter
+                                    #(not (or (string/starts-with? % "SCHEDULED")
+                                              (string/starts-with? % "DEADLINE")))
+                                    (string/split-lines body))
+           org? (= :org format)
+           key (string/lower-case (name key))
+           value (string/trim (str value))
+           start-idx (.indexOf lines properties-start)
+           end-idx (.indexOf lines properties-end)
+           result (cond
+                    (and org? (not has-properties?))
+                    (let [properties (build-properties-str format {key value})]
+                      (if title
+                        (string/join "\n" (concat [title] scheduled deadline [properties] body-without-timestamps))
+                        (str properties "\n" content)))
+
+                    (and has-properties? (>= start-idx 0) (> end-idx 0) (> end-idx start-idx))
+                    (let [exists? (atom false)
+                          before (subvec lines 0 start-idx)
+                          middle (doall
+                                  (->> (subvec lines (inc start-idx) end-idx)
+                                       (mapv (fn [text]
+                                               (let [[k v] (gp-util/split-first ":" (subs text 1))]
+                                                 (if (and k v)
+                                                   (let [key-exists? (= k key)
+                                                         _ (when key-exists? (reset! exists? true))
+                                                         v (if key-exists? value v)]
+                                                     (str ":" k ": "  (string/trim v)))
+                                                   text))))))
+                          middle (if @exists? middle (conj middle (str ":" key ": "  value)))
+                          after (subvec lines (inc end-idx))
+                          lines (concat before [properties-start] middle [properties-end] after)]
+                      (string/join "\n" lines))
+
+                    (not org?)
+                    (let [exists? (atom false)
+                          sym (if front-matter? ": " (str gp-property/colons " "))
+                          new-property-s (str key sym value)
+                          property-f (if front-matter? front-matter-property? simplified-property?)
+                          groups (partition-by property-f lines)
+                          compose-lines (fn []
+                                          (mapcat (fn [lines]
+                                                    (if (property-f (first lines))
+                                                      (let [lines (doall
+                                                                   (mapv (fn [text]
+                                                                           (let [[k v] (gp-util/split-first sym text)]
+                                                                             (if (and k v)
+                                                                               (let [key-exists? (= k key)
+                                                                                     _ (when key-exists? (reset! exists? true))
+                                                                                     v (if key-exists? value v)]
+                                                                                 (str k sym  (string/trim v)))
+                                                                               text)))
+                                                                         lines))
+                                                            lines (if @exists? lines (conj lines new-property-s))]
+                                                        lines)
+                                                      lines))
+                                                  groups))
+                          lines (cond
+                                  has-properties?
+                                  (compose-lines)
+
+                                  title?
+                                  (cons (first lines) (cons new-property-s (rest lines)))
+
+                                  :else
+                                  (cons new-property-s lines))]
+                      (string/join "\n" lines))
+
+                    :else
+                    content)]
+       (string/trimr result)))))
+
+(defn remove-property
+  ([format key content]
+   (remove-property format key content true))
+  ([format key content first?]
+   (when (not (string/blank? (name key)))
+     (let [format (or format :markdown)
+           key (string/lower-case (name key))
+           remove-f (if first? util/remove-first remove)]
+       (if (and (= format :org) (not (gp-property/contains-properties? content)))
+         content
+         (let [lines (->> (string/split-lines content)
+                          (remove-f (fn [line]
+                                      (let [s (string/triml (string/lower-case line))]
+                                        (or (string/starts-with? s (str ":" key ":"))
+                                            (string/starts-with? s (str key gp-property/colons " ")))))))]
+           (string/join "\n" lines)))))))

src/main/frontend/worker/file/util.cljs

@@ -0,0 +1,91 @@
+(ns frontend.worker.file.util
+  (:require [clojure.string :as string]
+            [logseq.graph-parser.util :as gp-util]
+            [frontend.worker.util :as util]))
+
+;; Update repo/invalid-graph-name-warning if characters change
+(def multiplatform-reserved-chars ":\\*\\?\"<>|\\#\\\\")
+
+(def reserved-chars-pattern
+  (re-pattern (str "[" multiplatform-reserved-chars "]+")))
+
+(defn- encode-url-lowbar
+  [input]
+  (string/replace input "_" "%5F"))
+
+(defn- encode-url-percent
+  [input]
+  (string/replace input "%" "%25"))
+
+(defn- escape-namespace-slashes-and-multilowbars
+  "Encode slashes / as triple lowbars ___
+   Don't encode _ in most cases, except causing ambiguation"
+  [string]
+  (-> string
+      ;; The ambiguation is caused by the unbounded _ (possible continuation of `_`s)
+      (string/replace "___" encode-url-lowbar)
+      (string/replace "_/" encode-url-lowbar)
+      (string/replace "/_" encode-url-lowbar)
+      ;; After ambiguaous _ encoded, encode the slash
+      (string/replace "/" "___")))
+
+(def windows-reserved-filebodies
+  (set '("CON" "PRN" "AUX" "NUL" "COM1" "COM2" "COM3" "COM4" "COM5" "COM6"
+               "COM7" "COM8" "COM9" "LPT1" "LPT2" "LPT3" "LPT4" "LPT5" "LPT6" "LPT7"
+               "LPT8" "LPT9")))
+
+(defn- escape-windows-reserved-filebodies
+  "Encode reserved file names in Windows"
+  [file-body]
+  (str file-body (when (or (contains? windows-reserved-filebodies file-body)
+                           (string/ends-with? file-body "."))
+                   "/"))) ;; "___" would not break the title, but follow the Windows ruling
+
+(defn- url-encode-file-name
+  [file-name]
+  (-> file-name
+      js/encodeURIComponent
+      (string/replace "*" "%2A") ;; extra token that not involved in URI encoding
+      ))
+
+(defn- tri-lb-file-name-sanity
+  "Sanitize page-name for file name (strict), for file name in file writing.
+   Use triple lowbar as namespace separator"
+  [title]
+  (some-> title
+          gp-util/page-name-sanity ;; we want to preserve the case sensitive nature of most file systems, don't lowercase
+          (string/replace gp-util/url-encoded-pattern encode-url-percent) ;; pre-encode % in title on demand
+          (string/replace reserved-chars-pattern url-encode-file-name)
+          (escape-windows-reserved-filebodies) ;; do this before the lowbar encoding to avoid ambiguity
+          (escape-namespace-slashes-and-multilowbars)))
+
+;; Register sanitization / parsing fns in:
+;; logseq.graph-parser.util (parsing only)
+;; frontend.util.fs         (sanitization only)
+;; frontend.handler.conversion (both)
+(defn file-name-sanity
+  [title _file-name-format]
+  (when (string? title)
+    (tri-lb-file-name-sanity title)))
+
+(defn include-reserved-chars?
+  "Includes reserved characters that would broken FS"
+  [s]
+  (util/safe-re-find reserved-chars-pattern s))
+
+;; A fast pprint alternative.
+(defn print-prefix-map* [prefix m print-one writer opts]
+  (pr-sequential-writer
+    writer
+    (fn [e w opts]
+      (print-one (key e) w opts)
+      (-write w \space)
+      (print-one (val e) w opts))
+    (str prefix "\n{") \newline "}"
+    opts (seq m)))
+
+(defn ugly-pr-str
+  "Ugly printing fast, with newlines so that git diffs are smaller"
+  [x]
+  (with-redefs [print-prefix-map print-prefix-map*]
+    (pr-str x)))

src/main/frontend/worker/mldoc.cljs

@@ -0,0 +1,47 @@
+(ns frontend.worker.mldoc
+  "Mldoc related fns"
+  (:require [logseq.graph-parser.mldoc :as gp-mldoc]
+            [cljs-bean.core :as bean]
+            [logseq.db.sqlite.util :as sqlite-util]
+            [clojure.string :as string]))
+
+(defn get-default-config
+  "Gets a mldoc default config for the given format. Works for DB and file graphs"
+  [repo format]
+  (let [db-based? (sqlite-util/db-based-graph? repo)]
+    (->>
+     (cond-> (gp-mldoc/default-config-map format)
+       db-based?
+       (assoc :enable_drawers false))
+     bean/->js
+     js/JSON.stringify)))
+
+(defn ->edn
+  "Wrapper around gp-mldoc/->edn that builds mldoc config given a format"
+  [repo content format]
+  (gp-mldoc/->edn content (get-default-config repo format)))
+
+(defn properties?
+  [ast]
+  (contains? #{"Properties" "Property_Drawer"} (ffirst ast)))
+
+(defn block-with-title?
+  [type]
+  (contains? #{"Paragraph"
+               "Raw_Html"
+               "Hiccup"
+               "Heading"} type))
+
+(defn- has-title?
+  [repo content format]
+  (let [ast (->edn repo content format)]
+    (block-with-title? (ffirst (map first ast)))))
+
+(defn get-title&body
+  "parses content and returns [title body]
+   returns nil if no title"
+  [repo content format]
+  (let [lines (string/split-lines content)]
+    (if (has-title? repo content format)
+      [(first lines) (string/join "\n" (rest lines))]
+      [nil (string/join "\n" lines)])))

src/main/frontend/worker/util.cljs

@@ -3,7 +3,8 @@
   (:require [clojure.string :as string]
             ["remove-accents" :as removeAccents]
             [medley.core :as medley]
-            [logseq.graph-parser.util :as gp-util]))
+            [logseq.graph-parser.util :as gp-util]
+            [goog.string :as gstring]))
 
 (defonce db-version-prefix "logseq_db_")
 (defonce local-db-prefix "logseq_local_")
@@ -55,3 +56,16 @@
 (defn distinct-by
   [f col]
   (medley/distinct-by f (seq col)))
+
+(defn format
+  [fmt & args]
+  (apply gstring/format fmt args))
+
+(defn remove-first [pred coll]
+  ((fn inner [coll]
+     (lazy-seq
+      (when-let [[x & xs] (seq coll)]
+        (if (pred x)
+          xs
+          (cons x (inner xs))))))
+   coll))