Merge pull request #5299 from logseq/enhance/graph-parser-part-trois

Enhance: Migrate block and extract to graph parser

.clj-kondo/config.edn

@@ -19,12 +19,18 @@
              frontend.db.react react
              frontend.db.query-react query-react
              frontend.util util
+             frontend.util.property property
              frontend.config config
              frontend.format.mldoc mldoc
+             frontend.format.block block
+             frontend.handler.extract extract
              logseq.graph-parser.text text
+             logseq.graph-parser.block gp-block
              logseq.graph-parser.mldoc gp-mldoc
              logseq.graph-parser.util gp-util
-             logseq.graph-parser.config gp-config}}}
+             logseq.graph-parser.property gp-property
+             logseq.graph-parser.config gp-config
+             logseq.graph-parser.date-time-util date-time-util}}}
 
  :hooks {:analyze-call {rum.core/defc hooks.rum/defc
                          rum.core/defcs hooks.rum/defcs}}

src/main/frontend/commands.cljs

@@ -517,7 +517,7 @@
 
 (defn compute-pos-delta-when-change-marker
   [edit-content marker pos]
-  (let [old-marker (some->> (first (gp-util/safe-re-find marker/bare-marker-pattern edit-content))
+  (let [old-marker (some->> (first (util/safe-re-find marker/bare-marker-pattern edit-content))
                             (string/trim))
         pos-delta (- (count marker)
                      (count old-marker))
@@ -542,7 +542,7 @@
                   (if-let [matches (seq (util/re-pos new-line-re-pattern prefix))]
                     (let [[start-pos content] (last matches)]
                       (+ start-pos (count content)))
-                    (count (gp-util/safe-re-find re-pattern prefix))))
+                    (count (util/safe-re-find re-pattern prefix))))
             new-value (str (subs edit-content 0 pos)
                            (string/replace-first (subs edit-content pos)
                                                  (marker/marker-pattern format)
@@ -583,7 +583,7 @@
       (let [edit-content (gobj/get current-input "value")
             heading-pattern #"^#+\s+"
             new-value (cond
-                        (gp-util/safe-re-find heading-pattern edit-content)
+                        (util/safe-re-find heading-pattern edit-content)
                         (string/replace-first edit-content
                                               heading-pattern
                                               (str heading " "))

src/main/frontend/components/block.cljs

@@ -55,6 +55,7 @@
             [logseq.graph-parser.config :as gp-config]
             [logseq.graph-parser.util :as gp-util]
             [logseq.graph-parser.mldoc :as gp-mldoc]
+            [logseq.graph-parser.block :as gp-block]
             [goog.dom :as gdom]
             [goog.object :as gobj]
             [lambdaisland.glogi :as log]
@@ -524,7 +525,7 @@
   "Accepts {:block/name sanitized / unsanitized page-name}"
   [{:keys [html-export? redirect-page-name label children contents-page? preview?] :as config} page]
   (when-let [page-name-in-block (:block/name page)]
-    (let [page-name-in-block (util/remove-boundary-slashes page-name-in-block)
+    (let [page-name-in-block (gp-util/remove-boundary-slashes page-name-in-block)
           page-name (util/page-name-sanity-lc page-name-in-block)
           page-entity (db/entity [:block/name page-name])
           redirect-page-name (or (and (= :org (state/get-preferred-format))
@@ -708,7 +709,7 @@
   [config id label]
   (when (and
          (not (string/blank? id))
-         (gp-util/uuid-string? id))
+         (util/uuid-string? id))
     (let [block-id (uuid id)
           block (db/pull-block block-id)
           block-type (keyword (get-in block [:block/properties :ls-type]))
@@ -989,7 +990,7 @@
                (= "Complex" protocol)
                (= (string/lower-case (:protocol path)) "id")
                (string? (:link path))
-               (gp-util/uuid-string? (:link path))) ; org mode id
+               (util/uuid-string? (:link path))) ; org mode id
           (let [id (uuid (:link path))
                 block (db/entity [:block/uuid id])]
             (if (:block/pre-block? block)
@@ -1009,7 +1010,7 @@
                   show-brackets? (state/show-brackets?)]
               (if (and page
                        (when-let [ext (util/get-file-ext href)]
-                         (config/mldoc-support? ext)))
+                         (gp-config/mldoc-support? ext)))
                 [:span.page-reference
                  (when show-brackets? [:span.text-gray-500 "[["])
                  (page-cp config page)
@@ -1105,7 +1106,7 @@
                        string/trim)]
         (when-let [id (and s
                            (let [s (string/trim s)]
-                             (and (gp-util/uuid-string? s)
+                             (and (util/uuid-string? s)
                                   (uuid s))))]
           (block-embed (assoc config :link-depth (inc link-depth)) id)))
 
@@ -1116,7 +1117,7 @@
   [_config arguments]
   (when-let [url (first arguments)]
     (let [Vimeo-regex #"^((?:https?:)?//)?((?:www).)?((?:player.vimeo.com|vimeo.com)?)((?:/video/)?)([\w-]+)(\S+)?$"]
-      (when-let [vimeo-id (nth (gp-util/safe-re-find Vimeo-regex url) 5)]
+      (when-let [vimeo-id (nth (util/safe-re-find Vimeo-regex url) 5)]
         (when-not (string/blank? vimeo-id)
           (let [width (min (- (util/get-width) 96)
                            560)
@@ -1137,7 +1138,7 @@
       (when-let [id (cond
                       (<= (count url) 15) url
                       :else
-                      (last (gp-util/safe-re-find id-regex url)))]
+                      (last (util/safe-re-find id-regex url)))]
         (when-not (string/blank? id)
           (let [width (min (- (util/get-width) 96)
                            560)
@@ -1267,7 +1268,7 @@
           (when-let [youtube-id (cond
                                   (== 11 (count url)) url
                                   :else
-                                  (nth (gp-util/safe-re-find YouTube-regex url) 5))]
+                                  (nth (util/safe-re-find YouTube-regex url) 5))]
             (when-not (string/blank? youtube-id)
               (youtube/youtube-video youtube-id)))))
 
@@ -1298,7 +1299,7 @@
           (when-let [id (cond
                           (<= (count url) 15) url
                           :else
-                          (last (gp-util/safe-re-find id-regex url)))]
+                          (last (util/safe-re-find id-regex url)))]
             (ui/tweet-embed id))))
 
       (= name "embed")
@@ -1336,7 +1337,7 @@
          (->elem :sub (map-inline config l))
 
          ["Tag" _]
-         (when-let [s (block/get-tag item)]
+         (when-let [s (gp-block/get-tag item)]
            (let [s (text/page-ref-un-brackets! s)]
              (page-cp (assoc config :tag? true) {:block/name s})))
 
@@ -2885,7 +2886,7 @@
 
         ["Paragraph" l]
              ;; TODO: speedup
-        (if (gp-util/safe-re-find #"\"Export_Snippet\" \"embed\"" (str l))
+        (if (util/safe-re-find #"\"Export_Snippet\" \"embed\"" (str l))
           (->elem :div (map-inline config l))
           (->elem :div.is-paragraph (map-inline config l)))
 

src/main/frontend/components/content.cljs

@@ -36,7 +36,7 @@
 
 (defn- lazy-load
   [format]
-  (let [format (format/normalize format)]
+  (let [format (gp-util/normalize-format format)]
     (when-let [record (format/get-format-record format)]
       (when-not (protocol/loaded? record)
         (set-format-js-loading! format true)
@@ -363,7 +363,7 @@
                            e
                            (custom-context-menu-content))
 
-                          (and block-id (gp-util/uuid-string? block-id))
+                          (and block-id (util/uuid-string? block-id))
                           (let [block (.closest target ".ls-block")]
                             (when block
                               (util/select-highlight! [block]))
@@ -388,7 +388,7 @@
                    :format format}
                   id
                   config)
-      (let [format (format/normalize format)
+      (let [format (gp-util/normalize-format format)
             loading? (get loading format)
             markup? (contains? config/html-render-formats format)
             on-click (fn [e]
@@ -446,5 +446,5 @@
   (if hiccup
     [:div
      (hiccup-content id option)]
-    (let [format (format/normalize format)]
+    (let [format (gp-util/normalize-format format)]
       (non-hiccup-content id content on-click on-hide config format))))

src/main/frontend/components/file.cljs

@@ -9,11 +9,11 @@
             [frontend.context.i18n :refer [t]]
             [frontend.date :as date]
             [frontend.db :as db]
-            [frontend.format :as format]
             [frontend.handler.export :as export-handler]
             [frontend.state :as state]
             [frontend.util :as util]
             [logseq.graph-parser.config :as gp-config]
+            [logseq.graph-parser.util :as gp-util]
             [goog.object :as gobj]
             [reitit.frontend.easy :as rfe]
             [rum.core :as rum]))
@@ -71,7 +71,7 @@
                    state)}
   [state]
   (let [path (get-path state)
-        format (format/get-format path)
+        format (gp-util/get-format path)
         original-name (db/get-file-page path)
         random-id (str (d/squuid))]
     [:div.file {:id (str "file-edit-wrapper-" random-id)}

src/main/frontend/components/journal.cljs

@@ -9,6 +9,7 @@
             [frontend.handler.page :as page-handler]
             [frontend.state :as state]
             [logseq.graph-parser.text :as text]
+            [logseq.graph-parser.util :as gp-util]
             [frontend.ui :as ui]
             [frontend.util :as util]
             [goog.object :as gobj]
@@ -51,7 +52,7 @@
                          :page))
                       (.preventDefault e)))}
        [:h1.title
-        (util/capitalize-all title)]]
+        (gp-util/capitalize-all title)]]
 
       (blocks-cp repo page format)
 

src/main/frontend/components/page.cljs

@@ -1,6 +1,6 @@
 (ns frontend.components.page
   (:require [clojure.string :as string]
-            [frontend.components.block :as block]
+            [frontend.components.block :as component-block]
             [frontend.components.content :as content]
             [frontend.components.editor :as editor]
             [frontend.components.hierarchy :as hierarchy]
@@ -15,7 +15,6 @@
             [frontend.db.model :as model]
             [frontend.extensions.graph :as graph]
             [frontend.extensions.pdf.assets :as pdf-assets]
-            [frontend.format.block :as format-block]
             [frontend.handler.common :as common-handler]
             [frontend.handler.config :as config-handler]
             [frontend.handler.editor :as editor-handler]
@@ -35,6 +34,7 @@
             [medley.core :as medley]
             [rum.core :as rum]
             [logseq.graph-parser.util :as gp-util]
+            [frontend.format.block :as block]
             [frontend.mobile.util :as mobile-util]))
 
 (defn- get-page-name
@@ -122,7 +122,7 @@
   (when page-e
     (let [page-name (or (:block/name page-e)
                         (str (:block/uuid page-e)))
-          block? (gp-util/uuid-string? page-name)
+          block? (util/uuid-string? page-name)
           block-id (and block? (uuid page-name))
           page-blocks (get-blocks repo page-name block-id)]
       (if (empty? page-blocks)
@@ -139,7 +139,7 @@
                               :document/mode? document-mode?}
                              config)
               hiccup-config (common-handler/config-with-document-mode hiccup-config)
-              hiccup (block/->hiccup page-blocks hiccup-config {})]
+              hiccup (component-block/->hiccup page-blocks hiccup-config {})]
           [:div
            (page-blocks-inner page-name page-blocks hiccup sidebar? block-id)
            (when-not config/publishing?
@@ -163,9 +163,9 @@
            (rum/with-key
              (ui/catch-error
               (ui/component-error "Failed default query:" {:content (pr-str query)})
-              (block/custom-query {:attr {:class "mt-10"}
-                                   :editor-box editor/box
-                                   :page page} query))
+              (component-block/custom-query {:attr {:class "mt-10"}
+                                             :editor-box editor/box
+                                             :page page} query))
              (str repo "-custom-query-" (:query query))))]))))
 
 (defn tagged-pages
@@ -317,7 +317,7 @@
     (let [current-repo (state/sub :git/current-repo)
           repo (or repo current-repo)
           page-name (util/page-name-sanity-lc path-page-name)
-          block? (gp-util/uuid-string? page-name)
+          block? (util/uuid-string? page-name)
           block-id (and block? (uuid page-name))
           format (let [page (if block-id
                               (:block/name (:block/page (db/entity [:block/uuid block-id])))
@@ -332,7 +332,7 @@
                       (db/entity repo))
                  (do
                    (when-not (db/entity repo [:block/name page-name])
-                     (let [m (format-block/page-name->map path-page-name true)]
+                     (let [m (block/page-name->map path-page-name true)]
                        (db/transact! repo [m])))
                    (db/pull [:block/name page-name])))
           {:keys [icon]} (:block/properties page)
@@ -377,7 +377,7 @@
            (let [config {:id "block-parent"
                          :block? true}]
              [:div.mb-4
-              (block/breadcrumb config repo block-id {:level-limit 3})]))
+              (component-block/breadcrumb config repo block-id {:level-limit 3})]))
 
          ;; blocks
          (let [page (if block?
@@ -640,7 +640,7 @@
               (date/today))
         theme (:ui/theme @state/state)
         dark? (= theme "dark")
-        graph (if (gp-util/uuid-string? page)
+        graph (if (util/uuid-string? page)
                 (graph-handler/build-block-graph (uuid page) theme)
                 (graph-handler/build-page-graph page theme))]
     (when (seq (:nodes graph))
@@ -712,7 +712,7 @@
          [:tr {:key name}
           [:td.n.w-12 [:span.opacity-70 (str (inc n) ".")]]
           [:td.name [:a {:href     (rfe/href :page {:name (:block/name page)})}
-                     (block/page-cp {} page)]]
+                     (component-block/page-cp {} page)]]
           [:td.backlinks [:span (or backlinks "0")]]
           (when-not orphaned-pages? [:td.created-at [:span (if created-at (date/int->local-time-2 created-at) "Unknown")]])
           (when-not orphaned-pages? [:td.updated-at [:span (if updated-at (date/int->local-time-2 updated-at) "Unknown")]])])]]
@@ -954,7 +954,7 @@
                                                (:db/id page)
                                                :page))))
                               :href     (rfe/href :page {:name (:block/name page)})}
-                          (block/page-cp {} page)]]
+                          (component-block/page-cp {} page)]]
 
                (when-not mobile?
                  [:td.backlinks [:span backlinks]])

src/main/frontend/components/page_menu.cljs

@@ -14,7 +14,6 @@
             [frontend.handler.shell :as shell]
             [frontend.handler.plugin :as plugin-handler]
             [frontend.mobile.util :as mobile-util]
-            [logseq.graph-parser.util :as gp-util]
             [electron.ipc :as ipc]
             [frontend.config :as config]
             [frontend.handler.user :as user-handler]
@@ -64,7 +63,7 @@
           repo (state/sub :git/current-repo)
           page (db/entity repo [:block/name page-name])
           page-original-name (:block/original-name page)
-          block? (and page (gp-util/uuid-string? page-name))
+          block? (and page (util/uuid-string? page-name))
           contents? (= page-name "contents")
           properties (:block/properties page)
           public? (true? (:public properties))

src/main/frontend/components/reference.cljs

@@ -12,7 +12,6 @@
             [frontend.state :as state]
             [frontend.ui :as ui]
             [frontend.util :as util]
-            [logseq.graph-parser.util :as gp-util]
             [medley.core :as medley]
             [rum.core :as rum]))
 
@@ -83,7 +82,7 @@
           default-collapsed? (>= (count refed-blocks-ids) threshold)
           filters-atom (get state ::filters)
           filter-state (rum/react filters-atom)
-          block? (gp-util/uuid-string? page-name)
+          block? (util/uuid-string? page-name)
           block-id (and block? (uuid page-name))
           page-name (string/lower-case page-name)
           journal? (date/valid-journal-title? (string/capitalize page-name))

src/main/frontend/components/search.cljs

@@ -20,7 +20,6 @@
             [clojure.string :as string]
             [frontend.context.i18n :refer [t]]
             [frontend.date :as date]
-            [logseq.graph-parser.util :as gp-util]
             [reitit.frontend.easy :as rfe]
             [frontend.modules.shortcut.core :as shortcut]))
 
@@ -33,7 +32,7 @@
             lc-content (util/search-normalize content)
             lc-q (util/search-normalize q)]
         (if (and (string/includes? lc-content lc-q)
-                 (not (gp-util/safe-re-find #" " q)))
+                 (not (util/safe-re-find #" " q)))
           (let [i (string/index-of lc-content lc-q)
                 [before after] [(subs content 0 i) (subs content (+ i (count q)))]]
             [:div

src/main/frontend/config.cljs

@@ -106,17 +106,9 @@
   (set/union (text-formats)
              (img-formats)))
 
-;; TODO: rename
-(defonce mldoc-support-formats
-  #{:org :markdown :md})
-
-(defn mldoc-support?
-  [format]
-  (contains? mldoc-support-formats (keyword format)))
-
 (def mobile?
   (when-not util/node-test?
-    (gp-util/safe-re-find #"Mobi" js/navigator.userAgent)))
+    (util/safe-re-find #"Mobi" js/navigator.userAgent)))
 
 ;; TODO: protocol design for future formats support
 

src/main/frontend/date.cljs

@@ -5,9 +5,10 @@
             [cljs-time.core :as t]
             [cljs-time.format :as tf]
             [cljs-time.local :as tl]
-            [clojure.string :as string]
             [frontend.state :as state]
             [frontend.util :as util]
+            [logseq.graph-parser.util :as gp-util]
+            [logseq.graph-parser.date-time-util :as date-time-util]
             [goog.object :as gobj]
             [lambdaisland.glogi :as log]))
 
@@ -16,11 +17,6 @@
   (when (string? s)
     ((gobj/get chrono "parseDate") s)))
 
-(defn format
-  [date]
-  (when-let [formatter-string (state/get-date-formatter)]
-    (tf/unparse (tf/formatter formatter-string) date)))
-
 (def custom-formatter (tf/formatter "yyyy-MM-dd'T'HH:mm:ssZZ"))
 
 (defn journal-title-formatters
@@ -55,13 +51,6 @@
      "yyyy年MM月dd日"}
    (state/get-date-formatter)))
 
-;; (tf/parse (tf/formatter "dd.MM.yyyy") "2021Q4") => 20040120T000000
-(defn safe-journal-title-formatters
-  []
-  (->> [(state/get-date-formatter) "yyyy-MM-dd" "yyyy_MM_dd"]
-       (remove string/blank?)
-       distinct))
-
 (defn get-date-time-string
   ([]
    (get-date-time-string (t/now)))
@@ -115,7 +104,7 @@
   ([]
    (journal-name (tl/local-now)))
   ([date]
-   (format date)))
+   (date-time-util/format date (state/get-date-formatter))))
 
 (defn journal-name-s [s]
   (try
@@ -183,34 +172,19 @@
 (defn valid-journal-title?
   [title]
   (and title
-       (valid? (util/capitalize-all title))))
+       (valid? (gp-util/capitalize-all title))))
 
 (defn journal-title->
   ([journal-title then-fn]
-   (journal-title-> journal-title then-fn (safe-journal-title-formatters)))
+   (journal-title-> journal-title then-fn (date-time-util/safe-journal-title-formatters (state/get-date-formatter))))
   ([journal-title then-fn formatters]
-   (when-not (string/blank? journal-title)
-     (when-let [time (->> (map
-                            (fn [formatter]
-                              (try
-                                (tf/parse (tf/formatter formatter) (util/capitalize-all journal-title))
-                                (catch js/Error _e
-                                  nil)))
-                            formatters)
-                          (filter some?)
-                          first)]
-       (then-fn time)))))
+   (date-time-util/journal-title-> journal-title then-fn formatters)))
 
 (defn journal-title->int
   [journal-title]
-  (when journal-title
-    (let [journal-title (util/capitalize-all journal-title)]
-      (journal-title-> journal-title #(util/parse-int (tf/unparse (tf/formatter "yyyyMMdd") %))))))
-
-(defn int->journal-title
-  [day]
-  (when day
-    (format (tf/parse (tf/formatter "yyyyMMdd") (str day)))))
+  (date-time-util/journal-title->int
+   journal-title
+   (date-time-util/safe-journal-title-formatters (state/get-date-formatter))))
 
 (defn journal-day->ts
   [day]
@@ -240,7 +214,7 @@
 
 (defn journal-title->custom-format
   [journal-title]
-  (journal-title-> journal-title format))
+  (journal-title-> journal-title #(date-time-util/format % (state/get-date-formatter))))
 
 (defn int->local-time-2
   [n]

src/main/frontend/db/model.cljs

@@ -12,7 +12,6 @@
             [frontend.db.conn :as conn]
             [frontend.db.react :as react]
             [frontend.db.utils :as db-utils]
-            [frontend.format :as format]
             [frontend.state :as state]
             [frontend.util :as util :refer [react]]
             [logseq.graph-parser.util :as gp-util]
@@ -294,7 +293,7 @@
       (:block/format page)
       (when-let [file (:block/file page)]
         (when-let [path (:file/path (db-utils/entity (:db/id file)))]
-          (format/get-format path)))))
+          (gp-util/get-format path)))))
    (state/get-preferred-format)
    :markdown))
 
@@ -913,7 +912,7 @@
 
 (defn get-page
   [page-name]
-  (if (gp-util/uuid-string? page-name)
+  (if (util/uuid-string? page-name)
     (db-utils/entity [:block/uuid (uuid page-name)])
     (db-utils/entity [:block/name (util/page-name-sanity-lc page-name)])))
 
@@ -1222,7 +1221,7 @@
 
 (defn get-referenced-blocks-ids
   [page-name-or-block-uuid]
-  (if (gp-util/uuid-string? (str page-name-or-block-uuid))
+  (if (util/uuid-string? (str page-name-or-block-uuid))
     (let [id (uuid page-name-or-block-uuid)]
       (get-block-referenced-blocks-ids id))
     (get-page-referenced-blocks-ids page-name-or-block-uuid)))

src/main/frontend/db/query_dsl.cljs

@@ -14,8 +14,7 @@
             [frontend.db.rules :as rules]
             [frontend.template :as template]
             [logseq.graph-parser.text :as text]
-            [frontend.util :as util]
-            [logseq.graph-parser.util :as gp-util]))
+            [frontend.util :as util]))
 
 
 ;; Query fields:
@@ -449,7 +448,7 @@ Some bindings in this fn:
                                                  (remove string/blank?)
                                                  (map (fn [x]
                                                         (if (or (contains? #{"+" "-"} (first x))
-                                                                (and (gp-util/safe-re-find #"\d" (first x))
+                                                                (and (util/safe-re-find #"\d" (first x))
                                                                      (some #(string/ends-with? x %) ["y" "m" "d" "h" "min"])))
                                                           (keyword (name x))
                                                           x)))

src/main/frontend/db/query_react.cljs

@@ -12,7 +12,6 @@
             [frontend.state :as state]
             [logseq.graph-parser.text :as text]
             [frontend.util :as util]
-            [logseq.graph-parser.util :as gp-util]
             [lambdaisland.glogi :as log]))
 
 (defn resolve-input
@@ -32,12 +31,12 @@
     ;; This sometimes runs when there isn't a current page e.g. :home route
     (some-> (state/get-current-page) string/lower-case)
     (and (keyword? input)
-         (gp-util/safe-re-find #"^\d+d(-before)?$" (name input)))
+         (util/safe-re-find #"^\d+d(-before)?$" (name input)))
     (let [input (name input)
           days (util/parse-int (subs input 0 (dec (count input))))]
       (date->int (t/minus (t/today) (t/days days))))
     (and (keyword? input)
-         (gp-util/safe-re-find #"^\d+d(-after)?$" (name input)))
+         (util/safe-re-find #"^\d+d(-after)?$" (name input)))
     (let [input (name input)
           days (util/parse-int (subs input 0 (dec (count input))))]
       (date->int (t/plus (t/today) (t/days days))))

src/main/frontend/db/react.cljs

@@ -10,7 +10,6 @@
             [frontend.db.utils :as db-utils]
             [frontend.state :as state]
             [frontend.util :as util :refer [react]]
-            [logseq.graph-parser.util :as gp-util]
             [cljs.spec.alpha :as s]
             [clojure.core.async :as async]))
 
@@ -230,7 +229,7 @@
         affected-keys (concat
                        (mapcat
                         (fn [block-id]
-                          (let [block-id (if (and (string? block-id) (gp-util/uuid-string? block-id))
+                          (let [block-id (if (and (string? block-id) (util/uuid-string? block-id))
                                            [:block/uuid block-id]
                                            block-id)]
                             (when-let [block (db-utils/entity block-id)]

src/main/frontend/extensions/html_parser.cljs

@@ -4,7 +4,6 @@
             [clojure.walk :as walk]
             [frontend.config :as config]
             [frontend.util :as util]
-            [logseq.graph-parser.util :as gp-util]
             [hickory.core :as hickory]))
 
 (defonce *inside-pre? (atom false))
@@ -75,7 +74,7 @@
                                 :h6 (block-transform 6 children)
                                 :a (let [href (:href attrs)
                                          label (map-join children)
-                                         has-img-tag? (gp-util/safe-re-find #"\[:img" (str x))]
+                                         has-img-tag? (util/safe-re-find #"\[:img" (str x))]
                                      (if has-img-tag?
                                        (export-hiccup x)
                                        (case format

src/main/frontend/format.cljs

@@ -4,6 +4,7 @@
             [frontend.format.protocol :as protocol]
             [logseq.graph-parser.mldoc :as gp-mldoc]
             [logseq.graph-parser.text :as text]
+            [logseq.graph-parser.util :as gp-util]
             [clojure.string :as string]))
 
 ;; TODO: Properly fix this circular dependency:
@@ -13,22 +14,9 @@
 (defonce mldoc-record (->MldocMode))
 (defonce adoc-record (->AdocMode))
 
-(defn normalize
-  [format]
-  (case (keyword format)
-    :md :markdown
-    :asciidoc :adoc
-    ;; default
-    (keyword format)))
-
-(defn get-format
-  [file]
-  (when file
-    (normalize (keyword (string/lower-case (last (string/split file #"\.")))))))
-
 (defn get-format-record
   [format]
-  (case (normalize format)
+  (case (gp-util/normalize-format format)
     :org
     mldoc-record
     :markdown

src/main/frontend/format/block.cljs

@@ -1,665 +1,30 @@
 (ns frontend.format.block
+  "Block code needed by app but not graph-parser"
   (:require [clojure.string :as string]
-            [clojure.walk :as walk]
-            [cljs.core.match :as match]
+            [logseq.graph-parser.block :as gp-block]
             [frontend.config :as config]
-            [frontend.date :as date]
             [frontend.db :as db]
             [frontend.format :as format]
             [frontend.state :as state]
-            [logseq.graph-parser.text :as text]
-            [logseq.graph-parser.utf8 :as utf8]
-            [frontend.util :as util]
-            [frontend.util.property :as property]
-            [logseq.graph-parser.util :as gp-util]
-            [logseq.graph-parser.config :as gp-config]
-            [logseq.graph-parser.mldoc :as gp-mldoc]
-            [lambdaisland.glogi :as log]
-            [medley.core :as medley]))
+            [logseq.graph-parser.property :as gp-property]
+            [logseq.graph-parser.mldoc :as gp-mldoc]))
 
-(defn heading-block?
-  [block]
-  (and
-   (vector? block)
-   (= "Heading" (first block))))
-
-(defn get-tag
-  [block]
-  (when-let [tag-value (and (vector? block)
-                            (= "Tag" (first block))
-                            (second block))]
-    (->
-     (map (fn [e]
-            (match/match e
-              ["Plain" s]
-              s
-              ["Link" t]
-              (let [{full_text :full_text} t]
-                full_text)
-              ["Nested_link" t]
-              (let [ {content :content} t]
-                content)
-              :else
-              ""
-              )) tag-value)
-     (string/join))))
-
-(defn get-page-reference
-  [block]
-  (let [page (cond
-               (and (vector? block) (= "Link" (first block)))
-               (let [typ (first (:url (second block)))
-                     value (second (:url (second block)))]
-                 ;; {:url ["File" "file:../pages/hello_world.org"], :label [["Plain" "hello world"]], :title nil}
-                 (or
-                  (and
-                   (= typ "Page_ref")
-                   (and (string? value)
-                        (not (or (gp-config/local-asset? value)
-                                 (gp-config/draw? value))))
-                   value)
-
-                  (and
-                   (= typ "Search")
-                   (text/page-ref? value)
-                   (text/page-ref-un-brackets! value))
-
-                  (and
-                   (= typ "Search")
-                   (not (contains? #{\# \* \/ \[} (first value)))
-                   (let [ext (some-> (util/get-file-ext value) keyword)]
-                     (when (and (not (util/starts-with? value "http:"))
-                                (not (util/starts-with? value "https:"))
-                                (not (util/starts-with? value "file:"))
-                                (not (gp-config/local-asset? value))
-                                (or (= ext :excalidraw)
-                                    (not (contains? (config/supported-formats) ext))))
-                       value)))
-
-                  (and
-                   (= typ "Complex")
-                   (= (:protocol value) "file")
-                   (:link value))
-
-                  (and
-                   (= typ "File")
-                   (second (first (:label (second block)))))))
-
-               (and (vector? block) (= "Nested_link" (first block)))
-               (let [content (:content (last block))]
-                 (subs content 2 (- (count content) 2)))
-
-               (and (vector? block)
-                    (= "Macro" (first block)))
-               (let [{:keys [name arguments]} (second block)
-                     argument (string/join ", " arguments)]
-                   (when (= name "embed")
-                     (text/page-ref-un-brackets! argument)))
-
-               (and (vector? block)
-                    (= "Tag" (first block)))
-               (let [text (get-tag block)]
-                 (text/page-ref-un-brackets! text))
-
-               :else
-               nil)]
-    (text/block-ref-un-brackets! page)))
-
-(defn get-block-reference
-  [block]
-  (when-let [block-id (cond
-                        (and (vector? block)
-                             (= "Block_reference" (first block)))
-                        (last block)
-
-                        (and (vector? block)
-                             (= "Link" (first block))
-                             (map? (second block))
-                             (= "Block_ref" (first (:url (second block)))))
-                        (second (:url (second block)))
-
-                        (and (vector? block)
-                             (= "Macro" (first block)))
-                        (let [{:keys [name arguments]} (second block)]
-                          (when (and (= name "embed")
-                                     (string? (first arguments))
-                                     (string/starts-with? (first arguments) "((")
-                                     (string/ends-with? (first arguments) "))"))
-                            (subs (first arguments) 2 (- (count (first arguments)) 2))))
-
-                        (and (vector? block)
-                             (= "Link" (first block))
-                             (map? (second block)))
-                        (if (= "id" (:protocol (second (:url (second block)))))
-                          (:link (second (:url (second block))))
-                          (let [id (second (:url (second block)))]
-                            (text/block-ref-un-brackets! id)))
-
-                        :else
-                        nil)]
-    (when (and block-id
-               (gp-util/uuid-string? block-id))
-      block-id)))
-
-(defn paragraph-block?
-  [block]
-  (and
-   (vector? block)
-   (= "Paragraph" (first block))))
-
-(defn timestamp-block?
-  [block]
-  (and
-   (vector? block)
-   (= "Timestamp" (first block))))
-
-;; TODO: we should move this to mldoc
-(defn extract-properties
-  [format properties]
-  (when (seq properties)
-    (let [properties (seq properties)
-          page-refs (->>
-                     properties
-                     (remove (fn [[k _]]
-                               (contains? #{:background-color :background_color} (keyword k))))
-                     (map last)
-                     (map (fn [v]
-                            (when (and (string? v)
-                                       (not (gp-mldoc/link? format v)))
-                              (let [v (string/trim v)
-                                    result (text/split-page-refs-without-brackets v {:un-brackets? false})]
-                                (if (coll? result)
-                                  (map text/page-ref-un-brackets! result)
-                                  [])))))
-                     (apply concat)
-                     (remove string/blank?))
-          properties (->> properties
-                          (map (fn [[k v]]
-                                 (let [k (-> (string/lower-case (name k))
-                                             (string/replace " " "-")
-                                             (string/replace "_" "-"))
-                                       k (if (contains? #{"custom_id" "custom-id"} k)
-                                           "id"
-                                           k)
-                                       v (if (coll? v)
-                                           (remove string/blank? v)
-                                           (if (string/blank? v)
-                                             nil
-                                             (text/parse-property format k v (state/get-config))))
-                                       k (keyword k)
-                                       v (if (and
-                                              (string? v)
-                                              (contains? #{:alias :aliases :tags} k))
-                                           (set [v])
-                                           v)
-                                       v (if (coll? v) (set v) v)]
-                                   [k v])))
-                          (remove #(nil? (second %))))]
-      {:properties (into {} properties)
-       :properties-order (map first properties)
-       :page-refs page-refs})))
-
-(defn- paragraph-timestamp-block?
-  [block]
-  (and (paragraph-block? block)
-       (or (timestamp-block? (first (second block)))
-           (timestamp-block? (second (second block))))))
-
-(defn extract-timestamps
-  [block]
-  (some->>
-   (second block)
-   (filter timestamp-block?)
-   (map last)
-   (into {})))
-
-;; {"Deadline" {:date {:year 2020, :month 10, :day 20}, :wday "Tue", :time {:hour 8, :min 0}, :repetition [["DoublePlus"] ["Day"] 1], :active true}}
-(defn timestamps->scheduled-and-deadline
-  [timestamps]
-  (let [timestamps (medley/map-keys (comp keyword string/lower-case) timestamps)
-        m (some->> (select-keys timestamps [:scheduled :deadline])
-                   (map (fn [[k v]]
-                          (let [{:keys [date repetition]} v
-                                {:keys [year month day]} date
-                                day (js/parseInt (str year (util/zero-pad month) (util/zero-pad day)))]
-                            (cond->
-                             (case k
-                               :scheduled
-                               {:scheduled day}
-                               :deadline
-                               {:deadline day})
-                              repetition
-                              (assoc :repeated? true))))))]
-    (apply merge m)))
-
-(defn convert-page-if-journal
-  "Convert journal file name to user' custom date format"
-  [original-page-name]
-  (when original-page-name
-    (let [page-name (util/page-name-sanity-lc original-page-name)
-          day (date/journal-title->int page-name)]
-     (if day
-       (let [original-page-name (date/int->journal-title day)]
-         [original-page-name (util/page-name-sanity-lc original-page-name) day])
-       [original-page-name page-name day]))))
+(defn extract-blocks
+  "Wrapper around logseq.graph-parser.block/extract-blocks that adds in system state"
+  [blocks content with-id? format]
+  (gp-block/extract-blocks blocks content with-id? format
+                           {:user-config (state/get-config)
+                            :block-pattern (config/get-block-pattern format)
+                            :supported-formats (config/supported-formats)
+                            :db (db/get-db (state/get-current-repo))
+                            :date-formatter (state/get-date-formatter)}))
 
 (defn page-name->map
-  "Create a page's map structure given a original page name (string).
-   map as input is supported for legacy compatibility.
-   with-timestamp?: assign timestampes to the map structure.
-    Useful when creating new pages from references or namespaces,
-    as there's no chance to introduce timestamps via editing in page"
+  "Wrapper around logseq.graph-parser.block/page-name->map that adds in db"
   ([original-page-name with-id?]
    (page-name->map original-page-name with-id? true))
   ([original-page-name with-id? with-timestamp?]
-   (cond
-     (and original-page-name (string? original-page-name))
-     (let [original-page-name (util/remove-boundary-slashes original-page-name)
-           [original-page-name page-name journal-day] (convert-page-if-journal original-page-name)
-           namespace? (and (not (boolean (text/get-nested-page-name original-page-name)))
-                           (text/namespace-page? original-page-name))
-           page-entity (db/entity [:block/name page-name])
-           original-page-name (or (:block/original-name page-entity) original-page-name)]
-       (merge
-        {:block/name page-name
-         :block/original-name original-page-name}
-        (when with-id?
-          (if page-entity
-            {:block/uuid (:block/uuid page-entity)}
-            {:block/uuid (db/new-block-id)}))
-        (when namespace?
-          (let [namespace (first (gp-util/split-last "/" original-page-name))]
-            (when-not (string/blank? namespace)
-              {:block/namespace {:block/name (util/page-name-sanity-lc namespace)}})))
-        (when (and with-timestamp? (not page-entity)) ;; Only assign timestamp on creating new entity
-          (let [current-ms (util/time-ms)]
-            {:block/created-at current-ms
-             :block/updated-at current-ms}))
-        (if journal-day
-          {:block/journal? true
-           :block/journal-day journal-day}
-          {:block/journal? false})))
-
-     (and (map? original-page-name) (:block/uuid original-page-name))
-     original-page-name
-
-     (and (map? original-page-name) with-id?)
-     (assoc original-page-name :block/uuid (db/new-block-id))
-
-     :else
-     nil)))
-
-(defn with-page-refs
-  [{:keys [title body tags refs marker priority] :as block} with-id?]
-  (let [refs (->> (concat tags refs [marker priority])
-                  (remove string/blank?)
-                  (distinct))
-        refs (atom refs)]
-    (walk/prewalk
-     (fn [form]
-       ;; skip custom queries
-       (when-not (and (vector? form)
-                      (= (first form) "Custom")
-                      (= (second form) "query"))
-         (when-let [page (get-page-reference form)]
-           (swap! refs conj page))
-         (when-let [tag (get-tag form)]
-           (let [tag (text/page-ref-un-brackets! tag)]
-             (when (gp-util/tag-valid? tag)
-               (swap! refs conj tag))))
-         form))
-     (concat title body))
-    (let [refs (remove string/blank? @refs)
-          children-pages (->> (mapcat (fn [p]
-                                        (let [p (if (map? p)
-                                                  (:block/original-name p)
-                                                  p)]
-                                          (when (string? p)
-                                            (let [p (or (text/get-nested-page-name p) p)]
-                                              (when (text/namespace-page? p)
-                                                (util/split-namespace-pages p))))))
-                                      refs)
-                              (remove string/blank?)
-                              (distinct))
-          refs (->> (distinct (concat refs children-pages))
-                    (remove nil?))
-          refs (map (fn [ref] (page-name->map ref with-id?)) refs)]
-      (assoc block :refs refs))))
-
-(defn with-block-refs
-  [{:keys [title body] :as block}]
-  (let [ref-blocks (atom nil)]
-    (walk/postwalk
-     (fn [form]
-       (when-let [block (get-block-reference form)]
-         (swap! ref-blocks conj block))
-       form)
-     (concat title body))
-    (let [ref-blocks (->> @ref-blocks
-                          (filter gp-util/uuid-string?))
-          ref-blocks (map
-                       (fn [id]
-                         [:block/uuid (medley/uuid id)])
-                       ref-blocks)
-          refs (distinct (concat (:refs block) ref-blocks))]
-      (assoc block :refs refs))))
-
-(defn- block-keywordize
-  [block]
-  (medley/map-keys
-   (fn [k]
-     (if (namespace k)
-       k
-       (keyword "block" k)))
-   block))
-
-(defn- sanity-blocks-data
-  [blocks]
-  (map (fn [block]
-         (if (map? block)
-           (block-keywordize (gp-util/remove-nils block))
-           block))
-       blocks))
-
-(defn with-path-refs
-  [blocks]
-  (loop [blocks blocks
-         acc []
-         parents []]
-    (if (empty? blocks)
-      acc
-      (let [block (first blocks)
-            cur-level (:block/level block)
-            level-diff (- cur-level
-                          (get (last parents) :block/level 0))
-            [path-refs parents]
-            (cond
-              (zero? level-diff)            ; sibling
-              (let [path-refs (mapcat :block/refs (drop-last parents))
-                    parents (conj (vec (butlast parents)) block)]
-                [path-refs parents])
-
-              (> level-diff 0)              ; child
-              (let [path-refs (mapcat :block/refs parents)]
-                [path-refs (conj parents block)])
-
-              (< level-diff 0)              ; new parent
-              (let [parents (vec (take-while (fn [p] (< (:block/level p) cur-level)) parents))
-                    path-refs (mapcat :block/refs parents)]
-                [path-refs (conj parents block)]))
-            path-ref-pages (->> path-refs
-                                (concat (:block/refs block))
-                                (map (fn [ref]
-                                       (cond
-                                         (map? ref)
-                                         (:block/name ref)
-
-                                         :else
-                                         ref)))
-                                (remove string/blank?)
-                                (map (fn [ref]
-                                       (if (string? ref)
-                                         {:block/name (util/page-name-sanity-lc ref)}
-                                         ref)))
-                                (remove vector?)
-                                (remove nil?)
-                                (distinct))]
-        (recur (rest blocks)
-               (conj acc (assoc block :block/path-refs path-ref-pages))
-               parents)))))
-
-(defn block-tags->pages
-  [{:keys [tags] :as block}]
-  (if (seq tags)
-    (assoc block :tags (map (fn [tag]
-                              (let [tag (text/page-ref-un-brackets! tag)]
-                                [:block/name (util/page-name-sanity-lc tag)])) tags))
-    block))
-
-(defn- get-block-content
-  [utf8-content block format meta]
-  (let [content (if-let [end-pos (:end_pos meta)]
-                  (utf8/substring utf8-content
-                                  (:start_pos meta)
-                                  end-pos)
-                  (utf8/substring utf8-content
-                                  (:start_pos meta)))
-        content (when content
-                  (let [content (text/remove-level-spaces content format (config/get-block-pattern format))]
-                    (if (or (:pre-block? block)
-                            (= (:format block) :org))
-                      content
-                      (gp-mldoc/remove-indentation-spaces content (inc (:level block)) false))))]
-    (if (= format :org)
-      content
-      (property/->new-properties content))))
-
-(defn get-custom-id-or-new-id
-  [properties]
-  (or (when-let [custom-id (or (get-in properties [:properties :custom-id])
-                               (get-in properties [:properties :custom_id])
-                               (get-in properties [:properties :id]))]
-        (let [custom-id (and (string? custom-id) (string/trim custom-id))]
-          (when (and custom-id (gp-util/uuid-string? custom-id))
-            (uuid custom-id))))
-      (db/new-block-id)))
-
-(defn get-page-refs-from-properties
-  [properties]
-  (let [page-refs (mapcat (fn [v] (cond
-                                   (coll? v)
-                                   v
-
-                                   (text/page-ref? v)
-                                   [(text/page-ref-un-brackets! v)]
-
-                                   :else
-                                   nil)) (vals properties))
-        page-refs (remove string/blank? page-refs)]
-    (map (fn [page] (page-name->map page true)) page-refs)))
-
-(defn with-page-block-refs
-  [block with-id?]
-  (some-> block
-          (with-page-refs with-id?)
-          with-block-refs
-          block-tags->pages
-          (update :refs (fn [col] (remove nil? col)))))
-
-(defn with-pre-block-if-exists
-  [blocks body pre-block-properties encoded-content]
-  (let [first-block (first blocks)
-        first-block-start-pos (get-in first-block [:block/meta :start_pos])
-
-        ;; Add pre-block
-        blocks (if (or (> first-block-start-pos 0)
-                       (empty? blocks))
-                 (cons
-                  (merge
-                   (let [content (utf8/substring encoded-content 0 first-block-start-pos)
-                         {:keys [properties properties-order]} pre-block-properties
-                         id (get-custom-id-or-new-id {:properties properties})
-                         property-refs (->> (get-page-refs-from-properties properties)
-                                            (map :block/original-name))
-                         block {:uuid id
-                                :content content
-                                :level 1
-                                :properties properties
-                                :properties-order properties-order
-                                :refs property-refs
-                                :pre-block? true
-                                :unordered true
-                                :body body}
-                         block (with-page-block-refs block false)]
-                     (block-keywordize block))
-                   (select-keys first-block [:block/format :block/page]))
-                  blocks)
-                 blocks)]
-    (with-path-refs blocks)))
-
-(defn- construct-block
-  [block properties timestamps body encoded-content format pos-meta with-id?]
-  (let [id (get-custom-id-or-new-id properties)
-        ref-pages-in-properties (->> (:page-refs properties)
-                                     (remove string/blank?))
-        block (second block)
-        unordered? (:unordered block)
-        markdown-heading? (and (:size block) (= :markdown format))
-        block (if markdown-heading?
-                (assoc block
-                       :type :heading
-                       :level (if unordered? (:level block) 1)
-                       :heading-level (or (:size block) 6))
-                block)
-        block (cond->
-                (assoc block
-                       :uuid id
-                       :refs ref-pages-in-properties
-                       :format format
-                       :meta pos-meta)
-                (seq (:properties properties))
-                (assoc :properties (:properties properties))
-
-                (seq (:properties-order properties))
-                (assoc :properties-order (:properties-order properties)))
-        block (if (get-in block [:properties :collapsed])
-                (assoc block :collapsed? true)
-                block)
-        block (assoc block
-                     :content (get-block-content encoded-content block format pos-meta))
-        block (if (seq timestamps)
-                (merge block (timestamps->scheduled-and-deadline timestamps))
-                block)
-        block (assoc block :body body)
-        block (with-page-block-refs block with-id?)
-        {:keys [created-at updated-at]} (:properties properties)
-        block (cond-> block
-                (and created-at (integer? created-at))
-                (assoc :block/created-at created-at)
-
-                (and updated-at (integer? updated-at))
-                (assoc :block/updated-at updated-at))]
-    (dissoc block :title :body :anchor)))
-
-(defn extract-blocks
-  "Extract headings from mldoc ast.
-  Args:
-    `blocks`: mldoc ast.
-    `content`: markdown or org-mode text.
-    `with-id?`: If `with-id?` equals to true, all the referenced pages will have new db ids.
-    `format`: content's format, it could be either :markdown or :org-mode."
-  [blocks content with-id? format]
-  {:pre [(seq blocks) (string? content) (boolean? with-id?) (contains? #{:markdown :org} format)]}
-  (try
-    (let [encoded-content (utf8/encode content)
-          [blocks body pre-block-properties]
-          (loop [headings []
-                 blocks (reverse blocks)
-                 timestamps {}
-                 properties {}
-                 body []]
-            (if (seq blocks)
-              (let [[block pos-meta] (first blocks)
-                    ;; fix start_pos
-                    pos-meta (assoc pos-meta :end_pos
-                                    (if (seq headings)
-                                      (get-in (last headings) [:meta :start_pos])
-                                      nil))]
-                (cond
-                  (paragraph-timestamp-block? block)
-                  (let [timestamps (extract-timestamps block)
-                        timestamps' (merge timestamps timestamps)]
-                    (recur headings (rest blocks) timestamps' properties body))
-
-                  (property/properties-ast? block)
-                  (let [properties (extract-properties format (second block))]
-                    (recur headings (rest blocks) timestamps properties body))
-
-                  (heading-block? block)
-                  (let [block (construct-block block properties timestamps body encoded-content format pos-meta with-id?)]
-                    (recur (conj headings block) (rest blocks) {} {} []))
-
-                  :else
-                  (recur headings (rest blocks) timestamps properties (conj body block))))
-              [(-> (reverse headings)
-                   sanity-blocks-data)
-               body
-               properties]))
-          result (with-pre-block-if-exists blocks body pre-block-properties encoded-content)]
-      (map #(dissoc % :block/meta) result))
-    (catch js/Error e
-      (js/console.error "extract-blocks-failed")
-      (log/error :exception e))))
-
-(defn with-parent-and-left
-  [page-id blocks]
-  (loop [blocks (map (fn [block] (assoc block :block/level-spaces (:block/level block))) blocks)
-         parents [{:page/id page-id     ; db id or a map {:block/name "xxx"}
-                   :block/level 0
-                   :block/level-spaces 0}]
-         result []]
-    (if (empty? blocks)
-      (map #(dissoc % :block/level-spaces) result)
-      (let [[block & others] blocks
-            level-spaces (:block/level-spaces block)
-            {:block/keys [uuid level parent] :as last-parent} (last parents)
-            parent-spaces (:block/level-spaces last-parent)
-            [blocks parents result]
-            (cond
-              (= level-spaces parent-spaces)        ; sibling
-              (let [block (assoc block
-                                 :block/parent parent
-                                 :block/left [:block/uuid uuid]
-                                 :block/level level)
-                    parents' (conj (vec (butlast parents)) block)
-                    result' (conj result block)]
-                [others parents' result'])
-
-              (> level-spaces parent-spaces)         ; child
-              (let [parent (if uuid [:block/uuid uuid] (:page/id last-parent))
-                    block (cond->
-                            (assoc block
-                                  :block/parent parent
-                                  :block/left parent)
-                            ;; fix block levels with wrong order
-                            ;; For example:
-                            ;;   - a
-                            ;; - b
-                            ;; What if the input indentation is two spaces instead of 4 spaces
-                            (>= (- level-spaces parent-spaces) 1)
-                            (assoc :block/level (inc level)))
-                    parents' (conj parents block)
-                    result' (conj result block)]
-                [others parents' result'])
-
-              (< level-spaces parent-spaces)
-              (cond
-                (some #(= (:block/level-spaces %) (:block/level-spaces block)) parents) ; outdent
-                (let [parents' (vec (filter (fn [p] (<= (:block/level-spaces p) level-spaces)) parents))
-                      left (last parents')
-                      blocks (cons (assoc (first blocks)
-                                          :block/level (dec level)
-                                          :block/left [:block/uuid (:block/uuid left)])
-                                   (rest blocks))]
-                  [blocks parents' result])
-
-                :else
-                (let [[f r] (split-with (fn [p] (<= (:block/level-spaces p) level-spaces)) parents)
-                      left (first r)
-                      parent-id (if-let [block-id (:block/uuid (last f))]
-                                  [:block/uuid block-id]
-                                  page-id)
-                      block (cond->
-                              (assoc block
-                                     :block/parent parent-id
-                                     :block/left [:block/uuid (:block/uuid left)]
-                                     :block/level (:block/level left)
-                                     :block/level-spaces (:block/level-spaces left)))
-
-                      parents' (->> (concat f [block]) vec)
-                      result' (conj result block)]
-                  [others parents' result'])))]
-        (recur blocks parents result)))))
+   (gp-block/page-name->map original-page-name with-id? (db/get-db (state/get-current-repo)) with-timestamp? (state/get-date-formatter))))
 
 (defn parse-block
   ([block]
@@ -704,10 +69,10 @@
          result
          (let [ast (->> (format/to-edn content format (gp-mldoc/default-config format))
                         (map first))
-               title (when (heading-block? (first ast))
+               title (when (gp-block/heading-block? (first ast))
                        (:title (second (first ast))))
                body (vec (if title (rest ast) ast))
-               body (drop-while property/properties-ast? body)
+               body (drop-while gp-property/properties-ast? body)
                result (cond->
                         (if (seq body) {:block/body body} {})
                         title

src/main/frontend/format/mldoc.cljs

@@ -1,4 +1,5 @@
 (ns frontend.format.mldoc
+  "Mldoc code needed by app but not graph-parser"
   (:require [clojure.string :as string]
             [frontend.format.protocol :as protocol]
             [frontend.state :as state]

src/main/frontend/fs/watcher_handler.cljs

@@ -4,7 +4,7 @@
             [frontend.db :as db]
             [frontend.db.model :as model]
             [frontend.handler.editor :as editor]
-            [frontend.handler.extract :as extract]
+            [logseq.graph-parser.extract :as extract]
             [frontend.handler.file :as file-handler]
             [frontend.handler.page :as page-handler]
             [frontend.handler.repo :as repo-handler]

src/main/frontend/handler/block.cljs

@@ -5,7 +5,7 @@
             [frontend.db.model :as db-model]
             [frontend.db.react :as react]
             [frontend.state :as state]
-            [frontend.format.block :as block]
+            [logseq.graph-parser.block :as gp-block]
             [frontend.util :as util]))
 
 
@@ -83,7 +83,7 @@
   [block typ]
   (walk-block block
               (fn [x]
-                (and (block/timestamp-block? x)
+                (and (gp-block/timestamp-block? x)
                      (= typ (first (second x)))))
               #(second (second %))))
 

src/main/frontend/handler/editor.cljs

@@ -54,6 +54,7 @@
             [frontend.util.keycode :as keycode]
             [logseq.graph-parser.util :as gp-util]
             [logseq.graph-parser.mldoc :as gp-mldoc]
+            [logseq.graph-parser.block :as gp-block]
             ["path" :as path]))
 
 ;; FIXME: should support multiple images concurrently uploading
@@ -256,7 +257,7 @@
 (defn- another-block-with-same-id-exists?
   [current-id block-id]
   (and (string? block-id)
-       (gp-util/uuid-string? block-id)
+       (util/uuid-string? block-id)
        (not= current-id (cljs.core/uuid block-id))
        (db/entity [:block/uuid (cljs.core/uuid block-id)])))
 
@@ -337,7 +338,7 @@
   (if (and (state/enable-timetracking?)
            (not= (:block/content block) value))
     (let [format (:block/format block)
-          new-marker (last (gp-util/safe-re-find (marker/marker-pattern format) (or value "")))
+          new-marker (last (util/safe-re-find (marker/marker-pattern format) (or value "")))
           new-value (with-marker-time value block format
                       new-marker
                       (:block/marker block))]
@@ -482,10 +483,10 @@
   (let [current-page (state/get-current-page)
         block-id (or
                   (and (:id config)
-                       (gp-util/uuid-string? (:id config))
+                       (util/uuid-string? (:id config))
                        (:id config))
                   (and current-page
-                       (gp-util/uuid-string? current-page)
+                       (util/uuid-string? current-page)
                        current-page))]
     (= uuid (and block-id (medley/uuid block-id)))))
 
@@ -669,7 +670,9 @@
 (defn properties-block
   [properties format page]
   (let [content (property/insert-properties format "" properties)
-        refs (block/get-page-refs-from-properties properties)]
+        refs (gp-block/get-page-refs-from-properties properties
+                                                     (db/get-db (state/get-current-repo))
+                                                     (state/get-date-formatter))]
     {:block/pre-block? true
      :block/uuid (db/new-block-id)
      :block/properties properties
@@ -1144,7 +1147,7 @@
   []
   (when-let [page (get-nearest-page)]
     (let [page-name (string/lower-case page)
-          block? (gp-util/uuid-string? page-name)]
+          block? (util/uuid-string? page-name)]
       (when-let [page (db/get-page page-name)]
         (if block?
           (state/sidebar-add-block!
@@ -1174,7 +1177,7 @@
     (let [page (state/get-current-page)
           block-id (and
                     (string? page)
-                    (gp-util/uuid-string? page)
+                    (util/uuid-string? page)
                     (medley/uuid page))]
       (when block-id
         (let [block-parent (db/get-block-parent block-id)]
@@ -1957,7 +1960,7 @@
   (let [blocks (block-tree->blocks tree-vec format)
         target-block (db/pull target-block-id)
         page-id (:db/id (:block/page target-block))
-        blocks (block/with-parent-and-left page-id blocks)]
+        blocks (gp-block/with-parent-and-left page-id blocks)]
     (paste-blocks
      blocks
      {:target-block target-block
@@ -2037,7 +2040,7 @@
 (defn- last-top-level-child?
   [{:keys [id]} current-node]
   (when id
-    (when-let [entity (if (gp-util/uuid-string? (str id))
+    (when-let [entity (if (util/uuid-string? (str id))
                         (db/entity [:block/uuid (uuid id)])
                         (db/entity [:block/name (util/page-name-sanity-lc id)]))]
       (= (:block/uuid entity) (tree/-get-parent-id current-node)))))
@@ -2854,7 +2857,7 @@
     (let [page-id (:db/id (:block/page editing-block))
           blocks (block/extract-blocks
                   (mldoc/->edn text (gp-mldoc/default-config format)) text true format)
-          blocks' (block/with-parent-and-left page-id blocks)]
+          blocks' (gp-block/with-parent-and-left page-id blocks)]
       (paste-blocks blocks' {}))))
 
 (defn- paste-segmented-text
@@ -2864,7 +2867,7 @@
         (string/join "\n"
                      (mapv (fn [p] (->> (string/trim p)
                                         ((fn [p]
-                                           (if (gp-util/safe-re-find (if (= format :org)
+                                           (if (util/safe-re-find (if (= format :org)
                                                                     #"\s*\*+\s+"
                                                                     #"\s*-\s+") p)
                                              p
@@ -2929,9 +2932,9 @@
       ;; from external
       (let [format (or (db/get-page-format (state/get-current-page)) :markdown)]
         (match [format
-                (nil? (gp-util/safe-re-find #"(?m)^\s*(?:[-+*]|#+)\s+" text))
-                (nil? (gp-util/safe-re-find #"(?m)^\s*\*+\s+" text))
-                (nil? (gp-util/safe-re-find #"(?:\r?\n){2,}" text))]
+                (nil? (util/safe-re-find #"(?m)^\s*(?:[-+*]|#+)\s+" text))
+                (nil? (util/safe-re-find #"(?m)^\s*\*+\s+" text))
+                (nil? (util/safe-re-find #"(?:\r?\n){2,}" text))]
           [:markdown false _ _]
           (paste-text-parseable format text)
 
@@ -3216,7 +3219,7 @@
     :or {collapse? false expanded? false incremental? true root-block nil}}]
   (when-let [page (or (state/get-current-page)
                       (date/today))]
-    (let [block? (gp-util/uuid-string? page)
+    (let [block? (util/uuid-string? page)
           block-id (or root-block (and block? (uuid page)))
           blocks (if block-id
                    (db/get-block-and-children (state/get-current-repo) block-id)

src/main/frontend/handler/export.cljs

@@ -17,6 +17,7 @@
             [frontend.util :as util]
             [frontend.format.mldoc :as mldoc]
             [logseq.graph-parser.mldoc :as gp-mldoc]
+            [logseq.graph-parser.util :as gp-util]
             [goog.dom :as gdom]
             [promesa.core :as p])
   (:import [goog.string StringBuffer]))
@@ -379,7 +380,7 @@
                                               [?e2 :block/file ?e]
                                               [?e2 :block/name ?n]
                                               [?e2 :block/original-name ?n2]] db path)
-                                :format (f/get-format path)})))))
+                                :format (gp-util/get-format path)})))))
 
 
 (defn export-repo-as-markdown!

src/main/frontend/handler/external.cljs

@@ -9,6 +9,8 @@
             [frontend.db :as db]
             [frontend.format.mldoc :as mldoc]
             [frontend.format.block :as block]
+            [logseq.graph-parser.util :as gp-util]
+            [logseq.graph-parser.date-time-util :as date-time-util]
             [frontend.handler.page :as page]
             [frontend.handler.editor :as editor]
             [frontend.util :as util]))
@@ -26,7 +28,7 @@
                                       (when journal?
                                         (date/journal-title->default title))
                                       (string/replace title "/" "-"))
-                               title (-> (util/page-name-sanity title)
+                               title (-> (gp-util/page-name-sanity title)
                                          (string/replace "\n" " "))
                                path (str (if journal?
                                            (config/get-journals-directory)
@@ -49,7 +51,7 @@
                              (map
                               (fn [title]
                                 (let [day (date/journal-title->int title)
-                                      page-name (util/page-name-sanity-lc (date/int->journal-title day))]
+                                      page-name (util/page-name-sanity-lc (date-time-util/int->journal-title day (state/get-date-formatter)))]
                                   {:block/name page-name
                                    :block/journal? true
                                    :block/journal-day day}))

src/main/frontend/handler/file.cljs

@@ -8,15 +8,15 @@
             [clojure.core.async :as async]
             [frontend.config :as config]
             [frontend.db :as db]
-            [frontend.format :as format]
             [frontend.fs :as fs]
             [frontend.fs.nfs :as nfs]
             [frontend.handler.common :as common-handler]
-            [frontend.handler.extract :as extract-handler]
+            [logseq.graph-parser.extract :as extract]
             [frontend.handler.ui :as ui-handler]
             [frontend.state :as state]
             [frontend.util :as util]
             [logseq.graph-parser.util :as gp-util]
+            [logseq.graph-parser.config :as gp-config]
             [lambdaisland.glogi :as log]
             [promesa.core :as p]
             [frontend.mobile.util :as mobile]
@@ -43,7 +43,7 @@
   [files formats]
   (filter
    (fn [file]
-     (let [format (format/get-format file)]
+     (let [format (gp-util/get-format file)]
        (contains? formats format)))
    files))
 
@@ -119,10 +119,19 @@
          file (gp-util/path-normalize file)
          new? (nil? (db/entity [:file/path file]))]
      (db/set-file-content! repo-url file content)
-     (let [format (format/get-format file)
+     (let [format (gp-util/get-format file)
            file-content [{:file/path file}]
-           tx (if (contains? config/mldoc-support-formats format)
-                (let [[pages blocks] (extract-handler/extract-blocks-pages repo-url file content)
+           tx (if (contains? gp-config/mldoc-support-formats format)
+                (let [[pages blocks]
+                      (extract/extract-blocks-pages
+                       file
+                       content
+                       {:user-config (state/get-config)
+                        :date-formatter (state/get-date-formatter)
+                        :page-name-order (state/page-name-order)
+                        :block-pattern (config/get-block-pattern format)
+                        :supported-formats (config/supported-formats)
+                        :db (db/get-db (state/get-current-repo))})
                       first-page (first pages)
                       delete-blocks (->
                                      (concat
@@ -144,7 +153,7 @@
                                           (seq))
                       ;; To prevent "unique constraint" on datascript
                       block-ids (set/union (set block-ids) (set block-refs-ids))
-                      pages (extract-handler/with-ref-pages pages blocks)
+                      pages (extract/with-ref-pages pages blocks)
                       pages-index (map #(select-keys % [:block/name]) pages)]
                   ;; does order matter?
                   (concat file-content pages-index delete-blocks pages block-ids blocks))

src/main/frontend/handler/graph.cljs

@@ -4,8 +4,7 @@
             [frontend.db :as db]
             [frontend.db.default :as default-db]
             [frontend.state :as state]
-            [frontend.util :as util]
-            [logseq.graph-parser.util :as gp-util]))
+            [frontend.util :as util]))
 
 (defn- build-links
   [links]
@@ -46,7 +45,7 @@
                   ;; slow
 (defn- uuid-or-asset?
   [id]
-  (or (gp-util/uuid-string? id)
+  (or (util/uuid-string? id)
       (string/starts-with? id "../assets/")
       (= id "..")
       (string/starts-with? id "assets/")

src/main/frontend/handler/page.cljs

@@ -11,7 +11,6 @@
             [frontend.db.model :as model]
             [frontend.db.utils :as db-utils]
             [frontend.db.conn :as conn]
-            [frontend.format.block :as block]
             [frontend.fs :as fs]
             [frontend.handler.common :as common-handler]
             [frontend.handler.editor :as editor-handler]
@@ -35,6 +34,8 @@
             [frontend.mobile.util :as mobile-util]
             [logseq.graph-parser.util :as gp-util]
             [logseq.graph-parser.config :as gp-config]
+            [logseq.graph-parser.block :as gp-block]
+            [frontend.format.block :as block]
             [goog.functions :refer [debounce]]))
 
 (defn- get-directory
@@ -47,7 +48,7 @@
   [journal? title]
   (when-let [s (if journal?
                  (date/journal-title->default title)
-                 (util/page-name-sanity (string/lower-case title)))]
+                 (gp-util/page-name-sanity (string/lower-case title)))]
     ;; Win10 file path has a length limit of 260 chars
     (gp-util/safe-subs s 0 200)))
 
@@ -72,7 +73,9 @@
    (let [p (common-handler/get-page-default-properties title)
          ps (merge p properties)
          content (page-property/insert-properties format "" ps)
-         refs (block/get-page-refs-from-properties properties)]
+         refs (gp-block/get-page-refs-from-properties properties
+                                                      (db/get-db (state/get-current-repo))
+                                                      (state/get-date-formatter))]
      {:block/uuid (db/new-block-id)
       :block/properties ps
       :block/properties-order (keys ps)
@@ -118,12 +121,12 @@
                   properties          nil
                   split-namespace?    true}}]
    (let [title (string/trim title)
-         title (util/remove-boundary-slashes title)
+         title (gp-util/remove-boundary-slashes title)
          page-name (util/page-name-sanity-lc title)
          repo (state/get-current-repo)]
      (when (db/page-empty? repo page-name)
        (let [pages    (if split-namespace?
-                        (util/split-namespace-pages title)
+                        (gp-util/split-namespace-pages title)
                         [title])
              format   (or format (state/get-preferred-format))
              pages    (map (fn [page]
@@ -170,7 +173,7 @@
 (defn- compute-new-file-path
   [old-path new-name]
   (let [result (string/split old-path "/")
-        file-name (util/page-name-sanity new-name true)
+        file-name (gp-util/page-name-sanity new-name true)
         ext (last (string/split (last result) "."))
         new-file (str file-name "." ext)
         parts (concat (butlast result) [new-file])]
@@ -398,7 +401,7 @@
 
         ;; If page name changed after sanitization
         (when (or (util/create-title-property? new-page-name)
-                  (not= (util/page-name-sanity new-name false) new-name))
+                  (not= (gp-util/page-name-sanity new-name false) new-name))
           (page-property/add-property! new-page-name :title new-name))
 
         (when (and file (not journal?))
@@ -634,7 +637,7 @@
   (->> (db/get-all-pages repo)
        (remove (fn [p]
                  (let [name (:block/name p)]
-                   (or (gp-util/uuid-string? name)
+                   (or (util/uuid-string? name)
                        (gp-config/draw? name)
                        (db/built-in-pages-names (string/upper-case name))))))
        (common-handler/fix-pages-timestamps)))
@@ -688,7 +691,7 @@
               chosen (if (string/starts-with? chosen "New page: ") ;; FIXME: What if a page named "New page: XXX"?
                        (subs chosen 10)
                        chosen)
-              chosen (if (and (gp-util/safe-re-find #"\s+" chosen) (not wrapped?))
+              chosen (if (and (util/safe-re-find #"\s+" chosen) (not wrapped?))
                        (util/format "[[%s]]" chosen)
                        chosen)
               q (if @editor-handler/*selected-text "" q)

src/main/frontend/handler/repo.cljs

@@ -5,7 +5,6 @@
             [frontend.context.i18n :refer [t]]
             [frontend.date :as date]
             [frontend.db :as db]
-            [frontend.format :as format]
             [frontend.fs :as fs]
             [frontend.fs.nfs :as nfs]
             [frontend.handler.common :as common-handler]
@@ -23,6 +22,7 @@
             [shadow.resource :as rc]
             [frontend.db.persist :as db-persist]
             [logseq.graph-parser.util :as gp-util]
+            [logseq.graph-parser.config :as gp-config]
             [electron.ipc :as ipc]
             [clojure.set :as set]
             [clojure.core.async :as async]
@@ -217,8 +217,8 @@
   [repo-url files delete-files delete-blocks file-paths db-encrypted? re-render? re-render-opts opts]
   (let [support-files (filter
                        (fn [file]
-                         (let [format (format/get-format (:file/path file))]
-                           (contains? (set/union #{:edn :css} config/mldoc-support-formats) format)))
+                         (let [format (gp-util/get-format (:file/path file))]
+                           (contains? (set/union #{:edn :css} gp-config/mldoc-support-formats) format)))
                        files)
         support-files (sort-by :file/path support-files)
         {journals true non-journals false} (group-by (fn [file] (string/includes? (:file/path file) "journals/")) support-files)

src/main/frontend/handler/route.cljs

@@ -9,7 +9,6 @@
             [frontend.state :as state]
             [logseq.graph-parser.text :as text]
             [frontend.util :as util]
-            [logseq.graph-parser.util :as gp-util]
             [medley.core :as medley]
             [reitit.frontend.easy :as rfe]))
 
@@ -79,7 +78,7 @@
     "Create a new page"
     :page
     (let [name (:name path-params)
-          block? (gp-util/uuid-string? name)]
+          block? (util/uuid-string? name)]
       (if block?
         (if-let [block (db/entity [:block/uuid (medley/uuid name)])]
           (let [content (text/remove-level-spaces (:block/content block)

src/main/frontend/handler/ui.cljs

@@ -14,7 +14,6 @@
             [clojure.string :as string]
             [rum.core :as rum]
             [frontend.mobile.util :as mobile]
-            [logseq.graph-parser.util :as gp-util]
             [electron.ipc :as ipc]))
 
 (defn- get-css-var-value
@@ -112,7 +111,7 @@
   (let [id (and
             (> (count fragment) 36)
             (subs fragment (- (count fragment) 36)))]
-    (if (and id (gp-util/uuid-string? id))
+    (if (and id (util/uuid-string? id))
       (let [elements (array-seq (js/document.getElementsByClassName id))]
         (when (first elements)
           (util/scroll-to-element (gobj/get (first elements) "id")))

src/main/frontend/mobile/intent.cljs

@@ -11,6 +11,7 @@
             [frontend.util :as util]
             [frontend.config :as config]
             [logseq.graph-parser.mldoc :as gp-mldoc]
+            [logseq.graph-parser.config :as gp-config]
             ["path" :as path]
             [frontend.mobile.util :as mobile-util]
             [frontend.handler.notification :as notification]
@@ -109,7 +110,7 @@
           format (db/get-page-format page)
           application-type (last (string/split type "/"))
           content (cond
-                    (config/mldoc-support? application-type)
+                    (gp-config/mldoc-support? application-type)
                     (embed-text-file url title)
 
                     (contains? (set/union (config/doc-formats) config/media-formats)

src/main/frontend/modules/outliner/tree.cljs

@@ -1,6 +1,6 @@
 (ns frontend.modules.outliner.tree
   (:require [frontend.db :as db]
-            [logseq.graph-parser.util :as gp-util]
+            [frontend.util :as util]
             [clojure.string :as string]
             [frontend.state :as state]))
 
@@ -45,7 +45,7 @@
 (defn- get-root-and-page
   [repo root-id]
   (if (string? root-id)
-    (if (gp-util/uuid-string? root-id)
+    (if (util/uuid-string? root-id)
       [false (db/entity repo [:block/uuid (uuid root-id)])]
       [true (db/entity repo [:block/name (string/lower-case root-id)])])
     [false root-id]))

src/main/frontend/search.cljs

@@ -1,7 +1,7 @@
 (ns frontend.search
   (:require [cljs-bean.core :as bean]
             [clojure.string :as string]
-            [frontend.config :as config]
+            [logseq.graph-parser.config :as gp-config]
             [frontend.db :as db]
             [frontend.regex :as regex]
             [frontend.search.browser :as search-browser]
@@ -143,7 +143,7 @@
   ([q limit]
    (let [q (clean-str q)]
      (when-not (string/blank? q)
-       (let [mldoc-exts (set (map name config/mldoc-support-formats))
+       (let [mldoc-exts (set (map name gp-config/mldoc-support-formats))
              files (->> (db/get-files (state/get-current-repo))
                         (map first)
                         (remove (fn [file]
@@ -237,4 +237,4 @@
 (defn cache-stale?
   [repo]
   (when-let [engine (get-engine repo)]
-    (protocol/cache-stale? engine repo)))
+    (protocol/cache-stale? engine repo)))

src/main/frontend/security.cljs

@@ -1,6 +1,6 @@
 (ns frontend.security
   (:require [clojure.walk :as walk]
-            [logseq.graph-parser.util :as gp-util]))
+            [frontend.util :as util]))
 
 ;; To prevent from cross-site scripting vulnerability, we should add security checks for both hiccup and raw html.
 ;; Hiccup: [:a {:href "javascript:alert('hei')"} "click me"]
@@ -12,7 +12,7 @@
    (= :a (first f))
    (:href (second f))
    (:href (second f))
-   (gp-util/safe-re-find #"(?i)javascript" (:href (second f)))))
+   (util/safe-re-find #"(?i)javascript" (:href (second f)))))
 
 (defn remove-javascript-links-in-href
   [hiccup]

src/main/frontend/state.cljs

@@ -13,7 +13,6 @@
             [goog.object :as gobj]
             [promesa.core :as p]
             [rum.core :as rum]
-            [logseq.graph-parser.util :as gp-util]
             [frontend.mobile.util :as mobile-util]))
 
 (defonce ^:large-vars/data-var state
@@ -452,7 +451,7 @@
     (or
       (when-let [workflow (:preferred-workflow (get-config))]
         (let [workflow (name workflow)]
-          (if (gp-util/safe-re-find #"now|NOW" workflow)
+          (if (util/safe-re-find #"now|NOW" workflow)
             :now
             :todo)))
       (get-in @state [:me :preferred_workflow] :now))))

src/main/frontend/util.cljc

@@ -39,6 +39,23 @@
 #?(:cljs (defn app-scroll-container-node []
            (gdom/getElement "main-content-container")))
 
+#?(:cljs
+   (defn safe-re-find
+     [pattern s]
+     (when-not (string? s)
+       ;; TODO: sentry
+       (js/console.trace))
+     (when (string? s)
+       (re-find pattern s))))
+
+#?(:cljs
+  (do
+    (def uuid-pattern "[0-9a-f]{8}-[0-9a-f]{4}-[0-5][0-9a-f]{3}-[089ab][0-9a-f]{3}-[0-9a-f]{12}")
+    (defonce exactly-uuid-pattern (re-pattern (str "(?i)^" uuid-pattern "$")))
+    (defn uuid-string?
+      [s]
+      (safe-re-find exactly-uuid-pattern s))))
+
 #?(:cljs
    (defn ios?
      []
@@ -55,7 +72,7 @@
    (defn mobile?
      []
      (when-not node-test?
-       (gp-util/safe-re-find #"Mobi" js/navigator.userAgent))))
+       (safe-re-find #"Mobi" js/navigator.userAgent))))
 
 #?(:cljs
    (defn electron?
@@ -349,7 +366,7 @@
 #?(:cljs
    (defn scroll-to-element
      [elem-id]
-     (when-not (gp-util/safe-re-find #"^/\d+$" elem-id)
+     (when-not (safe-re-find #"^/\d+$" elem-id)
        (when elem-id
          (when-let [elem (gdom/getElement elem-id)]
            (.scroll (app-scroll-container-node)
@@ -713,11 +730,6 @@
 (defn drop-nth [n coll]
   (keep-indexed #(when (not= %1 n) %2) coll))
 
-(defn capitalize-all [s]
-  (some->> (string/split s #" ")
-           (map string/capitalize)
-           (string/join " ")))
-
 #?(:cljs
    (defn react
      [ref]
@@ -834,8 +846,8 @@
      []
      (let [user-agent js/navigator.userAgent
            vendor js/navigator.vendor]
-       (and (gp-util/safe-re-find #"Chrome" user-agent)
-            (gp-util/safe-re-find #"Google Inc" vendor)))))
+       (and (safe-re-find #"Chrome" user-agent)
+            (safe-re-find #"Google Inc" vendor)))))
 
 #?(:cljs
    (defn indexeddb-check?
@@ -876,7 +888,7 @@
      [block-id]
      (when block-id
        (let [block-id (str block-id)]
-         (when (gp-util/uuid-string? block-id)
+         (when (uuid-string? block-id)
            (first (array-seq (js/document.getElementsByClassName block-id))))))))
 
 #?(:cljs
@@ -895,7 +907,7 @@
    (do
      (defn include-windows-reserved-chars?
       [s]
-       (gp-util/safe-re-find windows-reserved-chars s))
+       (safe-re-find windows-reserved-chars s))
 
      (defn create-title-property?
        [s]
@@ -907,44 +919,18 @@
                 (string/includes? s "%")
                 (string/includes? s "#"))))))
 
-(defn remove-boundary-slashes
-  [s]
-  (when (string? s)
-    (let [s (if (= \/ (first s))
-              (subs s 1)
-              s)]
-      (if (= \/ (last s))
-        (subs s 0 (dec (count s)))
-        s))))
-
-(defn normalize
-  [s]
-  (.normalize s "NFC"))
-
 #?(:cljs
    (defn search-normalize
      "Normalize string for searching (loose)"
      [s]
      (removeAccents (.normalize (string/lower-case s) "NFKC"))))
 
-(defn page-name-sanity
-  "Sanitize the page-name."
-  ([page-name]
-   (page-name-sanity page-name false))
-  ([page-name replace-slash?]
-   (let [page (some-> page-name
-                      (remove-boundary-slashes)
-                      (normalize))]
-     (if replace-slash?
-       (string/replace page #"/" "%2A")
-       page))))
-
 #?(:cljs
    (defn file-name-sanity
      "Sanitize page-name for file name (strict), for file writing."
      [page-name]
      (some-> page-name
-             page-name-sanity
+             gp-util/page-name-sanity
              ;; for android filesystem compatiblity
              (string/replace #"[\\#|%]+" url-encode)
              ;; Windows reserved path characters
@@ -952,15 +938,16 @@
              (string/replace #"/" url-encode)
              (string/replace "*" "%2A"))))
 
-(defn page-name-sanity-lc
-  "Sanitize the query string for a page name (mandate for :block/name)"
-  [s]
-  (page-name-sanity (string/lower-case s)))
+#?(:cljs
+   (def page-name-sanity-lc
+     "Delegate to gp-util to loosely couple app usages to graph-parser"
+     gp-util/page-name-sanity-lc))
 
-(defn safe-page-name-sanity-lc
-  [s]
-  (if (string? s)
-    (page-name-sanity-lc s) s))
+#?(:cljs
+ (defn safe-page-name-sanity-lc
+   [s]
+   (if (string? s)
+     (page-name-sanity-lc s) s)))
 
 (defn get-page-original-name
   [page]
@@ -1240,17 +1227,6 @@
   [text]
   (string/join (replace regex-char-esc-smap text)))
 
-(defn split-namespace-pages
-  [title]
-  (let [parts (string/split title "/")]
-    (loop [others (rest parts)
-           result [(first parts)]]
-      (if (seq others)
-        (let [prev (last result)]
-          (recur (rest others)
-                 (conj result (str prev "/" (first others)))))
-        result))))
-
 (comment
   (re-matches (re-pattern (regex-escape "$u^8(d)+w.*[dw]d?")) "$u^8(d)+w.*[dw]d?"))
 

src/main/frontend/util/drawer.cljs

@@ -1,9 +1,8 @@
 (ns frontend.util.drawer
   (:require [clojure.string :as string]
             [frontend.util :as util]
-            [logseq.graph-parser.util :as gp-util]
             [logseq.graph-parser.mldoc :as gp-mldoc]
-            [frontend.util.property :as property]
+            [logseq.graph-parser.property :as gp-property]
             [frontend.format.mldoc :as mldoc]))
 
 (defn drawer-start
@@ -55,8 +54,8 @@
                         (if has-properties?
                           (cond
                             (= :org format)
-                            (let [prop-start-idx (.indexOf body-without-timestamps property/properties-start)
-                                  prop-end-idx (.indexOf body-without-timestamps property/properties-end)
+                            (let [prop-start-idx (.indexOf body-without-timestamps gp-property/properties-start)
+                                  prop-end-idx (.indexOf body-without-timestamps gp-property/properties-end)
                                   properties (subvec body-without-timestamps prop-start-idx (inc prop-end-idx))
                                   after (subvec body-without-timestamps (inc prop-end-idx))]
                               (string/join "\n" (concat [title] scheduled deadline properties [drawer] after)))
@@ -88,8 +87,8 @@
 
 (defn contains-logbook?
   [content]
-  (and (gp-util/safe-re-find (re-pattern (str "(?i)" logbook-start)) content)
-       (gp-util/safe-re-find (re-pattern (str "(?i)" drawer-end)) content)))
+  (and (util/safe-re-find (re-pattern (str "(?i)" logbook-start)) content)
+       (util/safe-re-find (re-pattern (str "(?i)" drawer-end)) content)))
 
 ;; TODO: DRY
 (defn remove-logbook

src/main/frontend/util/marker.cljs

@@ -1,7 +1,6 @@
 (ns frontend.util.marker
   (:require [clojure.string :as string]
-            [frontend.util :as util]
-            [logseq.graph-parser.util :as gp-util]))
+            [frontend.util :as util]))
 
 (defn marker-pattern [format]
   (re-pattern
@@ -21,7 +20,7 @@
         (if-let [matches (seq (util/re-pos new-line-re-pattern content))]
           (let [[start-pos content] (last matches)]
             (+ start-pos (count content)))
-          (count (gp-util/safe-re-find re-pattern content)))
+          (count (util/safe-re-find re-pattern content)))
         new-content
         (str (subs content 0 pos)
              (string/replace-first (subs content pos)
@@ -60,6 +59,6 @@
   (let [content    (string/triml content)
         new-marker (or new-marker
                        (cycle-marker-state (or marker
-                                               (last (gp-util/safe-re-find (marker-pattern format) content))) ; Returns the last matching group (last vec)
+                                               (last (util/safe-re-find (marker-pattern format) content))) ; Returns the last matching group (last vec)
                                            preferred-workflow))]
     [(add-or-update-marker content format new-marker) new-marker]))

src/main/frontend/util/priority.cljs

@@ -1,13 +1,12 @@
 (ns frontend.util.priority
   (:require [clojure.string :as string]
             [frontend.util :as util]
-            [logseq.graph-parser.util :as gp-util]
             [frontend.util.marker :as marker]))
 
 (defn cycle-priority-state
   [content]
   (let [priority-reg #"\[#([ABC]{1})\]\s{1}"
-        priority (last (gp-util/safe-re-find priority-reg content))
+        priority (last (util/safe-re-find priority-reg content))
         next-priority (case priority
                         "A" "B"
 
@@ -29,7 +28,7 @@
         (if-let [matches (seq (util/re-pos new-line-re-pattern content))]
           (let [[start-pos content] (last matches)]
             (+ start-pos (count content)))
-          (count (gp-util/safe-re-find re-pattern content)))
+          (count (util/safe-re-find re-pattern content)))
         skip-marker-pos
         (if-let [matches (seq (util/re-pos marker/bare-marker-pattern (subs content skip-hash-pos)))]
           (let [[start-pos content] (last matches)]

src/main/frontend/util/property.cljs

@@ -1,4 +1,5 @@
 (ns frontend.util.property
+  "Property fns needed by the rest of the app and not graph-parser"
   (:require [clojure.string :as string]
             [frontend.util :as util]
             [clojure.set :as set]
@@ -6,15 +7,11 @@
             [medley.core :as medley]
             [logseq.graph-parser.util :as gp-util]
             [logseq.graph-parser.mldoc :as gp-mldoc]
+            [logseq.graph-parser.property :as gp-property :refer [properties-start properties-end]]
             [frontend.format.mldoc :as mldoc]
             [logseq.graph-parser.text :as text]
             [frontend.util.cursor :as cursor]))
 
-(defonce properties-start ":PROPERTIES:")
-(defonce properties-end ":END:")
-(defonce properties-end-pattern
-  (re-pattern (util/format "%s[\t\r ]*\n|(%s\\s*$)" properties-end properties-end)))
-
 (def built-in-extended-properties (atom #{}))
 (defn register-built-in-properties
   [props]
@@ -36,15 +33,9 @@
              built-in-properties-set (built-in-properties)]
          (every? built-in-properties-set ks))))
 
-(defn contains-properties?
-  [content]
-  (when content
-    (and (string/includes? content properties-start)
-         (gp-util/safe-re-find properties-end-pattern content))))
-
 (defn remove-empty-properties
   [content]
-  (if (contains-properties? content)
+  (if (gp-property/contains-properties? content)
     (string/replace content
                     (re-pattern ":PROPERTIES:\n+:END:\n*")
                     "")
@@ -54,28 +45,28 @@
   [line]
   (boolean
    (and (string? line)
-        (gp-util/safe-re-find #"^\s?[^ ]+:: " line))))
+        (util/safe-re-find #"^\s?[^ ]+:: " line))))
 
 (defn front-matter-property?
   [line]
   (boolean
    (and (string? line)
-        (gp-util/safe-re-find #"^\s*[^ ]+: " line))))
+        (util/safe-re-find #"^\s*[^ ]+: " line))))
 
 (defn get-property-key
   [line format]
   (and (string? line)
        (when-let [key (last
                        (if (= format :org)
-                         (gp-util/safe-re-find #"^\s*:([^: ]+): " line)
-                         (gp-util/safe-re-find #"^\s*([^ ]+):: " line)))]
+                         (util/safe-re-find #"^\s*:([^: ]+): " line)
+                         (util/safe-re-find #"^\s*([^ ]+):: " line)))]
          (keyword key))))
 
 (defn org-property?
   [line]
   (boolean
    (and (string? line)
-        (gp-util/safe-re-find #"^\s*:[^: ]+: " line)
+        (util/safe-re-find #"^\s*:[^: ]+: " line)
         (when-let [key (get-property-key line :org)]
           (not (contains? #{:PROPERTIES :END} key))))))
 
@@ -114,7 +105,7 @@
 (defn get-property-keys
   [format content]
   (cond
-    (contains-properties? content)
+    (gp-property/contains-properties? content)
     (get-org-property-keys content)
 
     (= :markdown format)
@@ -134,7 +125,7 @@
 (defn remove-properties
   [format content]
   (cond
-    (contains-properties? content)
+    (gp-property/contains-properties? content)
     (let [lines (string/split-lines content)
           [title-lines properties&body] (split-with #(-> (string/triml %)
                                                          string/upper-case
@@ -348,7 +339,7 @@
      (let [format (or format :markdown)
            key (string/lower-case (name key))
            remove-f (if first? util/remove-first remove)]
-       (if (and (= format :org) (not (contains-properties? content)))
+       (if (and (= format :org) (not (gp-property/contains-properties? content)))
          content
          (let [lines (->> (string/split-lines content)
                           (remove-f (fn [line]
@@ -371,31 +362,6 @@
       (string/replace-first content (re-pattern ":PROPERTIES:\n:END:\n*") "")
       content)))
 
-(defn ->new-properties
-  "New syntax: key:: value"
-  [content]
-  (if (contains-properties? content)
-    (let [lines (string/split-lines content)
-          start-idx (.indexOf lines properties-start)
-          end-idx (.indexOf lines properties-end)]
-      (if (and (>= start-idx 0) (> end-idx 0) (> end-idx start-idx))
-        (let [before (subvec lines 0 start-idx)
-              middle (->> (subvec lines (inc start-idx) end-idx)
-                          (map (fn [text]
-                                 (let [[k v] (gp-util/split-first ":" (subs text 1))]
-                                   (if (and k v)
-                                     (let [k (string/replace k "_" "-")
-                                           compare-k (keyword (string/lower-case k))
-                                           k (if (contains? #{:id :custom_id :custom-id} compare-k) "id" k)
-                                           k (if (contains? #{:last-modified-at} compare-k) "updated-at" k)]
-                                       (str k ":: " (string/trim v)))
-                                     text)))))
-              after (subvec lines (inc end-idx))
-              lines (concat before middle after)]
-          (string/join "\n" lines))
-        content))
-    content))
-
 (defn add-page-properties
   [page-format properties-content properties]
   (let [properties (medley/map-keys name properties)
@@ -435,10 +401,3 @@
     (util/format
      (config/properties-wrapper-pattern page-format)
      (string/join "\n" lines))))
-
-(defn properties-ast?
-  [block]
-  (and
-   (vector? block)
-   (contains? #{"Property_Drawer" "Properties"}
-              (first block))))

src/main/frontend/util/thingatpt.cljs

@@ -1,10 +1,10 @@
 (ns frontend.util.thingatpt
   (:require [clojure.string :as string]
             [frontend.state :as state]
-            [frontend.util.property :as property-util]
             [frontend.util.cursor :as cursor]
             [frontend.config :as config]
             [logseq.graph-parser.text :as text]
+            [logseq.graph-parser.property :as gp-property]
             [cljs.reader :as reader]
             [goog.object :as gobj]))
 
@@ -69,8 +69,8 @@
   (when-let [properties
              (case (state/get-preferred-format) ;; TODO fix me to block's format
                :org (thing-at-point
-                     [property-util/properties-start
-                      property-util/properties-end]
+                     [gp-property/properties-start
+                      gp-property/properties-end]
                      input)
                (when-let [line (line-at-point input)]
                  (when (re-matches #"^[^\s.]+:: .*$" (:raw-content line))

src/main/logseq/graph_parser/block.cljc

@@ -0,0 +1,653 @@
+(ns ^:nbb-compatible logseq.graph-parser.block
+  ;; Disable clj linters since we don't support clj
+  #?(:clj {:clj-kondo/config {:linters {:unresolved-namespace {:level :off}
+                                        :unresolved-symbol {:level :off}}}})
+  "Block related code needed for graph-parser"
+  (:require [clojure.string :as string]
+            [clojure.walk :as walk]
+            [datascript.core :as d]
+            [logseq.graph-parser.text :as text]
+            [logseq.graph-parser.utf8 :as utf8]
+            [logseq.graph-parser.property :as gp-property]
+            [logseq.graph-parser.util :as gp-util]
+            [logseq.graph-parser.config :as gp-config]
+            [logseq.graph-parser.mldoc :as gp-mldoc]
+            [logseq.graph-parser.date-time-util :as date-time-util]
+            #?(:org.babashka/nbb [logseq.graph-parser.log :as log]
+               :default [lambdaisland.glogi :as log])))
+
+(defn heading-block?
+  [block]
+  (and
+   (vector? block)
+   (= "Heading" (first block))))
+
+(defn get-tag
+  [block]
+  (when-let [tag-value (and (vector? block)
+                            (= "Tag" (first block))
+                            (second block))]
+    (->> tag-value
+         (map (fn [[elem value]]
+                (case elem
+                  "Plain" value
+                  "Link" (:full_text value)
+                  "Nested_link" (:content value)
+                  "")))
+         (string/join))))
+
+(defn- get-page-reference
+  [block supported-formats]
+  (let [page (cond
+               (and (vector? block) (= "Link" (first block)))
+               (let [typ (first (:url (second block)))
+                     value (second (:url (second block)))]
+                 ;; {:url ["File" "file:../pages/hello_world.org"], :label [["Plain" "hello world"]], :title nil}
+                 (or
+                  (and
+                   (= typ "Page_ref")
+                   (and (string? value)
+                        (not (or (gp-config/local-asset? value)
+                                 (gp-config/draw? value))))
+                   value)
+
+                  (and
+                   (= typ "Search")
+                   (text/page-ref? value)
+                   (text/page-ref-un-brackets! value))
+
+                  (and
+                   (= typ "Search")
+                   (not (contains? #{\# \* \/ \[} (first value)))
+                   (let [ext (some-> (gp-util/get-file-ext value) keyword)]
+                     (when (and (not (string/starts-with? value "http:"))
+                                (not (string/starts-with? value "https:"))
+                                (not (string/starts-with? value "file:"))
+                                (not (gp-config/local-asset? value))
+                                (or (= ext :excalidraw)
+                                    (not (contains? supported-formats ext))))
+                       value)))
+
+                  (and
+                   (= typ "Complex")
+                   (= (:protocol value) "file")
+                   (:link value))
+
+                  (and
+                   (= typ "File")
+                   (second (first (:label (second block)))))))
+
+               (and (vector? block) (= "Nested_link" (first block)))
+               (let [content (:content (last block))]
+                 (subs content 2 (- (count content) 2)))
+
+               (and (vector? block)
+                    (= "Macro" (first block)))
+               (let [{:keys [name arguments]} (second block)
+                     argument (string/join ", " arguments)]
+                   (when (= name "embed")
+                     (text/page-ref-un-brackets! argument)))
+
+               (and (vector? block)
+                    (= "Tag" (first block)))
+               (let [text (get-tag block)]
+                 (text/page-ref-un-brackets! text))
+
+               :else
+               nil)]
+    (text/block-ref-un-brackets! page)))
+
+(defn- get-block-reference
+  [block]
+  (when-let [block-id (cond
+                        (and (vector? block)
+                             (= "Block_reference" (first block)))
+                        (last block)
+
+                        (and (vector? block)
+                             (= "Link" (first block))
+                             (map? (second block))
+                             (= "Block_ref" (first (:url (second block)))))
+                        (second (:url (second block)))
+
+                        (and (vector? block)
+                             (= "Macro" (first block)))
+                        (let [{:keys [name arguments]} (second block)]
+                          (when (and (= name "embed")
+                                     (string? (first arguments))
+                                     (string/starts-with? (first arguments) "((")
+                                     (string/ends-with? (first arguments) "))"))
+                            (subs (first arguments) 2 (- (count (first arguments)) 2))))
+
+                        (and (vector? block)
+                             (= "Link" (first block))
+                             (map? (second block)))
+                        (if (= "id" (:protocol (second (:url (second block)))))
+                          (:link (second (:url (second block))))
+                          (let [id (second (:url (second block)))]
+                            (text/block-ref-un-brackets! id)))
+
+                        :else
+                        nil)]
+    (when (and block-id
+               (gp-util/uuid-string? block-id))
+      block-id)))
+
+(defn- paragraph-block?
+  [block]
+  (and
+   (vector? block)
+   (= "Paragraph" (first block))))
+
+(defn timestamp-block?
+  [block]
+  (and
+   (vector? block)
+   (= "Timestamp" (first block))))
+
+;; TODO: we should move this to mldoc
+(defn extract-properties
+  [format properties user-config]
+  (when (seq properties)
+    (let [properties (seq properties)
+          page-refs (->>
+                     properties
+                     (remove (fn [[k _]]
+                               (contains? #{:background-color :background_color} (keyword k))))
+                     (map last)
+                     (map (fn [v]
+                            (when (and (string? v)
+                                       (not (gp-mldoc/link? format v)))
+                              (let [v (string/trim v)
+                                    result (text/split-page-refs-without-brackets v {:un-brackets? false})]
+                                (if (coll? result)
+                                  (map text/page-ref-un-brackets! result)
+                                  [])))))
+                     (apply concat)
+                     (remove string/blank?))
+          properties (->> properties
+                          (map (fn [[k v]]
+                                 (let [k (-> (string/lower-case (name k))
+                                             (string/replace " " "-")
+                                             (string/replace "_" "-"))
+                                       k (if (contains? #{"custom_id" "custom-id"} k)
+                                           "id"
+                                           k)
+                                       v (if (coll? v)
+                                           (remove string/blank? v)
+                                           (if (string/blank? v)
+                                             nil
+                                             (text/parse-property format k v user-config)))
+                                       k (keyword k)
+                                       v (if (and
+                                              (string? v)
+                                              (contains? #{:alias :aliases :tags} k))
+                                           (set [v])
+                                           v)
+                                       v (if (coll? v) (set v) v)]
+                                   [k v])))
+                          (remove #(nil? (second %))))]
+      {:properties (into {} properties)
+       :properties-order (map first properties)
+       :page-refs page-refs})))
+
+(defn- paragraph-timestamp-block?
+  [block]
+  (and (paragraph-block? block)
+       (or (timestamp-block? (first (second block)))
+           (timestamp-block? (second (second block))))))
+
+(defn- extract-timestamps
+  [block]
+  (some->>
+   (second block)
+   (filter timestamp-block?)
+   (map last)
+   (into {})))
+
+;; {"Deadline" {:date {:year 2020, :month 10, :day 20}, :wday "Tue", :time {:hour 8, :min 0}, :repetition [["DoublePlus"] ["Day"] 1], :active true}}
+(defn timestamps->scheduled-and-deadline
+  [timestamps]
+  (let [timestamps (gp-util/map-keys (comp keyword string/lower-case) timestamps)
+        m (some->> (select-keys timestamps [:scheduled :deadline])
+                   (map (fn [[k v]]
+                          (let [{:keys [date repetition]} v
+                                {:keys [year month day]} date
+                                day (js/parseInt (str year (gp-util/zero-pad month) (gp-util/zero-pad day)))]
+                            (cond->
+                             (case k
+                               :scheduled
+                               {:scheduled day}
+                               :deadline
+                               {:deadline day})
+                              repetition
+                              (assoc :repeated? true))))))]
+    (apply merge m)))
+
+(defn convert-page-if-journal
+  "Convert journal file name to user' custom date format"
+  [original-page-name date-formatter]
+  (when original-page-name
+    (let [page-name (gp-util/page-name-sanity-lc original-page-name)
+          day (date-time-util/journal-title->int page-name (date-time-util/safe-journal-title-formatters date-formatter))]
+     (if day
+       (let [original-page-name (date-time-util/int->journal-title day date-formatter)]
+         [original-page-name (gp-util/page-name-sanity-lc original-page-name) day])
+       [original-page-name page-name day]))))
+
+(defn page-name->map
+  "Create a page's map structure given a original page name (string).
+   map as input is supported for legacy compatibility.
+   with-timestamp?: assign timestampes to the map structure.
+    Useful when creating new pages from references or namespaces,
+    as there's no chance to introduce timestamps via editing in page"
+  [original-page-name with-id? db with-timestamp? date-formatter]
+  (cond
+    (and original-page-name (string? original-page-name))
+    (let [original-page-name (gp-util/remove-boundary-slashes original-page-name)
+          [original-page-name page-name journal-day] (convert-page-if-journal original-page-name date-formatter)
+          namespace? (and (not (boolean (text/get-nested-page-name original-page-name)))
+                          (text/namespace-page? original-page-name))
+          page-entity (some-> db (d/entity [:block/name page-name]))
+          original-page-name (or (:block/original-name page-entity) original-page-name)]
+      (merge
+       {:block/name page-name
+        :block/original-name original-page-name}
+       (when with-id?
+         (if page-entity
+           {:block/uuid (:block/uuid page-entity)}
+           {:block/uuid (d/squuid)}))
+       (when namespace?
+         (let [namespace (first (gp-util/split-last "/" original-page-name))]
+           (when-not (string/blank? namespace)
+             {:block/namespace {:block/name (gp-util/page-name-sanity-lc namespace)}})))
+       (when (and with-timestamp? (not page-entity)) ;; Only assign timestamp on creating new entity
+         (let [current-ms (date-time-util/time-ms)]
+           {:block/created-at current-ms
+            :block/updated-at current-ms}))
+       (if journal-day
+         {:block/journal? true
+          :block/journal-day journal-day}
+         {:block/journal? false})))
+
+    (and (map? original-page-name) (:block/uuid original-page-name))
+    original-page-name
+
+    (and (map? original-page-name) with-id?)
+    (assoc original-page-name :block/uuid (d/squuid))
+
+    :else
+    nil))
+
+(defn- with-page-refs
+  [{:keys [title body tags refs marker priority] :as block} with-id? supported-formats db date-formatter]
+  (let [refs (->> (concat tags refs [marker priority])
+                  (remove string/blank?)
+                  (distinct))
+        refs (atom refs)]
+    (walk/prewalk
+     (fn [form]
+       ;; skip custom queries
+       (when-not (and (vector? form)
+                      (= (first form) "Custom")
+                      (= (second form) "query"))
+         (when-let [page (get-page-reference form supported-formats)]
+           (swap! refs conj page))
+         (when-let [tag (get-tag form)]
+           (let [tag (text/page-ref-un-brackets! tag)]
+             (when (gp-util/tag-valid? tag)
+               (swap! refs conj tag))))
+         form))
+     (concat title body))
+    (let [refs (remove string/blank? @refs)
+          children-pages (->> (mapcat (fn [p]
+                                        (let [p (if (map? p)
+                                                  (:block/original-name p)
+                                                  p)]
+                                          (when (string? p)
+                                            (let [p (or (text/get-nested-page-name p) p)]
+                                              (when (text/namespace-page? p)
+                                                (gp-util/split-namespace-pages p))))))
+                                      refs)
+                              (remove string/blank?)
+                              (distinct))
+          refs (->> (distinct (concat refs children-pages))
+                    (remove nil?))
+          refs (map (fn [ref] (page-name->map ref with-id? db true date-formatter)) refs)]
+      (assoc block :refs refs))))
+
+(defn- with-block-refs
+  [{:keys [title body] :as block}]
+  (let [ref-blocks (atom nil)]
+    (walk/postwalk
+     (fn [form]
+       (when-let [block (get-block-reference form)]
+         (swap! ref-blocks conj block))
+       form)
+     (concat title body))
+    (let [ref-blocks (->> @ref-blocks
+                          (filter gp-util/uuid-string?))
+          ref-blocks (map
+                       (fn [id]
+                         [:block/uuid (uuid id)])
+                       ref-blocks)
+          refs (distinct (concat (:refs block) ref-blocks))]
+      (assoc block :refs refs))))
+
+(defn- block-keywordize
+  [block]
+  (gp-util/map-keys
+   (fn [k]
+     (if (namespace k)
+       k
+       (keyword "block" k)))
+   block))
+
+(defn- sanity-blocks-data
+  [blocks]
+  (map (fn [block]
+         (if (map? block)
+           (block-keywordize (gp-util/remove-nils block))
+           block))
+       blocks))
+
+(defn- with-path-refs
+  [blocks]
+  (loop [blocks blocks
+         acc []
+         parents []]
+    (if (empty? blocks)
+      acc
+      (let [block (first blocks)
+            cur-level (:block/level block)
+            level-diff (- cur-level
+                          (get (last parents) :block/level 0))
+            [path-refs parents]
+            (cond
+              (zero? level-diff)            ; sibling
+              (let [path-refs (mapcat :block/refs (drop-last parents))
+                    parents (conj (vec (butlast parents)) block)]
+                [path-refs parents])
+
+              (> level-diff 0)              ; child
+              (let [path-refs (mapcat :block/refs parents)]
+                [path-refs (conj parents block)])
+
+              (< level-diff 0)              ; new parent
+              (let [parents (vec (take-while (fn [p] (< (:block/level p) cur-level)) parents))
+                    path-refs (mapcat :block/refs parents)]
+                [path-refs (conj parents block)]))
+            path-ref-pages (->> path-refs
+                                (concat (:block/refs block))
+                                (map (fn [ref]
+                                       (cond
+                                         (map? ref)
+                                         (:block/name ref)
+
+                                         :else
+                                         ref)))
+                                (remove string/blank?)
+                                (map (fn [ref]
+                                       (if (string? ref)
+                                         {:block/name (gp-util/page-name-sanity-lc ref)}
+                                         ref)))
+                                (remove vector?)
+                                (remove nil?)
+                                (distinct))]
+        (recur (rest blocks)
+               (conj acc (assoc block :block/path-refs path-ref-pages))
+               parents)))))
+
+(defn- block-tags->pages
+  [{:keys [tags] :as block}]
+  (if (seq tags)
+    (assoc block :tags (map (fn [tag]
+                              (let [tag (text/page-ref-un-brackets! tag)]
+                                [:block/name (gp-util/page-name-sanity-lc tag)])) tags))
+    block))
+
+(defn- get-block-content
+  [utf8-content block format meta block-pattern]
+  (let [content (if-let [end-pos (:end_pos meta)]
+                  (utf8/substring utf8-content
+                                  (:start_pos meta)
+                                  end-pos)
+                  (utf8/substring utf8-content
+                                  (:start_pos meta)))
+        content (when content
+                  (let [content (text/remove-level-spaces content format block-pattern)]
+                    (if (or (:pre-block? block)
+                            (= (:format block) :org))
+                      content
+                      (gp-mldoc/remove-indentation-spaces content (inc (:level block)) false))))]
+    (if (= format :org)
+      content
+      (gp-property/->new-properties content))))
+
+(defn- get-custom-id-or-new-id
+  [properties]
+  (or (when-let [custom-id (or (get-in properties [:properties :custom-id])
+                               (get-in properties [:properties :custom_id])
+                               (get-in properties [:properties :id]))]
+        (let [custom-id (and (string? custom-id) (string/trim custom-id))]
+          (when (and custom-id (gp-util/uuid-string? custom-id))
+            (uuid custom-id))))
+      (d/squuid)))
+
+(defn get-page-refs-from-properties
+  [properties db date-formatter]
+  (let [page-refs (mapcat (fn [v] (cond
+                                   (coll? v)
+                                   v
+
+                                   (text/page-ref? v)
+                                   [(text/page-ref-un-brackets! v)]
+
+                                   :else
+                                   nil)) (vals properties))
+        page-refs (remove string/blank? page-refs)]
+    (map (fn [page] (page-name->map page true db true date-formatter)) page-refs)))
+
+(defn- with-page-block-refs
+  [block with-id? supported-formats db date-formatter]
+  (some-> block
+          (with-page-refs with-id? supported-formats db date-formatter)
+          with-block-refs
+          block-tags->pages
+          (update :refs (fn [col] (remove nil? col)))))
+
+(defn- with-pre-block-if-exists
+  [blocks body pre-block-properties encoded-content {:keys [supported-formats db date-formatter]}]
+  (let [first-block (first blocks)
+        first-block-start-pos (get-in first-block [:block/meta :start_pos])
+
+        ;; Add pre-block
+        blocks (if (or (> first-block-start-pos 0)
+                       (empty? blocks))
+                 (cons
+                  (merge
+                   (let [content (utf8/substring encoded-content 0 first-block-start-pos)
+                         {:keys [properties properties-order]} pre-block-properties
+                         id (get-custom-id-or-new-id {:properties properties})
+                         property-refs (->> (get-page-refs-from-properties properties db date-formatter)
+                                            (map :block/original-name))
+                         block {:uuid id
+                                :content content
+                                :level 1
+                                :properties properties
+                                :properties-order properties-order
+                                :refs property-refs
+                                :pre-block? true
+                                :unordered true
+                                :body body}
+                         block (with-page-block-refs block false supported-formats db date-formatter)]
+                     (block-keywordize block))
+                   (select-keys first-block [:block/format :block/page]))
+                  blocks)
+                 blocks)]
+    (with-path-refs blocks)))
+
+(defn- construct-block
+  [block properties timestamps body encoded-content format pos-meta with-id? {:keys [block-pattern supported-formats db date-formatter]}]
+  (let [id (get-custom-id-or-new-id properties)
+        ref-pages-in-properties (->> (:page-refs properties)
+                                     (remove string/blank?))
+        block (second block)
+        unordered? (:unordered block)
+        markdown-heading? (and (:size block) (= :markdown format))
+        block (if markdown-heading?
+                (assoc block
+                       :type :heading
+                       :level (if unordered? (:level block) 1)
+                       :heading-level (or (:size block) 6))
+                block)
+        block (cond->
+                (assoc block
+                       :uuid id
+                       :refs ref-pages-in-properties
+                       :format format
+                       :meta pos-meta)
+                (seq (:properties properties))
+                (assoc :properties (:properties properties))
+
+                (seq (:properties-order properties))
+                (assoc :properties-order (:properties-order properties)))
+        block (if (get-in block [:properties :collapsed])
+                (assoc block :collapsed? true)
+                block)
+        block (assoc block
+                     :content (get-block-content encoded-content block format pos-meta block-pattern))
+        block (if (seq timestamps)
+                (merge block (timestamps->scheduled-and-deadline timestamps))
+                block)
+        block (assoc block :body body)
+        block (with-page-block-refs block with-id? supported-formats db date-formatter)
+        {:keys [created-at updated-at]} (:properties properties)
+        block (cond-> block
+                (and created-at (integer? created-at))
+                (assoc :block/created-at created-at)
+
+                (and updated-at (integer? updated-at))
+                (assoc :block/updated-at updated-at))]
+    (dissoc block :title :body :anchor)))
+
+(defn extract-blocks
+  "Extract headings from mldoc ast.
+  Args:
+    `blocks`: mldoc ast.
+    `content`: markdown or org-mode text.
+    `with-id?`: If `with-id?` equals to true, all the referenced pages will have new db ids.
+    `format`: content's format, it could be either :markdown or :org-mode.
+    `options`: Options supported are :user-config, :block-pattern :supported-formats,
+     :date-formatter and :db"
+  [blocks content with-id? format {:keys [user-config] :as options}]
+  {:pre [(seq blocks) (string? content) (boolean? with-id?) (contains? #{:markdown :org} format)]}
+  (try
+    (let [encoded-content (utf8/encode content)
+          [blocks body pre-block-properties]
+          (loop [headings []
+                 blocks (reverse blocks)
+                 timestamps {}
+                 properties {}
+                 body []]
+            (if (seq blocks)
+              (let [[block pos-meta] (first blocks)
+                    ;; fix start_pos
+                    pos-meta (assoc pos-meta :end_pos
+                                    (if (seq headings)
+                                      (get-in (last headings) [:meta :start_pos])
+                                      nil))]
+                (cond
+                  (paragraph-timestamp-block? block)
+                  (let [timestamps (extract-timestamps block)
+                        timestamps' (merge timestamps timestamps)]
+                    (recur headings (rest blocks) timestamps' properties body))
+
+                  (gp-property/properties-ast? block)
+                  (let [properties (extract-properties format (second block) user-config)]
+                    (recur headings (rest blocks) timestamps properties body))
+
+                  (heading-block? block)
+                  (let [block (construct-block block properties timestamps body encoded-content format pos-meta with-id? options)]
+                    (recur (conj headings block) (rest blocks) {} {} []))
+
+                  :else
+                  (recur headings (rest blocks) timestamps properties (conj body block))))
+              [(-> (reverse headings)
+                   sanity-blocks-data)
+               body
+               properties]))
+          result (with-pre-block-if-exists blocks body pre-block-properties encoded-content options)]
+      (map #(dissoc % :block/meta) result))
+    (catch :default e
+      (log/error :extract-blocks-failure e))))
+
+(defn with-parent-and-left
+  [page-id blocks]
+  (loop [blocks (map (fn [block] (assoc block :block/level-spaces (:block/level block))) blocks)
+         parents [{:page/id page-id     ; db id or a map {:block/name "xxx"}
+                   :block/level 0
+                   :block/level-spaces 0}]
+         result []]
+    (if (empty? blocks)
+      (map #(dissoc % :block/level-spaces) result)
+      (let [[block & others] blocks
+            level-spaces (:block/level-spaces block)
+            {:block/keys [uuid level parent] :as last-parent} (last parents)
+            parent-spaces (:block/level-spaces last-parent)
+            [blocks parents result]
+            (cond
+              (= level-spaces parent-spaces)        ; sibling
+              (let [block (assoc block
+                                 :block/parent parent
+                                 :block/left [:block/uuid uuid]
+                                 :block/level level)
+                    parents' (conj (vec (butlast parents)) block)
+                    result' (conj result block)]
+                [others parents' result'])
+
+              (> level-spaces parent-spaces)         ; child
+              (let [parent (if uuid [:block/uuid uuid] (:page/id last-parent))
+                    block (cond->
+                            (assoc block
+                                  :block/parent parent
+                                  :block/left parent)
+                            ;; fix block levels with wrong order
+                            ;; For example:
+                            ;;   - a
+                            ;; - b
+                            ;; What if the input indentation is two spaces instead of 4 spaces
+                            (>= (- level-spaces parent-spaces) 1)
+                            (assoc :block/level (inc level)))
+                    parents' (conj parents block)
+                    result' (conj result block)]
+                [others parents' result'])
+
+              (< level-spaces parent-spaces)
+              (cond
+                (some #(= (:block/level-spaces %) (:block/level-spaces block)) parents) ; outdent
+                (let [parents' (vec (filter (fn [p] (<= (:block/level-spaces p) level-spaces)) parents))
+                      left (last parents')
+                      blocks (cons (assoc (first blocks)
+                                          :block/level (dec level)
+                                          :block/left [:block/uuid (:block/uuid left)])
+                                   (rest blocks))]
+                  [blocks parents' result])
+
+                :else
+                (let [[f r] (split-with (fn [p] (<= (:block/level-spaces p) level-spaces)) parents)
+                      left (first r)
+                      parent-id (if-let [block-id (:block/uuid (last f))]
+                                  [:block/uuid block-id]
+                                  page-id)
+                      block (cond->
+                              (assoc block
+                                     :block/parent parent-id
+                                     :block/left [:block/uuid (:block/uuid left)]
+                                     :block/level (:block/level left)
+                                     :block/level-spaces (:block/level-spaces left)))
+
+                      parents' (->> (concat f [block]) vec)
+                      result' (conj result block)]
+                  [others parents' result'])))]
+        (recur blocks parents result)))))

src/main/logseq/graph_parser/config.cljs

@@ -14,3 +14,11 @@
 (defn draw?
   [path]
   (string/starts-with? path default-draw-directory))
+
+;; TODO: rename
+(defonce mldoc-support-formats
+  #{:org :markdown :md})
+
+(defn mldoc-support?
+  [format]
+  (contains? mldoc-support-formats (keyword format)))

src/main/logseq/graph_parser/date_time_util.cljs

@@ -0,0 +1,51 @@
+(ns ^:nbb-compatible logseq.graph-parser.date-time-util
+  "cljs-time util fns for graph-parser"
+  (:require [cljs-time.coerce :as tc]
+            [cljs-time.core :as t]
+            [cljs-time.format :as tf]
+            [clojure.string :as string]
+            [logseq.graph-parser.util :as gp-util]))
+
+(defn time-ms
+  "Copy of util/time-ms. Too basic to couple this to main app"
+  []
+  (tc/to-long (t/now)))
+
+;; (tf/parse (tf/formatter "dd.MM.yyyy") "2021Q4") => 20040120T000000
+(defn safe-journal-title-formatters
+  [date-formatter]
+  (->> [date-formatter "MMM do, yyyy" "yyyy-MM-dd" "yyyy_MM_dd"]
+       (remove string/blank?)
+       distinct))
+
+(defn journal-title->
+  [journal-title then-fn formatters]
+  (when-not (string/blank? journal-title)
+    (when-let [time (->> (map
+                          (fn [formatter]
+                            (try
+                              (tf/parse (tf/formatter formatter) (gp-util/capitalize-all journal-title))
+                              (catch js/Error _e
+                                nil)))
+                          formatters)
+                         (filter some?)
+                         first)]
+      (then-fn time))))
+
+(defn journal-title->int
+  [journal-title formatters]
+  (when journal-title
+    (let [journal-title (gp-util/capitalize-all journal-title)]
+      (journal-title-> journal-title
+                       #(gp-util/parse-int (tf/unparse (tf/formatter "yyyyMMdd") %))
+                       formatters))))
+
+(defn format
+  [date date-formatter]
+  (when date-formatter
+    (tf/unparse (tf/formatter date-formatter) date)))
+
+(defn int->journal-title
+  [day date-formatter]
+  (when day
+    (format (tf/parse (tf/formatter "yyyyMMdd") (str day)) date-formatter)))

src/main/frontend/handler/extract.cljs → src/main/logseq/graph_parser/extract.cljc

@@ -1,28 +1,27 @@
-(ns frontend.handler.extract
-  "Extract helper."
+(ns ^:nbb-compatible logseq.graph-parser.extract
+  ;; Disable clj linters since we don't support clj
+  #?(:clj {:clj-kondo/config {:linters {:unresolved-namespace {:level :off}
+                                        :unresolved-symbol {:level :off}}}})
   (:require [clojure.set :as set]
             [clojure.string :as string]
             [clojure.walk :as walk]
-            [frontend.config :as config]
-            [frontend.db :as db]
-            [frontend.format :as format]
-            [frontend.format.block :as block]
-            [frontend.format.mldoc :as mldoc]
-            [frontend.state :as state]
+            [datascript.core :as d]
             [logseq.graph-parser.text :as text]
-            [frontend.util :as util]
             [logseq.graph-parser.util :as gp-util]
             [logseq.graph-parser.mldoc :as gp-mldoc]
-            [frontend.util.property :as property]
-            [lambdaisland.glogi :as log]))
+            [logseq.graph-parser.block :as gp-block]
+            [logseq.graph-parser.property :as gp-property]
+            [logseq.graph-parser.config :as gp-config]
+            #?(:org.babashka/nbb [logseq.graph-parser.log :as log]
+               :default [lambdaisland.glogi :as log])))
 
-(defn get-page-name
-  [file ast]
+(defn- get-page-name
+  [file ast page-name-order]
   ;; headline
   (let [ast (map first ast)]
     (if (string/includes? file "pages/contents.")
       "Contents"
-      (let [first-block (last (first (filter block/heading-block? ast)))
+      (let [first-block (last (first (filter gp-block/heading-block? ast)))
             property-name (when (and (contains? #{"Properties" "Property_Drawer"} (ffirst ast))
                                      (not (string/blank? (:title (last (first ast))))))
                             (:title (last (first ast))))
@@ -32,24 +31,24 @@
                                     title))
             file-name (when-let [file-name (last (string/split file #"/"))]
                         (let [result (first (gp-util/split-last "." file-name))]
-                          (if (config/mldoc-support? (string/lower-case (util/get-file-ext file)))
-                            (util/url-decode (string/replace result "." "/"))
+                          (if (gp-config/mldoc-support? (string/lower-case (gp-util/get-file-ext file)))
+                            (js/decodeURIComponent (string/replace result "." "/"))
                             result)))]
         (or property-name
-            (if (= (state/page-name-order) "heading")
+            (if (= page-name-order "heading")
               (or first-block-name file-name)
               (or file-name first-block-name)))))))
 
 
 ;; TODO: performance improvement
 (defn- extract-pages-and-blocks
-  #_:clj-kondo/ignore
-  [repo-url format ast properties file content]
+  [format ast properties file content {:keys [date-formatter page-name-order db] :as options}]
   (try
-    (let [page (get-page-name file ast)
-          [_original-page-name page-name _journal-day] (block/convert-page-if-journal page)
-          blocks (->> (block/extract-blocks ast content false format)
-                      (block/with-parent-and-left {:block/name page-name}))
+    #_:clj-kondo/ignore ;;clj-kondo bug
+    (let [page (get-page-name file ast page-name-order)
+          [_original-page-name page-name _journal-day] (gp-block/convert-page-if-journal page date-formatter)
+          blocks (->> (gp-block/extract-blocks ast content false format (dissoc options :page-name-order))
+                      (gp-block/with-parent-and-left {:block/name page-name}))
           ref-pages (atom #{})
           ref-tags (atom #{})
           blocks (map (fn [block]
@@ -65,94 +64,95 @@
                                      :block/page [:block/name page-name]
                                      :block/refs block-ref-pages
                                      :block/path-refs block-path-ref-pages))))
-                   blocks)
+                      blocks)
           page-entity (let [alias (:alias properties)
                             alias (if (string? alias) [alias] alias)
                             aliases (and alias
-                                         (seq (remove #(or (= page-name (util/page-name-sanity-lc %))
+                                         (seq (remove #(or (= page-name (gp-util/page-name-sanity-lc %))
                                                            (string/blank? %)) ;; disable blank alias
                                                       alias)))
                             aliases (->>
                                      (map
-                                       (fn [alias]
-                                         (let [page-name (util/page-name-sanity-lc alias)
-                                               aliases (distinct
-                                                        (conj
-                                                         (remove #{alias} aliases)
-                                                         page))
-                                               aliases (when (seq aliases)
-                                                         (map
-                                                           (fn [alias]
-                                                             {:block/name (util/page-name-sanity-lc alias)})
-                                                           aliases))]
-                                           (if (seq aliases)
-                                             {:block/name page-name
-                                              :block/alias aliases}
-                                             {:block/name page-name})))
-                                       aliases)
+                                      (fn [alias]
+                                        (let [page-name (gp-util/page-name-sanity-lc alias)
+                                              aliases (distinct
+                                                       (conj
+                                                        (remove #{alias} aliases)
+                                                        page))
+                                              aliases (when (seq aliases)
+                                                        (map
+                                                         (fn [alias]
+                                                           {:block/name (gp-util/page-name-sanity-lc alias)})
+                                                         aliases))]
+                                          (if (seq aliases)
+                                            {:block/name page-name
+                                             :block/alias aliases}
+                                            {:block/name page-name})))
+                                      aliases)
                                      (remove nil?))]
                         (cond->
-                          (gp-util/remove-nils
-                           (assoc
-                            (block/page-name->map page false)
-                            :block/file {:file/path (gp-util/path-normalize file)}))
-                          (seq properties)
-                          (assoc :block/properties properties)
+                         (gp-util/remove-nils
+                          (assoc
+                           (gp-block/page-name->map page false db true date-formatter)
+                           :block/file {:file/path (gp-util/path-normalize file)}))
+                         (seq properties)
+                         (assoc :block/properties properties)
 
-                          (seq aliases)
-                          (assoc :block/alias aliases)
+                         (seq aliases)
+                         (assoc :block/alias aliases)
 
-                          (:tags properties)
-                          (assoc :block/tags (let [tags (:tags properties)
-                                                   tags (if (string? tags) [tags] tags)
-                                                   tags (remove string/blank? tags)]
-                                               (swap! ref-tags set/union (set tags))
-                                               (map (fn [tag] {:block/name (util/page-name-sanity-lc tag)
-                                                               :block/original-name tag})
-                                                 tags)))))
+                         (:tags properties)
+                         (assoc :block/tags (let [tags (:tags properties)
+                                                  tags (if (string? tags) [tags] tags)
+                                                  tags (remove string/blank? tags)]
+                                              (swap! ref-tags set/union (set tags))
+                                              (map (fn [tag] {:block/name (gp-util/page-name-sanity-lc tag)
+                                                              :block/original-name tag})
+                                                   tags)))))
           namespace-pages (let [page (:block/original-name page-entity)]
                             (when (text/namespace-page? page)
-                              (->> (util/split-namespace-pages page)
+                              (->> (gp-util/split-namespace-pages page)
                                    (map (fn [page]
-                                          (-> (block/page-name->map page true)
+                                          (-> (gp-block/page-name->map page true db true date-formatter)
                                               (assoc :block/format format)))))))
           pages (->> (concat
                       [page-entity]
                       @ref-pages
                       (map
-                        (fn [page]
-                          {:block/original-name page
-                           :block/name (util/page-name-sanity-lc page)})
-                        @ref-tags)
+                       (fn [page]
+                         {:block/original-name page
+                          :block/name (gp-util/page-name-sanity-lc page)})
+                       @ref-tags)
                       namespace-pages)
                      ;; remove block references
                      (remove vector?)
                      (remove nil?))
-          pages (util/distinct-by :block/name pages)
+          pages (gp-util/distinct-by :block/name pages)
           pages (remove nil? pages)
-          pages (map (fn [page] (assoc page :block/uuid (db/new-block-id))) pages)
+          pages (map (fn [page] (assoc page :block/uuid (d/squuid))) pages)
           blocks (->> (remove nil? blocks)
                       (map (fn [b] (dissoc b :block/title :block/body :block/level :block/children :block/meta :block/anchor))))]
       [pages blocks])
-    (catch js/Error e
+    (catch :default e
       (log/error :exception e))))
 
 (defn extract-blocks-pages
-  [repo-url file content]
+  [file content {:keys [user-config] :as options}]
   (if (string/blank? content)
     []
-    (let [format (format/get-format file)
+    (let [format (gp-util/get-format file)
           _ (println "Parsing start: " file)
-          ast (mldoc/->edn content (gp-mldoc/default-config format
+          ast (gp-mldoc/->edn content (gp-mldoc/default-config format
                                                          ;; {:parse_outline_only? true}
-                                                         ))]
+                                                         )
+                           user-config)]
       (println "Parsing finished : " file)
       (let [first-block (ffirst ast)
-            properties (let [properties (and (property/properties-ast? first-block)
+            properties (let [properties (and (gp-property/properties-ast? first-block)
                                              (->> (last first-block)
                                                   (map (fn [[x y]]
                                                          [x (if (string? y)
-                                                              (text/parse-property format x y (state/get-config))
+                                                              (text/parse-property format x y user-config)
                                                               y)]))
                                                   (into {})
                                                   (walk/keywordize-keys)))]
@@ -163,17 +163,16 @@
                                        (string/replace (or v "") "\\" "")))
                              properties)))]
         (extract-pages-and-blocks
-         repo-url
          format ast properties
-         file content)))))
+         file content options)))))
 
-(defn with-block-uuid
+(defn- with-block-uuid
   [pages]
-  (->> (util/distinct-by :block/name pages)
+  (->> (gp-util/distinct-by :block/name pages)
        (map (fn [page]
               (if (:block/uuid page)
                 page
-                (assoc page :block/uuid (db/new-block-id)))))))
+                (assoc page :block/uuid (d/squuid)))))))
 
 (defn with-ref-pages
   [pages blocks]

src/main/logseq/graph_parser/property.cljs

@@ -0,0 +1,48 @@
+(ns logseq.graph-parser.property
+  "Property fns needed by graph-parser"
+  (:require [logseq.graph-parser.util :as gp-util]
+            [clojure.string :as string]
+            [goog.string :as gstring]))
+
+(defn properties-ast?
+  [block]
+  (and
+   (vector? block)
+   (contains? #{"Property_Drawer" "Properties"}
+              (first block))))
+
+(defonce properties-start ":PROPERTIES:")
+(defonce properties-end ":END:")
+(defonce properties-end-pattern
+  (re-pattern (gstring/format "%s[\t\r ]*\n|(%s\\s*$)" properties-end properties-end)))
+
+(defn contains-properties?
+  [content]
+  (when content
+    (and (string/includes? content properties-start)
+         (gp-util/safe-re-find properties-end-pattern content))))
+
+(defn ->new-properties
+  "New syntax: key:: value"
+  [content]
+  (if (contains-properties? content)
+    (let [lines (string/split-lines content)
+          start-idx (.indexOf lines properties-start)
+          end-idx (.indexOf lines properties-end)]
+      (if (and (>= start-idx 0) (> end-idx 0) (> end-idx start-idx))
+        (let [before (subvec lines 0 start-idx)
+              middle (->> (subvec lines (inc start-idx) end-idx)
+                          (map (fn [text]
+                                 (let [[k v] (gp-util/split-first ":" (subs text 1))]
+                                   (if (and k v)
+                                     (let [k (string/replace k "_" "-")
+                                           compare-k (keyword (string/lower-case k))
+                                           k (if (contains? #{:id :custom_id :custom-id} compare-k) "id" k)
+                                           k (if (contains? #{:last-modified-at} compare-k) "updated-at" k)]
+                                       (str k ":: " (string/trim v)))
+                                     text)))))
+              after (subvec lines (inc end-idx))
+              lines (concat before middle after)]
+          (string/join "\n" lines))
+        content))
+    content))

src/main/logseq/graph_parser/util.cljs

@@ -8,6 +8,7 @@
 (defonce exactly-uuid-pattern (re-pattern (str "(?i)^" uuid-pattern "$")))
 
 (defn safe-re-find
+  "Copy of frontend.util/safe-re-find. Too basic to couple to main app"
   [pattern s]
   (when-not (string? s)
     ;; TODO: sentry
@@ -16,6 +17,7 @@
     (re-find pattern s)))
 
 (defn uuid-string?
+  "Copy of frontend.util/uuid-string?. Too basic to couple to main app"
   [s]
   (safe-re-find exactly-uuid-pattern s))
 
@@ -66,14 +68,14 @@
   (and (string? v) (>= (count v) 2) (= "\"" (first v) (last v))))
 
 (defn parse-int
-  "Copy of frontend.util/parse-int. Don't want to couple to main app too much"
+  "Copy of frontend.util/parse-int. Too basic to couple to main app"
   [x]
   (if (string? x)
     (js/parseInt x)
     x))
 
 (defn safe-parse-int
-  "Copy of frontend.util/safe-parse-int. Don't want to couple to main app too much"
+  "Copy of frontend.util/safe-parse-int. Too basic to couple to main app"
   [x]
   (let [result (parse-int x)]
     (if (js/isNaN result)
@@ -94,3 +96,94 @@
   (-> json-string
       (js/JSON.parse)
       (js->clj :keywordize-keys true)))
+
+;; TODO: Use update-keys once its available in cljs and nbb
+(defn map-keys
+  "Maps function `f` over the keys of map `m` to produce a new map."
+  [f m]
+  (reduce-kv
+   (fn [m_ k v]
+     (assoc m_ (f k) v)) {} m))
+
+(defn zero-pad
+  "Copy of frontend.util/zero-pad. Too basic to couple to main app"
+  [n]
+  (if (< n 10)
+    (str "0" n)
+    (str n)))
+
+(defn get-file-ext
+  "Copy of frontend.util/get-file-ext. Too basic to couple to main app"
+  [file]
+  (and
+   (string? file)
+   (string/includes? file ".")
+   (some-> (last (string/split file #"\.")) string/lower-case)))
+
+(defn remove-boundary-slashes
+  [s]
+  (when (string? s)
+    (let [s (if (= \/ (first s))
+              (subs s 1)
+              s)]
+      (if (= \/ (last s))
+        (subs s 0 (dec (count s)))
+        s))))
+
+(defn split-namespace-pages
+  [title]
+  (let [parts (string/split title "/")]
+    (loop [others (rest parts)
+           result [(first parts)]]
+      (if (seq others)
+        (let [prev (last result)]
+          (recur (rest others)
+                 (conj result (str prev "/" (first others)))))
+        result))))
+
+(defn page-name-sanity
+  "Sanitize the page-name."
+  ([page-name]
+   (page-name-sanity page-name false))
+  ([page-name replace-slash?]
+   (let [page (some-> page-name
+                      (remove-boundary-slashes)
+                      (path-normalize))]
+     (if replace-slash?
+       (string/replace page #"/" "%2A")
+       page))))
+
+(defn page-name-sanity-lc
+  "Sanitize the query string for a page name (mandate for :block/name)"
+  [s]
+  (page-name-sanity (string/lower-case s)))
+
+(defn capitalize-all
+  [s]
+  (some->> (string/split s #" ")
+           (map string/capitalize)
+           (string/join " ")))
+
+(defn distinct-by
+  "Copy of frontend.util/distinct-by. Too basic to couple to main app"
+  [f col]
+  (reduce
+   (fn [acc x]
+     (if (some #(= (f x) (f %)) acc)
+       acc
+       (vec (conj acc x))))
+   []
+   col))
+
+(defn normalize-format
+  [format]
+  (case (keyword format)
+    :md :markdown
+    :asciidoc :adoc
+    ;; default
+    (keyword format)))
+
+(defn get-format
+  [file]
+  (when file
+    (normalize-format (keyword (string/lower-case (last (string/split file #"\.")))))))

src/test/frontend/db/conn.clj

@@ -1,3 +0,0 @@
-(ns frontend.db.conn)
-
-

src/test/frontend/handler/block_test.cljs

@@ -1,9 +0,0 @@
-(ns frontend.handler.block-test)
-
-
-
-(comment
-  (defn clip-block [x]
-    (map #(select-keys % [:block/parent :block/left :block/pre-block? :block/uuid :block/level
-                          :db/id])
-      x)))

src/test/frontend/modules/outliner/core_test.cljs

@@ -8,7 +8,7 @@
             [frontend.db :as db]
             [frontend.db.model :as db-model]
             [clojure.walk :as walk]
-            [frontend.format.block :as block]
+            [logseq.graph-parser.block :as gp-block]
             [datascript.core :as d]
             [frontend.test.helper :as helper]))
 
@@ -65,7 +65,7 @@
 
 (defn- build-blocks
   [tree]
-  (block/with-parent-and-left 1 (build-node-tree tree)))
+  (gp-block/with-parent-and-left 1 (build-node-tree tree)))
 
 (defn transact-tree!
   [tree]

src/test/frontend/react.cljc

@@ -1,67 +0,0 @@
-(ns frontend.react
-  "To facilitate testing, imitate the behavior of react.
-  Note: don't run component parallel"
-  #?(:cljs (:require-macros [frontend.react])))
-
-#_{:component-key {:result nil
-                   :watches []
-                   :root-info nil}}
-(def react-components (atom {}))
-(def ^:dynamic *with-key* nil)
-(def ^:dynamic *comp-key* nil)
-(def ^:dynamic *root-info* nil)
-
-(defn react
-  [react-ref]
-  (let [comp-key *comp-key*
-        component (get @react-components comp-key)]
-    (cond
-      (some? component)
-      (do
-        (when-not ((:watches component) react-ref)
-          (let [new-component (update component :watches conj react-ref)]
-            (swap! react-components assoc comp-key new-component)
-            (add-watch react-ref comp-key
-                       (fn [_key _atom old-state new-state]
-                         (when-not (= old-state new-state)
-                           (let [root-info (get-in @react-components [comp-key :root-info])
-                                 {:keys [f comp-key]} root-info]
-                             (binding [*with-key* comp-key
-                                       *root-info* root-info]
-                               (let [component (get @react-components comp-key)]
-                                 (reset! (:result component) (f))))))))))
-        @react-ref)
-
-      ;; Sometime react is not used in component by accident, return the val.
-      :else
-      @react-ref)))
-
-(defn set-comp-and-calc-result
-  [f]
-  (let [{result :result :as component} (get @react-components *comp-key*)]
-    (if component
-      (do (reset! result (f)) result)
-      (let [result (atom nil)]
-        (binding [*root-info* (if *root-info* *root-info* {:f f :comp-key *comp-key*})]
-          (let [component {:result result
-                           :watches #{}
-                           :root-info *root-info*}]
-            (swap! react-components assoc *comp-key* component))
-          (reset! result (f))
-          result)))))
-
-#?(:clj (defmacro defc
-          [sym args & body]
-          `(defn ~sym ~args
-             (let [f# (fn []
-                        (binding [*comp-key* *with-key*
-                                  ;; inner component should specify own *with-key*
-                                  *with-key* nil]
-                          ~@body))]
-               (binding [*comp-key* *with-key*]
-                 (set-comp-and-calc-result f#))))))
-
-#?(:clj (defmacro with-key
-          [key & body]
-          `(binding [*with-key* ~key]
-             ~@body)))

src/test/frontend/react_test.cljs

@@ -1,64 +0,0 @@
-(ns frontend.react-test
-  ;; namespace local config for r/defc tests
-  {:clj-kondo/config {:linters {:inline-def {:level :off}}}}
-  (:require [frontend.react :as r]
-            [cljs.test :refer [deftest is use-fixtures]]
-            [frontend.test.fixtures :as fixtures]))
-
-(use-fixtures :each
-  fixtures/react-components)
-
-(deftest simple-react-test
-  (let [react-ref (atom 1)]
-
-    (r/defc simple-component
-      []
-      (+ 2 (r/react react-ref)))
-
-    (let [result (r/with-key 1 (simple-component))]
-
-      (is (= 3 @result))
-      (reset! react-ref 2)
-      (is (= 4 @result)))))
-
-(deftest nest-component-test
-  (let [a (atom 1)
-        b (atom 2)]
-
-    (r/defc inner
-      []
-
-      (let [r (r/react b)]
-        r))
-
-    (r/defc out
-      []
-      (let [out (r/react a)
-            inner-result (r/with-key "1" (inner))]
-        (+ out @inner-result)))
-
-    (let [out-result (r/with-key "2" (out))]
-      (is (= 3 @out-result))
-      (reset! b 4)
-      (is (= 5 @out-result)))))
-
-#_(deftest defc-params-test
-  (let [a (atom 1)
-        b (atom 2)]
-
-    (r/defc inner-1
-      [c]
-      (+ c (r/react b)))
-
-    (r/defc out-1
-      []
-      (let [out (r/react a)
-            inner-result (r/with-key 1 (inner-1 5))]
-        (+ out @inner-result)))
-
-    (let [out-result (r/with-key 2 (out-1))]
-      (is (= 8 @out-result))
-
-      (reset! b 4)
-
-      (is (= 10 @out-result)))))

src/test/frontend/test/docs_graph_helper.cljs

@@ -5,7 +5,7 @@
             [clojure.string :as string]))
 
 
-(defn- slurp
+(defn slurp
   "Like clojure.core/slurp"
   [file]
   (str (fs/readFileSync file)))

src/test/frontend/util/property_test.cljs

@@ -146,29 +146,6 @@ SCHEDULED: <2021-10-25 Mon>\n:PROPERTIES:\n:a: b\n:END:\nworld\n" "c" "d")
     (property/insert-properties :markdown "" {:foo "\"bar, baz\""})
     "foo:: \"bar, baz\""))
 
-(deftest test->new-properties
-  (are [x y] (= (property/->new-properties x) y)
-    ":PROPERTIES:\n:foo: bar\n:END:"
-    "foo:: bar"
-
-    "hello\n:PROPERTIES:\n:foo: bar\n:END:"
-    "hello\nfoo:: bar"
-
-    "hello\n:PROPERTIES:\n:foo: bar\n:nice: bingo\n:END:"
-    "hello\nfoo:: bar\nnice:: bingo"
-
-    "hello\n:PROPERTIES:\n:foo: bar\n:nice: bingo\n:END:"
-    "hello\nfoo:: bar\nnice:: bingo"
-
-    "hello\n:PROPERTIES:\n:foo: bar\n:nice: bingo\n:END:\nnice"
-    "hello\nfoo:: bar\nnice:: bingo\nnice"
-
-    "hello\n:PROPERTIES:\n:foo: bar\n:nice:\n:END:\nnice"
-    "hello\nfoo:: bar\nnice:: \nnice"
-
-    "hello\n:PROPERTIES:\n:foo: bar\n:nice\n:END:\nnice"
-    "hello\nfoo:: bar\n:nice\nnice"))
-
 (deftest test-build-properties-str
   (are [x y] (= (property/build-properties-str :mardown x) y)
     {:title "a"}

src/test/frontend/format/block_test.cljs → src/test/logseq/graph_parser/block_test.cljs

@@ -1,9 +1,9 @@
-(ns frontend.format.block-test
-  (:require [frontend.format.block :as block]
+(ns logseq.graph-parser.block-test
+  (:require [logseq.graph-parser.block :as gp-block]
             [cljs.test :refer [deftest are]]))
 
 (deftest test-extract-properties
-  (are [x y] (= (:properties (block/extract-properties :markdown x)) y)
+  (are [x y] (= (:properties (gp-block/extract-properties :markdown x {})) y)
     [["year" "1000"]] {:year 1000}
     [["year" "\"1000\""]] {:year "\"1000\""}
     [["background-color" "#000000"]] {:background-color "#000000"}
@@ -23,7 +23,7 @@
     [["foo" "bar, [[baz, test]]"]] {:foo #{"bar" "baz, test"}}
     [["foo" "bar, [[baz, test, [[nested]]]]"]] {:foo #{"bar" "baz, test, [[nested]]"}})
 
-  (are [x y] (= (vec (:page-refs (block/extract-properties :markdown x))) y)
+  (are [x y] (= (vec (:page-refs (gp-block/extract-properties :markdown x {}))) y)
     [["year" "1000"]] []
     [["year" "\"1000\""]] []
     [["foo" "[[bar]] test"]] ["bar" "test"]

src/test/frontend/handler/extract_test.cljs → src/test/logseq/graph_parser/extract_test.cljs

@@ -1,17 +1,17 @@
-(ns frontend.handler.extract-test
+(ns logseq.graph-parser.extract-test
   (:require [cljs.test :refer [async deftest is]]
-            [frontend.handler.extract :as extract]
-            [frontend.util :as util]
+            [logseq.graph-parser.extract :as extract]
+            [clojure.pprint :as pprint]
             [promesa.core :as p]))
 
 (defn- extract
   [text]
-  (p/let [result (extract/extract-blocks-pages "repo" "a.md" text)
+  (p/let [result (extract/extract-blocks-pages "a.md" text {:block-pattern "-"})
           result (last result)
           lefts (map (juxt :block/parent :block/left) result)]
     (if (not= (count lefts) (count (distinct lefts)))
       (do
-        (util/pprint (map (fn [x] (select-keys x [:block/uuid :block/level :block/content :block/left])) result))
+        (pprint/pprint (map (fn [x] (select-keys x [:block/uuid :block/level :block/content :block/left])) result))
         (throw (js/Error. ":block/parent && :block/left conflicts")))
       (mapv :block/content result))))
 

src/test/logseq/graph_parser/nbb_test_runner.cljs

@@ -4,7 +4,10 @@
             [logseq.graph-parser.mldoc :as gp-mldoc]
             [logseq.graph-parser.text :as text]
             [logseq.graph-parser.text-test]
-            [logseq.graph-parser.mldoc-test]))
+            [logseq.graph-parser.mldoc-test]
+            [logseq.graph-parser.block-test]
+            [logseq.graph-parser.property-test]
+            [logseq.graph-parser.extract-test]))
 
 (defmethod cljs.test/report [:cljs.test/default :end-run-tests] [m]
   (when-not (cljs.test/successful? m)
@@ -17,5 +20,7 @@
   #_:clj-kondo/ignore
   (alter-var-root #'gp-mldoc/parse-property (constantly text/parse-property))
   (t/run-tests 'logseq.graph-parser.mldoc-test
-               ;; TODO: Enable when https://github.com/babashka/nbb/issues/187 works
-               'logseq.graph-parser.text-test))
+               'logseq.graph-parser.text-test
+               'logseq.graph-parser.property-test
+               'logseq.graph-parser.block-test
+               'logseq.graph-parser.extract-test))

src/test/logseq/graph_parser/property_test.cljs

@@ -0,0 +1,26 @@
+(ns logseq.graph-parser.property-test
+  (:require [cljs.test :refer [are deftest]]
+            [logseq.graph-parser.property :as gp-property]))
+
+(deftest test->new-properties
+  (are [x y] (= (gp-property/->new-properties x) y)
+    ":PROPERTIES:\n:foo: bar\n:END:"
+    "foo:: bar"
+
+    "hello\n:PROPERTIES:\n:foo: bar\n:END:"
+    "hello\nfoo:: bar"
+
+    "hello\n:PROPERTIES:\n:foo: bar\n:nice: bingo\n:END:"
+    "hello\nfoo:: bar\nnice:: bingo"
+
+    "hello\n:PROPERTIES:\n:foo: bar\n:nice: bingo\n:END:"
+    "hello\nfoo:: bar\nnice:: bingo"
+
+    "hello\n:PROPERTIES:\n:foo: bar\n:nice: bingo\n:END:\nnice"
+    "hello\nfoo:: bar\nnice:: bingo\nnice"
+
+    "hello\n:PROPERTIES:\n:foo: bar\n:nice:\n:END:\nnice"
+    "hello\nfoo:: bar\nnice:: \nnice"
+
+    "hello\n:PROPERTIES:\n:foo: bar\n:nice\n:END:\nnice"
+    "hello\nfoo:: bar\n:nice\nnice"))