Move graph-parser related block code to proper ns and nbb migration wip

.clj-kondo/config.edn

@@ -21,7 +21,9 @@
              frontend.util util
              frontend.config config
              frontend.format.mldoc mldoc
+             frontend.format.block block
              logseq.graph-parser.text text
+             logseq.graph-parser.block gp-block
              logseq.graph-parser.mldoc gp-mldoc
              logseq.graph-parser.util gp-util
              logseq.graph-parser.config gp-config}}}

src/main/frontend/components/block.cljs

@@ -55,6 +55,7 @@
             [logseq.graph-parser.config :as gp-config]
             [logseq.graph-parser.util :as gp-util]
             [logseq.graph-parser.mldoc :as gp-mldoc]
+            [logseq.graph-parser.block :as gp-block]
             [goog.dom :as gdom]
             [goog.object :as gobj]
             [lambdaisland.glogi :as log]
@@ -1307,7 +1308,7 @@
          (->elem :sub (map-inline config l))
 
          ["Tag" _]
-         (when-let [s (block/get-tag item)]
+         (when-let [s (gp-block/get-tag item)]
            (let [s (text/page-ref-un-brackets! s)]
              (page-cp (assoc config :tag? true) {:block/name s})))
 

src/main/frontend/components/page.cljs

@@ -1,6 +1,6 @@
 (ns frontend.components.page
   (:require [clojure.string :as string]
-            [frontend.components.block :as block]
+            [frontend.components.block :as component-block]
             [frontend.components.content :as content]
             [frontend.components.editor :as editor]
             [frontend.components.hierarchy :as hierarchy]
@@ -15,7 +15,6 @@
             [frontend.db.model :as model]
             [frontend.extensions.graph :as graph]
             [frontend.extensions.pdf.assets :as pdf-assets]
-            [frontend.format.block :as format-block]
             [frontend.handler.common :as common-handler]
             [frontend.handler.config :as config-handler]
             [frontend.handler.editor :as editor-handler]
@@ -35,6 +34,7 @@
             [medley.core :as medley]
             [rum.core :as rum]
             [logseq.graph-parser.util :as gp-util]
+            [logseq.graph-parser.block :as gp-block]
             [frontend.mobile.util :as mobile-util]))
 
 (defn- get-page-name
@@ -139,7 +139,7 @@
                               :document/mode? document-mode?}
                              config)
               hiccup-config (common-handler/config-with-document-mode hiccup-config)
-              hiccup (block/->hiccup page-blocks hiccup-config {})]
+              hiccup (component-block/->hiccup page-blocks hiccup-config {})]
           [:div
            (page-blocks-inner page-name page-blocks hiccup sidebar? block-id)
            (when-not config/publishing?
@@ -163,9 +163,9 @@
            (rum/with-key
              (ui/catch-error
               (ui/component-error "Failed default query:" {:content (pr-str query)})
-              (block/custom-query {:attr {:class "mt-10"}
-                                   :editor-box editor/box
-                                   :page page} query))
+              (component-block/custom-query {:attr {:class "mt-10"}
+                                             :editor-box editor/box
+                                             :page page} query))
              (str repo "-custom-query-" (:query query))))]))))
 
 (defn tagged-pages
@@ -332,7 +332,7 @@
                       (db/entity repo))
                  (do
                    (when-not (db/entity repo [:block/name page-name])
-                     (let [m (format-block/page-name->map path-page-name true)]
+                     (let [m (gp-block/page-name->map path-page-name true)]
                        (db/transact! repo [m])))
                    (db/pull [:block/name page-name])))
           {:keys [icon]} (:block/properties page)
@@ -377,7 +377,7 @@
            (let [config {:id "block-parent"
                          :block? true}]
              [:div.mb-4
-              (block/breadcrumb config repo block-id {:level-limit 3})]))
+              (component-block/breadcrumb config repo block-id {:level-limit 3})]))
 
          ;; blocks
          (let [page (if block?
@@ -712,7 +712,7 @@
          [:tr {:key name}
           [:td.n.w-12 [:span.opacity-70 (str (inc n) ".")]]
           [:td.name [:a {:href     (rfe/href :page {:name (:block/name page)})}
-                     (block/page-cp {} page)]]
+                     (component-block/page-cp {} page)]]
           [:td.backlinks [:span (or backlinks "0")]]
           (when-not orphaned-pages? [:td.created-at [:span (if created-at (date/int->local-time-2 created-at) "Unknown")]])
           (when-not orphaned-pages? [:td.updated-at [:span (if updated-at (date/int->local-time-2 updated-at) "Unknown")]])])]]
@@ -954,7 +954,7 @@
                                                (:db/id page)
                                                :page))))
                               :href     (rfe/href :page {:name (:block/name page)})}
-                          (block/page-cp {} page)]]
+                          (component-block/page-cp {} page)]]
 
                (when-not mobile?
                  [:td.backlinks [:span backlinks]])

src/main/frontend/format/block.cljs

@@ -1,595 +1,13 @@
 (ns frontend.format.block
+  "Block code needed by app but not graph-parser"
   (:require [clojure.string :as string]
-            [clojure.walk :as walk]
-            [cljs.core.match :as match]
+            [logseq.graph-parser.block :as gp-block]
             [frontend.config :as config]
-            [frontend.date :as date]
             [frontend.db :as db]
             [frontend.format :as format]
             [frontend.state :as state]
-            [logseq.graph-parser.text :as text]
-            [logseq.graph-parser.utf8 :as utf8]
-            [frontend.util :as util]
             [frontend.util.property :as property]
-            [logseq.graph-parser.util :as gp-util]
-            [logseq.graph-parser.config :as gp-config]
-            [logseq.graph-parser.mldoc :as gp-mldoc]
-            [lambdaisland.glogi :as log]
-            [medley.core :as medley]))
-
-(defn heading-block?
-  [block]
-  (and
-   (vector? block)
-   (= "Heading" (first block))))
-
-(defn get-tag
-  [block]
-  (when-let [tag-value (and (vector? block)
-                            (= "Tag" (first block))
-                            (second block))]
-    (->
-     (map (fn [e]
-            (match/match e
-              ["Plain" s]
-              s
-              ["Link" t]
-              (let [{full_text :full_text} t]
-                full_text)
-              ["Nested_link" t]
-              (let [ {content :content} t]
-                content)
-              :else
-              ""
-              )) tag-value)
-     (string/join))))
-
-(defn get-page-reference
-  [block]
-  (let [page (cond
-               (and (vector? block) (= "Link" (first block)))
-               (let [typ (first (:url (second block)))
-                     value (second (:url (second block)))]
-                 ;; {:url ["File" "file:../pages/hello_world.org"], :label [["Plain" "hello world"]], :title nil}
-                 (or
-                  (and
-                   (= typ "Page_ref")
-                   (and (string? value)
-                        (not (or (gp-config/local-asset? value)
-                                 (gp-config/draw? value))))
-                   value)
-
-                  (and
-                   (= typ "Search")
-                   (text/page-ref? value)
-                   (text/page-ref-un-brackets! value))
-
-                  (and
-                   (= typ "Search")
-                   (not (contains? #{\# \* \/ \[} (first value)))
-                   (let [ext (some-> (util/get-file-ext value) keyword)]
-                     (when (and (not (util/starts-with? value "http:"))
-                                (not (util/starts-with? value "https:"))
-                                (not (util/starts-with? value "file:"))
-                                (not (gp-config/local-asset? value))
-                                (or (= ext :excalidraw)
-                                    (not (contains? (config/supported-formats) ext))))
-                       value)))
-
-                  (and
-                   (= typ "Complex")
-                   (= (:protocol value) "file")
-                   (:link value))
-
-                  (and
-                   (= typ "File")
-                   (second (first (:label (second block)))))))
-
-               (and (vector? block) (= "Nested_link" (first block)))
-               (let [content (:content (last block))]
-                 (subs content 2 (- (count content) 2)))
-
-               (and (vector? block)
-                    (= "Macro" (first block)))
-               (let [{:keys [name arguments]} (second block)
-                     argument (string/join ", " arguments)]
-                   (when (= name "embed")
-                     (text/page-ref-un-brackets! argument)))
-
-               (and (vector? block)
-                    (= "Tag" (first block)))
-               (let [text (get-tag block)]
-                 (text/page-ref-un-brackets! text))
-
-               :else
-               nil)]
-    (text/block-ref-un-brackets! page)))
-
-(defn get-block-reference
-  [block]
-  (when-let [block-id (cond
-                        (and (vector? block)
-                             (= "Block_reference" (first block)))
-                        (last block)
-
-                        (and (vector? block)
-                             (= "Link" (first block))
-                             (map? (second block))
-                             (= "Block_ref" (first (:url (second block)))))
-                        (second (:url (second block)))
-
-                        (and (vector? block)
-                             (= "Macro" (first block)))
-                        (let [{:keys [name arguments]} (second block)]
-                          (when (and (= name "embed")
-                                     (string? (first arguments))
-                                     (string/starts-with? (first arguments) "((")
-                                     (string/ends-with? (first arguments) "))"))
-                            (subs (first arguments) 2 (- (count (first arguments)) 2))))
-
-                        (and (vector? block)
-                             (= "Link" (first block))
-                             (map? (second block)))
-                        (if (= "id" (:protocol (second (:url (second block)))))
-                          (:link (second (:url (second block))))
-                          (let [id (second (:url (second block)))]
-                            (text/block-ref-un-brackets! id)))
-
-                        :else
-                        nil)]
-    (when (and block-id
-               (gp-util/uuid-string? block-id))
-      block-id)))
-
-(defn paragraph-block?
-  [block]
-  (and
-   (vector? block)
-   (= "Paragraph" (first block))))
-
-(defn timestamp-block?
-  [block]
-  (and
-   (vector? block)
-   (= "Timestamp" (first block))))
-
-;; TODO: we should move this to mldoc
-(defn extract-properties
-  [format properties]
-  (when (seq properties)
-    (let [properties (seq properties)
-          page-refs (->>
-                     properties
-                     (remove (fn [[k _]]
-                               (contains? #{:background-color :background_color} (keyword k))))
-                     (map last)
-                     (map (fn [v]
-                            (when (and (string? v)
-                                       (not (gp-mldoc/link? format v)))
-                              (let [v (string/trim v)
-                                    result (text/split-page-refs-without-brackets v {:un-brackets? false})]
-                                (if (coll? result)
-                                  (map text/page-ref-un-brackets! result)
-                                  [])))))
-                     (apply concat)
-                     (remove string/blank?))
-          properties (->> properties
-                          (map (fn [[k v]]
-                                 (let [k (-> (string/lower-case (name k))
-                                             (string/replace " " "-")
-                                             (string/replace "_" "-"))
-                                       k (if (contains? #{"custom_id" "custom-id"} k)
-                                           "id"
-                                           k)
-                                       v (if (coll? v)
-                                           (remove string/blank? v)
-                                           (if (string/blank? v)
-                                             nil
-                                             (text/parse-property format k v (state/get-config))))
-                                       k (keyword k)
-                                       v (if (and
-                                              (string? v)
-                                              (contains? #{:alias :aliases :tags} k))
-                                           (set [v])
-                                           v)
-                                       v (if (coll? v) (set v) v)]
-                                   [k v])))
-                          (remove #(nil? (second %))))]
-      {:properties (into {} properties)
-       :properties-order (map first properties)
-       :page-refs page-refs})))
-
-(defn- paragraph-timestamp-block?
-  [block]
-  (and (paragraph-block? block)
-       (or (timestamp-block? (first (second block)))
-           (timestamp-block? (second (second block))))))
-
-(defn extract-timestamps
-  [block]
-  (some->>
-   (second block)
-   (filter timestamp-block?)
-   (map last)
-   (into {})))
-
-;; {"Deadline" {:date {:year 2020, :month 10, :day 20}, :wday "Tue", :time {:hour 8, :min 0}, :repetition [["DoublePlus"] ["Day"] 1], :active true}}
-(defn timestamps->scheduled-and-deadline
-  [timestamps]
-  (let [timestamps (medley/map-keys (comp keyword string/lower-case) timestamps)
-        m (some->> (select-keys timestamps [:scheduled :deadline])
-                   (map (fn [[k v]]
-                          (let [{:keys [date repetition]} v
-                                {:keys [year month day]} date
-                                day (js/parseInt (str year (util/zero-pad month) (util/zero-pad day)))]
-                            (cond->
-                             (case k
-                               :scheduled
-                               {:scheduled day}
-                               :deadline
-                               {:deadline day})
-                              repetition
-                              (assoc :repeated? true))))))]
-    (apply merge m)))
-
-(defn convert-page-if-journal
-  "Convert journal file name to user' custom date format"
-  [original-page-name]
-  (when original-page-name
-    (let [page-name (util/page-name-sanity-lc original-page-name)
-          day (date/journal-title->int page-name)]
-     (if day
-       (let [original-page-name (date/int->journal-title day)]
-         [original-page-name (util/page-name-sanity-lc original-page-name) day])
-       [original-page-name page-name day]))))
-
-(defn page-name->map
-  "Create a page's map structure given a original page name (string).
-   map as input is supported for legacy compatibility.
-   with-timestamp?: assign timestampes to the map structure.
-    Useful when creating new pages from references or namespaces,
-    as there's no chance to introduce timestamps via editing in page"
-  ([original-page-name with-id?]
-   (page-name->map original-page-name with-id? true))
-  ([original-page-name with-id? with-timestamp?]
-   (cond
-     (and original-page-name (string? original-page-name))
-     (let [original-page-name (util/remove-boundary-slashes original-page-name)
-           [original-page-name page-name journal-day] (convert-page-if-journal original-page-name)
-           namespace? (and (not (boolean (text/get-nested-page-name original-page-name)))
-                           (text/namespace-page? original-page-name))
-           page-entity (db/entity [:block/name page-name])
-           original-page-name (or (:block/original-name page-entity) original-page-name)]
-       (merge
-        {:block/name page-name
-         :block/original-name original-page-name}
-        (when with-id?
-          (if page-entity
-            {:block/uuid (:block/uuid page-entity)}
-            {:block/uuid (db/new-block-id)}))
-        (when namespace?
-          (let [namespace (first (gp-util/split-last "/" original-page-name))]
-            (when-not (string/blank? namespace)
-              {:block/namespace {:block/name (util/page-name-sanity-lc namespace)}})))
-        (when (and with-timestamp? (not page-entity)) ;; Only assign timestamp on creating new entity
-          (let [current-ms (util/time-ms)]
-            {:block/created-at current-ms
-             :block/updated-at current-ms}))
-        (if journal-day
-          {:block/journal? true
-           :block/journal-day journal-day}
-          {:block/journal? false})))
-
-     (and (map? original-page-name) (:block/uuid original-page-name))
-     original-page-name
-
-     (and (map? original-page-name) with-id?)
-     (assoc original-page-name :block/uuid (db/new-block-id))
-
-     :else
-     nil)))
-
-(defn with-page-refs
-  [{:keys [title body tags refs marker priority] :as block} with-id?]
-  (let [refs (->> (concat tags refs [marker priority])
-                  (remove string/blank?)
-                  (distinct))
-        refs (atom refs)]
-    (walk/prewalk
-     (fn [form]
-       ;; skip custom queries
-       (when-not (and (vector? form)
-                      (= (first form) "Custom")
-                      (= (second form) "query"))
-         (when-let [page (get-page-reference form)]
-           (swap! refs conj page))
-         (when-let [tag (get-tag form)]
-           (let [tag (text/page-ref-un-brackets! tag)]
-             (when (gp-util/tag-valid? tag)
-               (swap! refs conj tag))))
-         form))
-     (concat title body))
-    (let [refs (remove string/blank? @refs)
-          children-pages (->> (mapcat (fn [p]
-                                        (let [p (if (map? p)
-                                                  (:block/original-name p)
-                                                  p)]
-                                          (when (string? p)
-                                            (let [p (or (text/get-nested-page-name p) p)]
-                                              (when (text/namespace-page? p)
-                                                (util/split-namespace-pages p))))))
-                                      refs)
-                              (remove string/blank?)
-                              (distinct))
-          refs (->> (distinct (concat refs children-pages))
-                    (remove nil?))
-          refs (map (fn [ref] (page-name->map ref with-id?)) refs)]
-      (assoc block :refs refs))))
-
-(defn with-block-refs
-  [{:keys [title body] :as block}]
-  (let [ref-blocks (atom nil)]
-    (walk/postwalk
-     (fn [form]
-       (when-let [block (get-block-reference form)]
-         (swap! ref-blocks conj block))
-       form)
-     (concat title body))
-    (let [ref-blocks (->> @ref-blocks
-                          (filter gp-util/uuid-string?))
-          ref-blocks (map
-                       (fn [id]
-                         [:block/uuid (medley/uuid id)])
-                       ref-blocks)
-          refs (distinct (concat (:refs block) ref-blocks))]
-      (assoc block :refs refs))))
-
-(defn- block-keywordize
-  [block]
-  (medley/map-keys
-   (fn [k]
-     (if (namespace k)
-       k
-       (keyword "block" k)))
-   block))
-
-(defn- sanity-blocks-data
-  [blocks]
-  (map (fn [block]
-         (if (map? block)
-           (block-keywordize (gp-util/remove-nils block))
-           block))
-       blocks))
-
-(defn with-path-refs
-  [blocks]
-  (loop [blocks blocks
-         acc []
-         parents []]
-    (if (empty? blocks)
-      acc
-      (let [block (first blocks)
-            cur-level (:block/level block)
-            level-diff (- cur-level
-                          (get (last parents) :block/level 0))
-            [path-refs parents]
-            (cond
-              (zero? level-diff)            ; sibling
-              (let [path-refs (mapcat :block/refs (drop-last parents))
-                    parents (conj (vec (butlast parents)) block)]
-                [path-refs parents])
-
-              (> level-diff 0)              ; child
-              (let [path-refs (mapcat :block/refs parents)]
-                [path-refs (conj parents block)])
-
-              (< level-diff 0)              ; new parent
-              (let [parents (vec (take-while (fn [p] (< (:block/level p) cur-level)) parents))
-                    path-refs (mapcat :block/refs parents)]
-                [path-refs (conj parents block)]))
-            path-ref-pages (->> path-refs
-                                (concat (:block/refs block))
-                                (map (fn [ref]
-                                       (cond
-                                         (map? ref)
-                                         (:block/name ref)
-
-                                         :else
-                                         ref)))
-                                (remove string/blank?)
-                                (map (fn [ref]
-                                       (if (string? ref)
-                                         {:block/name (util/page-name-sanity-lc ref)}
-                                         ref)))
-                                (remove vector?)
-                                (remove nil?)
-                                (distinct))]
-        (recur (rest blocks)
-               (conj acc (assoc block :block/path-refs path-ref-pages))
-               parents)))))
-
-(defn block-tags->pages
-  [{:keys [tags] :as block}]
-  (if (seq tags)
-    (assoc block :tags (map (fn [tag]
-                              (let [tag (text/page-ref-un-brackets! tag)]
-                                [:block/name (util/page-name-sanity-lc tag)])) tags))
-    block))
-
-(defn- get-block-content
-  [utf8-content block format meta]
-  (let [content (if-let [end-pos (:end_pos meta)]
-                  (utf8/substring utf8-content
-                                  (:start_pos meta)
-                                  end-pos)
-                  (utf8/substring utf8-content
-                                  (:start_pos meta)))
-        content (when content
-                  (let [content (text/remove-level-spaces content format (config/get-block-pattern format))]
-                    (if (or (:pre-block? block)
-                            (= (:format block) :org))
-                      content
-                      (gp-mldoc/remove-indentation-spaces content (inc (:level block)) false))))]
-    (if (= format :org)
-      content
-      (property/->new-properties content))))
-
-(defn get-custom-id-or-new-id
-  [properties]
-  (or (when-let [custom-id (or (get-in properties [:properties :custom-id])
-                               (get-in properties [:properties :custom_id])
-                               (get-in properties [:properties :id]))]
-        (let [custom-id (and (string? custom-id) (string/trim custom-id))]
-          (when (and custom-id (gp-util/uuid-string? custom-id))
-            (uuid custom-id))))
-      (db/new-block-id)))
-
-(defn get-page-refs-from-properties
-  [properties]
-  (let [page-refs (mapcat (fn [v] (cond
-                                   (coll? v)
-                                   v
-
-                                   (text/page-ref? v)
-                                   [(text/page-ref-un-brackets! v)]
-
-                                   :else
-                                   nil)) (vals properties))
-        page-refs (remove string/blank? page-refs)]
-    (map (fn [page] (page-name->map page true)) page-refs)))
-
-(defn with-page-block-refs
-  [block with-id?]
-  (some-> block
-          (with-page-refs with-id?)
-          with-block-refs
-          block-tags->pages
-          (update :refs (fn [col] (remove nil? col)))))
-
-(defn with-pre-block-if-exists
-  [blocks body pre-block-properties encoded-content]
-  (let [first-block (first blocks)
-        first-block-start-pos (get-in first-block [:block/meta :start_pos])
-
-        ;; Add pre-block
-        blocks (if (or (> first-block-start-pos 0)
-                       (empty? blocks))
-                 (cons
-                  (merge
-                   (let [content (utf8/substring encoded-content 0 first-block-start-pos)
-                         {:keys [properties properties-order]} pre-block-properties
-                         id (get-custom-id-or-new-id {:properties properties})
-                         property-refs (->> (get-page-refs-from-properties properties)
-                                            (map :block/original-name))
-                         block {:uuid id
-                                :content content
-                                :level 1
-                                :properties properties
-                                :properties-order properties-order
-                                :refs property-refs
-                                :pre-block? true
-                                :unordered true
-                                :body body}
-                         block (with-page-block-refs block false)]
-                     (block-keywordize block))
-                   (select-keys first-block [:block/format :block/page]))
-                  blocks)
-                 blocks)]
-    (with-path-refs blocks)))
-
-(defn- construct-block
-  [block properties timestamps body encoded-content format pos-meta with-id?]
-  (let [id (get-custom-id-or-new-id properties)
-        ref-pages-in-properties (->> (:page-refs properties)
-                                     (remove string/blank?))
-        block (second block)
-        unordered? (:unordered block)
-        markdown-heading? (and (:size block) (= :markdown format))
-        block (if markdown-heading?
-                (assoc block
-                       :type :heading
-                       :level (if unordered? (:level block) 1)
-                       :heading-level (or (:size block) 6))
-                block)
-        block (cond->
-                (assoc block
-                       :uuid id
-                       :refs ref-pages-in-properties
-                       :format format
-                       :meta pos-meta)
-                (seq (:properties properties))
-                (assoc :properties (:properties properties))
-
-                (seq (:properties-order properties))
-                (assoc :properties-order (:properties-order properties)))
-        block (if (get-in block [:properties :collapsed])
-                (assoc block :collapsed? true)
-                block)
-        block (assoc block
-                     :content (get-block-content encoded-content block format pos-meta))
-        block (if (seq timestamps)
-                (merge block (timestamps->scheduled-and-deadline timestamps))
-                block)
-        block (assoc block :body body)
-        block (with-page-block-refs block with-id?)
-        {:keys [created-at updated-at]} (:properties properties)
-        block (cond-> block
-                (and created-at (integer? created-at))
-                (assoc :block/created-at created-at)
-
-                (and updated-at (integer? updated-at))
-                (assoc :block/updated-at updated-at))]
-    (dissoc block :title :body :anchor)))
-
-(defn extract-blocks
-  "Extract headings from mldoc ast.
-  Args:
-    `blocks`: mldoc ast.
-    `content`: markdown or org-mode text.
-    `with-id?`: If `with-id?` equals to true, all the referenced pages will have new db ids.
-    `format`: content's format, it could be either :markdown or :org-mode."
-  [blocks content with-id? format]
-  {:pre [(seq blocks) (string? content) (boolean? with-id?) (contains? #{:markdown :org} format)]}
-  (try
-    (let [encoded-content (utf8/encode content)
-          [blocks body pre-block-properties]
-          (loop [headings []
-                 blocks (reverse blocks)
-                 timestamps {}
-                 properties {}
-                 body []]
-            (if (seq blocks)
-              (let [[block pos-meta] (first blocks)
-                    ;; fix start_pos
-                    pos-meta (assoc pos-meta :end_pos
-                                    (if (seq headings)
-                                      (get-in (last headings) [:meta :start_pos])
-                                      nil))]
-                (cond
-                  (paragraph-timestamp-block? block)
-                  (let [timestamps (extract-timestamps block)
-                        timestamps' (merge timestamps timestamps)]
-                    (recur headings (rest blocks) timestamps' properties body))
-
-                  (property/properties-ast? block)
-                  (let [properties (extract-properties format (second block))]
-                    (recur headings (rest blocks) timestamps properties body))
-
-                  (heading-block? block)
-                  (let [block (construct-block block properties timestamps body encoded-content format pos-meta with-id?)]
-                    (recur (conj headings block) (rest blocks) {} {} []))
-
-                  :else
-                  (recur headings (rest blocks) timestamps properties (conj body block))))
-              [(-> (reverse headings)
-                   sanity-blocks-data)
-               body
-               properties]))
-          result (with-pre-block-if-exists blocks body pre-block-properties encoded-content)]
-      (map #(dissoc % :block/meta) result))
-    (catch js/Error e
-      (js/console.error "extract-blocks-failed")
-      (log/error :exception e))))
+            [logseq.graph-parser.mldoc :as gp-mldoc]))
 
 (defn with-parent-and-left
   [page-id blocks]
@@ -669,7 +87,7 @@
    (when-not (string/blank? content)
      (let [block (dissoc block :block/pre-block?)
            ast (format/to-edn content format nil)
-           blocks (extract-blocks ast content with-id? format)
+           blocks (gp-block/extract-blocks ast content with-id? format)
            new-block (first blocks)
            parent-refs (->> (db/get-block-parent (state/get-current-repo) uuid)
                             :block/path-refs
@@ -704,7 +122,7 @@
          result
          (let [ast (->> (format/to-edn content format (gp-mldoc/default-config format))
                         (map first))
-               title (when (heading-block? (first ast))
+               title (when (gp-block/heading-block? (first ast))
                        (:title (second (first ast))))
                body (vec (if title (rest ast) ast))
                body (drop-while property/properties-ast? body)

src/main/frontend/format/mldoc.cljs

@@ -1,4 +1,5 @@
 (ns frontend.format.mldoc
+  "Mldoc code needed by app but not graph-parser"
   (:require [clojure.string :as string]
             [frontend.format.protocol :as protocol]
             [frontend.state :as state]

src/main/frontend/handler/block.cljs

@@ -5,7 +5,7 @@
             [frontend.db.model :as db-model]
             [frontend.db.react :as react]
             [frontend.state :as state]
-            [frontend.format.block :as block]
+            [logseq.graph-parser.block :as gp-block]
             [frontend.util :as util]))
 
 
@@ -83,7 +83,7 @@
   [block typ]
   (walk-block block
               (fn [x]
-                (and (block/timestamp-block? x)
+                (and (gp-block/timestamp-block? x)
                      (= typ (first (second x)))))
               #(second (second %))))
 

src/main/frontend/handler/editor.cljs

@@ -54,6 +54,7 @@
             [frontend.util.keycode :as keycode]
             [logseq.graph-parser.util :as gp-util]
             [logseq.graph-parser.mldoc :as gp-mldoc]
+            [logseq.graph-parser.block :as gp-block]
             ["path" :as path]))
 
 ;; FIXME: should support multiple images concurrently uploading
@@ -265,7 +266,7 @@
   (if (and (:block/pre-block? block)
            (seq (:block/properties block)))
     (let [page-properties (:block/properties block)
-          str->page (fn [n] (block/page-name->map n true))
+          str->page (fn [n] (gp-block/page-name->map n true))
           refs (->> page-properties
                     (filter (fn [[_ v]] (coll? v)))
                     (vals)
@@ -669,7 +670,7 @@
 (defn properties-block
   [properties format page]
   (let [content (property/insert-properties format "" properties)
-        refs (block/get-page-refs-from-properties properties)]
+        refs (gp-block/get-page-refs-from-properties properties)]
     {:block/pre-block? true
      :block/uuid (db/new-block-id)
      :block/properties properties
@@ -1945,7 +1946,7 @@
                     content* (str (if (= :markdown format) "- " "* ")
                                   (property/insert-properties format content props))
                     ast (mldoc/->edn content* (gp-mldoc/default-config format))
-                    blocks (block/extract-blocks ast content* true format)
+                    blocks (gp-block/extract-blocks ast content* true format)
                     fst-block (first blocks)]
                 (assert fst-block "fst-block shouldn't be nil")
                 (assoc fst-block :block/level (:block/level block)))))))
@@ -2852,7 +2853,7 @@
   [format text]
   (when-let [editing-block (state/get-edit-block)]
     (let [page-id (:db/id (:block/page editing-block))
-          blocks (block/extract-blocks
+          blocks (gp-block/extract-blocks
                   (mldoc/->edn text (gp-mldoc/default-config format)) text true format)
           blocks' (block/with-parent-and-left page-id blocks)]
       (paste-blocks blocks' {}))))

src/main/frontend/handler/external.cljs

@@ -8,7 +8,7 @@
             [clojure.string :as string]
             [frontend.db :as db]
             [frontend.format.mldoc :as mldoc]
-            [frontend.format.block :as block]
+            [logseq.graph-parser.block :as gp-block]
             [frontend.handler.page :as page]
             [frontend.handler.editor :as editor]
             [frontend.util :as util]))
@@ -73,7 +73,7 @@
   (when-let [repo (state/get-current-repo)]
     (let [[headers parsed-blocks] (mldoc/opml->edn data)
           parsed-blocks (->>
-                         (block/extract-blocks parsed-blocks "" true :markdown)
+                         (gp-block/extract-blocks parsed-blocks "" true :markdown)
                          (mapv editor/wrap-parse-block))
           page-name (:title headers)]
       (when (not (db/page-exists? page-name))

src/main/frontend/handler/extract.cljs

@@ -13,6 +13,7 @@
             [frontend.util :as util]
             [logseq.graph-parser.util :as gp-util]
             [logseq.graph-parser.mldoc :as gp-mldoc]
+            [logseq.graph-parser.block :as gp-block]
             [frontend.util.property :as property]
             [lambdaisland.glogi :as log]))
 
@@ -22,7 +23,7 @@
   (let [ast (map first ast)]
     (if (string/includes? file "pages/contents.")
       "Contents"
-      (let [first-block (last (first (filter block/heading-block? ast)))
+      (let [first-block (last (first (filter gp-block/heading-block? ast)))
             property-name (when (and (contains? #{"Properties" "Property_Drawer"} (ffirst ast))
                                      (not (string/blank? (:title (last (first ast))))))
                             (:title (last (first ast))))
@@ -47,8 +48,8 @@
   [repo-url format ast properties file content]
   (try
     (let [page (get-page-name file ast)
-          [_original-page-name page-name _journal-day] (block/convert-page-if-journal page)
-          blocks (->> (block/extract-blocks ast content false format)
+          [_original-page-name page-name _journal-day] (gp-block/convert-page-if-journal page)
+          blocks (->> (gp-block/extract-blocks ast content false format)
                       (block/with-parent-and-left {:block/name page-name}))
           ref-pages (atom #{})
           ref-tags (atom #{})
@@ -94,7 +95,7 @@
                         (cond->
                           (gp-util/remove-nils
                            (assoc
-                            (block/page-name->map page false)
+                            (gp-block/page-name->map page false)
                             :block/file {:file/path (gp-util/path-normalize file)}))
                           (seq properties)
                           (assoc :block/properties properties)
@@ -114,7 +115,7 @@
                             (when (text/namespace-page? page)
                               (->> (util/split-namespace-pages page)
                                    (map (fn [page]
-                                          (-> (block/page-name->map page true)
+                                          (-> (gp-block/page-name->map page true)
                                               (assoc :block/format format)))))))
           pages (->> (concat
                       [page-entity]

src/main/frontend/handler/page.cljs

@@ -11,7 +11,6 @@
             [frontend.db.model :as model]
             [frontend.db.utils :as db-utils]
             [frontend.db.conn :as conn]
-            [frontend.format.block :as block]
             [frontend.fs :as fs]
             [frontend.handler.common :as common-handler]
             [frontend.handler.editor :as editor-handler]
@@ -35,6 +34,7 @@
             [frontend.mobile.util :as mobile-util]
             [logseq.graph-parser.util :as gp-util]
             [logseq.graph-parser.config :as gp-config]
+            [logseq.graph-parser.block :as gp-block]
             [goog.functions :refer [debounce]]))
 
 (defn- get-directory
@@ -72,7 +72,7 @@
    (let [p (common-handler/get-page-default-properties title)
          ps (merge p properties)
          content (page-property/insert-properties format "" ps)
-         refs (block/get-page-refs-from-properties properties)]
+         refs (gp-block/get-page-refs-from-properties properties)]
      {:block/uuid (db/new-block-id)
       :block/properties ps
       :block/properties-order (keys ps)
@@ -127,7 +127,7 @@
                         [title])
              format   (or format (state/get-preferred-format))
              pages    (map (fn [page]
-                             (-> (block/page-name->map page true)
+                             (-> (gp-block/page-name->map page true)
                                  (assoc :block/format format)))
                         pages)
              txs      (->> pages

src/main/logseq/graph_parser/block.cljs

@@ -0,0 +1,584 @@
+(ns logseq.graph-parser.block
+  "Block related code needed for graph-parser"
+  (:require [clojure.string :as string]
+            [clojure.walk :as walk]
+            [frontend.config :as config]
+            [frontend.date :as date]
+            [frontend.db :as db]
+            [frontend.state :as state]
+            [logseq.graph-parser.text :as text]
+            [logseq.graph-parser.utf8 :as utf8]
+            [frontend.util :as util]
+            [frontend.util.property :as property]
+            [logseq.graph-parser.util :as gp-util]
+            [logseq.graph-parser.config :as gp-config]
+            [logseq.graph-parser.mldoc :as gp-mldoc]
+            [lambdaisland.glogi :as log]
+            [medley.core :as medley]))
+
+(defn heading-block?
+  [block]
+  (and
+   (vector? block)
+   (= "Heading" (first block))))
+
+(defn get-tag
+  [block]
+  (when-let [tag-value (and (vector? block)
+                            (= "Tag" (first block))
+                            (second block))]
+    (->> tag-value
+         (map (fn [[elem value]]
+                (case elem
+                  "Plain" value
+                  "Link" (:full_text value)
+                  "Nested_link" (:content value)
+                  "")))
+         (string/join))))
+
+(defn get-page-reference
+  [block]
+  (let [page (cond
+               (and (vector? block) (= "Link" (first block)))
+               (let [typ (first (:url (second block)))
+                     value (second (:url (second block)))]
+                 ;; {:url ["File" "file:../pages/hello_world.org"], :label [["Plain" "hello world"]], :title nil}
+                 (or
+                  (and
+                   (= typ "Page_ref")
+                   (and (string? value)
+                        (not (or (gp-config/local-asset? value)
+                                 (gp-config/draw? value))))
+                   value)
+
+                  (and
+                   (= typ "Search")
+                   (text/page-ref? value)
+                   (text/page-ref-un-brackets! value))
+
+                  (and
+                   (= typ "Search")
+                   (not (contains? #{\# \* \/ \[} (first value)))
+                   (let [ext (some-> (util/get-file-ext value) keyword)]
+                     (when (and (not (util/starts-with? value "http:"))
+                                (not (util/starts-with? value "https:"))
+                                (not (util/starts-with? value "file:"))
+                                (not (gp-config/local-asset? value))
+                                (or (= ext :excalidraw)
+                                    (not (contains? (config/supported-formats) ext))))
+                       value)))
+
+                  (and
+                   (= typ "Complex")
+                   (= (:protocol value) "file")
+                   (:link value))
+
+                  (and
+                   (= typ "File")
+                   (second (first (:label (second block)))))))
+
+               (and (vector? block) (= "Nested_link" (first block)))
+               (let [content (:content (last block))]
+                 (subs content 2 (- (count content) 2)))
+
+               (and (vector? block)
+                    (= "Macro" (first block)))
+               (let [{:keys [name arguments]} (second block)
+                     argument (string/join ", " arguments)]
+                   (when (= name "embed")
+                     (text/page-ref-un-brackets! argument)))
+
+               (and (vector? block)
+                    (= "Tag" (first block)))
+               (let [text (get-tag block)]
+                 (text/page-ref-un-brackets! text))
+
+               :else
+               nil)]
+    (text/block-ref-un-brackets! page)))
+
+(defn get-block-reference
+  [block]
+  (when-let [block-id (cond
+                        (and (vector? block)
+                             (= "Block_reference" (first block)))
+                        (last block)
+
+                        (and (vector? block)
+                             (= "Link" (first block))
+                             (map? (second block))
+                             (= "Block_ref" (first (:url (second block)))))
+                        (second (:url (second block)))
+
+                        (and (vector? block)
+                             (= "Macro" (first block)))
+                        (let [{:keys [name arguments]} (second block)]
+                          (when (and (= name "embed")
+                                     (string? (first arguments))
+                                     (string/starts-with? (first arguments) "((")
+                                     (string/ends-with? (first arguments) "))"))
+                            (subs (first arguments) 2 (- (count (first arguments)) 2))))
+
+                        (and (vector? block)
+                             (= "Link" (first block))
+                             (map? (second block)))
+                        (if (= "id" (:protocol (second (:url (second block)))))
+                          (:link (second (:url (second block))))
+                          (let [id (second (:url (second block)))]
+                            (text/block-ref-un-brackets! id)))
+
+                        :else
+                        nil)]
+    (when (and block-id
+               (gp-util/uuid-string? block-id))
+      block-id)))
+
+(defn paragraph-block?
+  [block]
+  (and
+   (vector? block)
+   (= "Paragraph" (first block))))
+
+(defn timestamp-block?
+  [block]
+  (and
+   (vector? block)
+   (= "Timestamp" (first block))))
+
+;; TODO: we should move this to mldoc
+(defn extract-properties
+  [format properties]
+  (when (seq properties)
+    (let [properties (seq properties)
+          page-refs (->>
+                     properties
+                     (remove (fn [[k _]]
+                               (contains? #{:background-color :background_color} (keyword k))))
+                     (map last)
+                     (map (fn [v]
+                            (when (and (string? v)
+                                       (not (gp-mldoc/link? format v)))
+                              (let [v (string/trim v)
+                                    result (text/split-page-refs-without-brackets v {:un-brackets? false})]
+                                (if (coll? result)
+                                  (map text/page-ref-un-brackets! result)
+                                  [])))))
+                     (apply concat)
+                     (remove string/blank?))
+          properties (->> properties
+                          (map (fn [[k v]]
+                                 (let [k (-> (string/lower-case (name k))
+                                             (string/replace " " "-")
+                                             (string/replace "_" "-"))
+                                       k (if (contains? #{"custom_id" "custom-id"} k)
+                                           "id"
+                                           k)
+                                       v (if (coll? v)
+                                           (remove string/blank? v)
+                                           (if (string/blank? v)
+                                             nil
+                                             (text/parse-property format k v (state/get-config))))
+                                       k (keyword k)
+                                       v (if (and
+                                              (string? v)
+                                              (contains? #{:alias :aliases :tags} k))
+                                           (set [v])
+                                           v)
+                                       v (if (coll? v) (set v) v)]
+                                   [k v])))
+                          (remove #(nil? (second %))))]
+      {:properties (into {} properties)
+       :properties-order (map first properties)
+       :page-refs page-refs})))
+
+(defn- paragraph-timestamp-block?
+  [block]
+  (and (paragraph-block? block)
+       (or (timestamp-block? (first (second block)))
+           (timestamp-block? (second (second block))))))
+
+(defn extract-timestamps
+  [block]
+  (some->>
+   (second block)
+   (filter timestamp-block?)
+   (map last)
+   (into {})))
+
+;; {"Deadline" {:date {:year 2020, :month 10, :day 20}, :wday "Tue", :time {:hour 8, :min 0}, :repetition [["DoublePlus"] ["Day"] 1], :active true}}
+(defn timestamps->scheduled-and-deadline
+  [timestamps]
+  (let [timestamps (medley/map-keys (comp keyword string/lower-case) timestamps)
+        m (some->> (select-keys timestamps [:scheduled :deadline])
+                   (map (fn [[k v]]
+                          (let [{:keys [date repetition]} v
+                                {:keys [year month day]} date
+                                day (js/parseInt (str year (util/zero-pad month) (util/zero-pad day)))]
+                            (cond->
+                             (case k
+                               :scheduled
+                               {:scheduled day}
+                               :deadline
+                               {:deadline day})
+                              repetition
+                              (assoc :repeated? true))))))]
+    (apply merge m)))
+
+(defn convert-page-if-journal
+  "Convert journal file name to user' custom date format"
+  [original-page-name]
+  (when original-page-name
+    (let [page-name (util/page-name-sanity-lc original-page-name)
+          day (date/journal-title->int page-name)]
+     (if day
+       (let [original-page-name (date/int->journal-title day)]
+         [original-page-name (util/page-name-sanity-lc original-page-name) day])
+       [original-page-name page-name day]))))
+
+(defn page-name->map
+  "Create a page's map structure given a original page name (string).
+   map as input is supported for legacy compatibility.
+   with-timestamp?: assign timestampes to the map structure.
+    Useful when creating new pages from references or namespaces,
+    as there's no chance to introduce timestamps via editing in page"
+  ([original-page-name with-id?]
+   (page-name->map original-page-name with-id? true))
+  ([original-page-name with-id? with-timestamp?]
+   (cond
+     (and original-page-name (string? original-page-name))
+     (let [original-page-name (util/remove-boundary-slashes original-page-name)
+           [original-page-name page-name journal-day] (convert-page-if-journal original-page-name)
+           namespace? (and (not (boolean (text/get-nested-page-name original-page-name)))
+                           (text/namespace-page? original-page-name))
+           page-entity (db/entity [:block/name page-name])
+           original-page-name (or (:block/original-name page-entity) original-page-name)]
+       (merge
+        {:block/name page-name
+         :block/original-name original-page-name}
+        (when with-id?
+          (if page-entity
+            {:block/uuid (:block/uuid page-entity)}
+            {:block/uuid (db/new-block-id)}))
+        (when namespace?
+          (let [namespace (first (gp-util/split-last "/" original-page-name))]
+            (when-not (string/blank? namespace)
+              {:block/namespace {:block/name (util/page-name-sanity-lc namespace)}})))
+        (when (and with-timestamp? (not page-entity)) ;; Only assign timestamp on creating new entity
+          (let [current-ms (util/time-ms)]
+            {:block/created-at current-ms
+             :block/updated-at current-ms}))
+        (if journal-day
+          {:block/journal? true
+           :block/journal-day journal-day}
+          {:block/journal? false})))
+
+     (and (map? original-page-name) (:block/uuid original-page-name))
+     original-page-name
+
+     (and (map? original-page-name) with-id?)
+     (assoc original-page-name :block/uuid (db/new-block-id))
+
+     :else
+     nil)))
+
+(defn with-page-refs
+  [{:keys [title body tags refs marker priority] :as block} with-id?]
+  (let [refs (->> (concat tags refs [marker priority])
+                  (remove string/blank?)
+                  (distinct))
+        refs (atom refs)]
+    (walk/prewalk
+     (fn [form]
+       ;; skip custom queries
+       (when-not (and (vector? form)
+                      (= (first form) "Custom")
+                      (= (second form) "query"))
+         (when-let [page (get-page-reference form)]
+           (swap! refs conj page))
+         (when-let [tag (get-tag form)]
+           (let [tag (text/page-ref-un-brackets! tag)]
+             (when (gp-util/tag-valid? tag)
+               (swap! refs conj tag))))
+         form))
+     (concat title body))
+    (let [refs (remove string/blank? @refs)
+          children-pages (->> (mapcat (fn [p]
+                                        (let [p (if (map? p)
+                                                  (:block/original-name p)
+                                                  p)]
+                                          (when (string? p)
+                                            (let [p (or (text/get-nested-page-name p) p)]
+                                              (when (text/namespace-page? p)
+                                                (util/split-namespace-pages p))))))
+                                      refs)
+                              (remove string/blank?)
+                              (distinct))
+          refs (->> (distinct (concat refs children-pages))
+                    (remove nil?))
+          refs (map (fn [ref] (page-name->map ref with-id?)) refs)]
+      (assoc block :refs refs))))
+
+(defn with-block-refs
+  [{:keys [title body] :as block}]
+  (let [ref-blocks (atom nil)]
+    (walk/postwalk
+     (fn [form]
+       (when-let [block (get-block-reference form)]
+         (swap! ref-blocks conj block))
+       form)
+     (concat title body))
+    (let [ref-blocks (->> @ref-blocks
+                          (filter gp-util/uuid-string?))
+          ref-blocks (map
+                       (fn [id]
+                         [:block/uuid (medley/uuid id)])
+                       ref-blocks)
+          refs (distinct (concat (:refs block) ref-blocks))]
+      (assoc block :refs refs))))
+
+(defn- block-keywordize
+  [block]
+  (medley/map-keys
+   (fn [k]
+     (if (namespace k)
+       k
+       (keyword "block" k)))
+   block))
+
+(defn- sanity-blocks-data
+  [blocks]
+  (map (fn [block]
+         (if (map? block)
+           (block-keywordize (gp-util/remove-nils block))
+           block))
+       blocks))
+
+(defn with-path-refs
+  [blocks]
+  (loop [blocks blocks
+         acc []
+         parents []]
+    (if (empty? blocks)
+      acc
+      (let [block (first blocks)
+            cur-level (:block/level block)
+            level-diff (- cur-level
+                          (get (last parents) :block/level 0))
+            [path-refs parents]
+            (cond
+              (zero? level-diff)            ; sibling
+              (let [path-refs (mapcat :block/refs (drop-last parents))
+                    parents (conj (vec (butlast parents)) block)]
+                [path-refs parents])
+
+              (> level-diff 0)              ; child
+              (let [path-refs (mapcat :block/refs parents)]
+                [path-refs (conj parents block)])
+
+              (< level-diff 0)              ; new parent
+              (let [parents (vec (take-while (fn [p] (< (:block/level p) cur-level)) parents))
+                    path-refs (mapcat :block/refs parents)]
+                [path-refs (conj parents block)]))
+            path-ref-pages (->> path-refs
+                                (concat (:block/refs block))
+                                (map (fn [ref]
+                                       (cond
+                                         (map? ref)
+                                         (:block/name ref)
+
+                                         :else
+                                         ref)))
+                                (remove string/blank?)
+                                (map (fn [ref]
+                                       (if (string? ref)
+                                         {:block/name (util/page-name-sanity-lc ref)}
+                                         ref)))
+                                (remove vector?)
+                                (remove nil?)
+                                (distinct))]
+        (recur (rest blocks)
+               (conj acc (assoc block :block/path-refs path-ref-pages))
+               parents)))))
+
+(defn block-tags->pages
+  [{:keys [tags] :as block}]
+  (if (seq tags)
+    (assoc block :tags (map (fn [tag]
+                              (let [tag (text/page-ref-un-brackets! tag)]
+                                [:block/name (util/page-name-sanity-lc tag)])) tags))
+    block))
+
+(defn- get-block-content
+  [utf8-content block format meta]
+  (let [content (if-let [end-pos (:end_pos meta)]
+                  (utf8/substring utf8-content
+                                  (:start_pos meta)
+                                  end-pos)
+                  (utf8/substring utf8-content
+                                  (:start_pos meta)))
+        content (when content
+                  (let [content (text/remove-level-spaces content format (config/get-block-pattern format))]
+                    (if (or (:pre-block? block)
+                            (= (:format block) :org))
+                      content
+                      (gp-mldoc/remove-indentation-spaces content (inc (:level block)) false))))]
+    (if (= format :org)
+      content
+      (property/->new-properties content))))
+
+(defn get-custom-id-or-new-id
+  [properties]
+  (or (when-let [custom-id (or (get-in properties [:properties :custom-id])
+                               (get-in properties [:properties :custom_id])
+                               (get-in properties [:properties :id]))]
+        (let [custom-id (and (string? custom-id) (string/trim custom-id))]
+          (when (and custom-id (gp-util/uuid-string? custom-id))
+            (uuid custom-id))))
+      (db/new-block-id)))
+
+(defn get-page-refs-from-properties
+  [properties]
+  (let [page-refs (mapcat (fn [v] (cond
+                                   (coll? v)
+                                   v
+
+                                   (text/page-ref? v)
+                                   [(text/page-ref-un-brackets! v)]
+
+                                   :else
+                                   nil)) (vals properties))
+        page-refs (remove string/blank? page-refs)]
+    (map (fn [page] (page-name->map page true)) page-refs)))
+
+(defn with-page-block-refs
+  [block with-id?]
+  (some-> block
+          (with-page-refs with-id?)
+          with-block-refs
+          block-tags->pages
+          (update :refs (fn [col] (remove nil? col)))))
+
+(defn with-pre-block-if-exists
+  [blocks body pre-block-properties encoded-content]
+  (let [first-block (first blocks)
+        first-block-start-pos (get-in first-block [:block/meta :start_pos])
+
+        ;; Add pre-block
+        blocks (if (or (> first-block-start-pos 0)
+                       (empty? blocks))
+                 (cons
+                  (merge
+                   (let [content (utf8/substring encoded-content 0 first-block-start-pos)
+                         {:keys [properties properties-order]} pre-block-properties
+                         id (get-custom-id-or-new-id {:properties properties})
+                         property-refs (->> (get-page-refs-from-properties properties)
+                                            (map :block/original-name))
+                         block {:uuid id
+                                :content content
+                                :level 1
+                                :properties properties
+                                :properties-order properties-order
+                                :refs property-refs
+                                :pre-block? true
+                                :unordered true
+                                :body body}
+                         block (with-page-block-refs block false)]
+                     (block-keywordize block))
+                   (select-keys first-block [:block/format :block/page]))
+                  blocks)
+                 blocks)]
+    (with-path-refs blocks)))
+
+(defn- construct-block
+  [block properties timestamps body encoded-content format pos-meta with-id?]
+  (let [id (get-custom-id-or-new-id properties)
+        ref-pages-in-properties (->> (:page-refs properties)
+                                     (remove string/blank?))
+        block (second block)
+        unordered? (:unordered block)
+        markdown-heading? (and (:size block) (= :markdown format))
+        block (if markdown-heading?
+                (assoc block
+                       :type :heading
+                       :level (if unordered? (:level block) 1)
+                       :heading-level (or (:size block) 6))
+                block)
+        block (cond->
+                (assoc block
+                       :uuid id
+                       :refs ref-pages-in-properties
+                       :format format
+                       :meta pos-meta)
+                (seq (:properties properties))
+                (assoc :properties (:properties properties))
+
+                (seq (:properties-order properties))
+                (assoc :properties-order (:properties-order properties)))
+        block (if (get-in block [:properties :collapsed])
+                (assoc block :collapsed? true)
+                block)
+        block (assoc block
+                     :content (get-block-content encoded-content block format pos-meta))
+        block (if (seq timestamps)
+                (merge block (timestamps->scheduled-and-deadline timestamps))
+                block)
+        block (assoc block :body body)
+        block (with-page-block-refs block with-id?)
+        {:keys [created-at updated-at]} (:properties properties)
+        block (cond-> block
+                (and created-at (integer? created-at))
+                (assoc :block/created-at created-at)
+
+                (and updated-at (integer? updated-at))
+                (assoc :block/updated-at updated-at))]
+    (dissoc block :title :body :anchor)))
+
+(defn extract-blocks
+  "Extract headings from mldoc ast.
+  Args:
+    `blocks`: mldoc ast.
+    `content`: markdown or org-mode text.
+    `with-id?`: If `with-id?` equals to true, all the referenced pages will have new db ids.
+    `format`: content's format, it could be either :markdown or :org-mode."
+  [blocks content with-id? format]
+  {:pre [(seq blocks) (string? content) (boolean? with-id?) (contains? #{:markdown :org} format)]}
+  (try
+    (let [encoded-content (utf8/encode content)
+          [blocks body pre-block-properties]
+          (loop [headings []
+                 blocks (reverse blocks)
+                 timestamps {}
+                 properties {}
+                 body []]
+            (if (seq blocks)
+              (let [[block pos-meta] (first blocks)
+                    ;; fix start_pos
+                    pos-meta (assoc pos-meta :end_pos
+                                    (if (seq headings)
+                                      (get-in (last headings) [:meta :start_pos])
+                                      nil))]
+                (cond
+                  (paragraph-timestamp-block? block)
+                  (let [timestamps (extract-timestamps block)
+                        timestamps' (merge timestamps timestamps)]
+                    (recur headings (rest blocks) timestamps' properties body))
+
+                  (property/properties-ast? block)
+                  (let [properties (extract-properties format (second block))]
+                    (recur headings (rest blocks) timestamps properties body))
+
+                  (heading-block? block)
+                  (let [block (construct-block block properties timestamps body encoded-content format pos-meta with-id?)]
+                    (recur (conj headings block) (rest blocks) {} {} []))
+
+                  :else
+                  (recur headings (rest blocks) timestamps properties (conj body block))))
+              [(-> (reverse headings)
+                   sanity-blocks-data)
+               body
+               properties]))
+          result (with-pre-block-if-exists blocks body pre-block-properties encoded-content)]
+      (map #(dissoc % :block/meta) result))
+    (catch js/Error e
+      (js/console.error "extract-blocks-failed")
+      (log/error :exception e))))

src/test/frontend/format/block_test.cljs → src/test/logseq/graph_parser/block_test.cljs

@@ -1,9 +1,9 @@
-(ns frontend.format.block-test
-  (:require [frontend.format.block :as block]
+(ns logseq.graph-parser.block-test
+  (:require [logseq.graph-parser.block :as gp-block]
             [cljs.test :refer [deftest are]]))
 
 (deftest test-extract-properties
-  (are [x y] (= (:properties (block/extract-properties :markdown x)) y)
+  (are [x y] (= (:properties (gp-block/extract-properties :markdown x)) y)
     [["year" "1000"]] {:year 1000}
     [["year" "\"1000\""]] {:year "\"1000\""}
     [["background-color" "#000000"]] {:background-color "#000000"}
@@ -23,7 +23,7 @@
     [["foo" "bar, [[baz, test]]"]] {:foo #{"bar" "baz, test"}}
     [["foo" "bar, [[baz, test, [[nested]]]]"]] {:foo #{"bar" "baz, test, [[nested]]"}})
 
-  (are [x y] (= (vec (:page-refs (block/extract-properties :markdown x))) y)
+  (are [x y] (= (vec (:page-refs (gp-block/extract-properties :markdown x))) y)
     [["year" "1000"]] []
     [["year" "\"1000\""]] []
     [["foo" "[[bar]] test"]] ["bar" "test"]