Finish migrating extract to nbb

.clj-kondo/config.edn

@@ -23,6 +23,7 @@
              frontend.config config
              frontend.format.mldoc mldoc
              frontend.format.block block
+             frontend.handler.extract extract
              logseq.graph-parser.text text
              logseq.graph-parser.block gp-block
              logseq.graph-parser.mldoc gp-mldoc

src/main/frontend/format/block.cljs

@@ -26,76 +26,6 @@
   ([original-page-name with-id? with-timestamp?]
    (gp-block/page-name->map original-page-name with-id? (db/get-db (state/get-current-repo)) with-timestamp? (state/get-date-formatter))))
 
-(defn with-parent-and-left
-  [page-id blocks]
-  (loop [blocks (map (fn [block] (assoc block :block/level-spaces (:block/level block))) blocks)
-         parents [{:page/id page-id     ; db id or a map {:block/name "xxx"}
-                   :block/level 0
-                   :block/level-spaces 0}]
-         result []]
-    (if (empty? blocks)
-      (map #(dissoc % :block/level-spaces) result)
-      (let [[block & others] blocks
-            level-spaces (:block/level-spaces block)
-            {:block/keys [uuid level parent] :as last-parent} (last parents)
-            parent-spaces (:block/level-spaces last-parent)
-            [blocks parents result]
-            (cond
-              (= level-spaces parent-spaces)        ; sibling
-              (let [block (assoc block
-                                 :block/parent parent
-                                 :block/left [:block/uuid uuid]
-                                 :block/level level)
-                    parents' (conj (vec (butlast parents)) block)
-                    result' (conj result block)]
-                [others parents' result'])
-
-              (> level-spaces parent-spaces)         ; child
-              (let [parent (if uuid [:block/uuid uuid] (:page/id last-parent))
-                    block (cond->
-                            (assoc block
-                                  :block/parent parent
-                                  :block/left parent)
-                            ;; fix block levels with wrong order
-                            ;; For example:
-                            ;;   - a
-                            ;; - b
-                            ;; What if the input indentation is two spaces instead of 4 spaces
-                            (>= (- level-spaces parent-spaces) 1)
-                            (assoc :block/level (inc level)))
-                    parents' (conj parents block)
-                    result' (conj result block)]
-                [others parents' result'])
-
-              (< level-spaces parent-spaces)
-              (cond
-                (some #(= (:block/level-spaces %) (:block/level-spaces block)) parents) ; outdent
-                (let [parents' (vec (filter (fn [p] (<= (:block/level-spaces p) level-spaces)) parents))
-                      left (last parents')
-                      blocks (cons (assoc (first blocks)
-                                          :block/level (dec level)
-                                          :block/left [:block/uuid (:block/uuid left)])
-                                   (rest blocks))]
-                  [blocks parents' result])
-
-                :else
-                (let [[f r] (split-with (fn [p] (<= (:block/level-spaces p) level-spaces)) parents)
-                      left (first r)
-                      parent-id (if-let [block-id (:block/uuid (last f))]
-                                  [:block/uuid block-id]
-                                  page-id)
-                      block (cond->
-                              (assoc block
-                                     :block/parent parent-id
-                                     :block/left [:block/uuid (:block/uuid left)]
-                                     :block/level (:block/level left)
-                                     :block/level-spaces (:block/level-spaces left)))
-
-                      parents' (->> (concat f [block]) vec)
-                      result' (conj result block)]
-                  [others parents' result'])))]
-        (recur blocks parents result)))))
-
 (defn parse-block
   ([block]
    (parse-block block nil))

src/main/frontend/fs/watcher_handler.cljs

@@ -4,7 +4,7 @@
             [frontend.db :as db]
             [frontend.db.model :as model]
             [frontend.handler.editor :as editor]
-            [frontend.handler.extract :as extract]
+            [logseq.graph-parser.extract :as extract]
             [frontend.handler.file :as file-handler]
             [frontend.handler.page :as page-handler]
             [frontend.handler.repo :as repo-handler]

src/main/frontend/handler/editor.cljs

@@ -1960,7 +1960,7 @@
   (let [blocks (block-tree->blocks tree-vec format)
         target-block (db/pull target-block-id)
         page-id (:db/id (:block/page target-block))
-        blocks (block/with-parent-and-left page-id blocks)]
+        blocks (gp-block/with-parent-and-left page-id blocks)]
     (paste-blocks
      blocks
      {:target-block target-block
@@ -2857,7 +2857,7 @@
     (let [page-id (:db/id (:block/page editing-block))
           blocks (block/extract-blocks
                   (mldoc/->edn text (gp-mldoc/default-config format)) text true format)
-          blocks' (block/with-parent-and-left page-id blocks)]
+          blocks' (gp-block/with-parent-and-left page-id blocks)]
       (paste-blocks blocks' {}))))
 
 (defn- paste-segmented-text

src/main/frontend/handler/file.cljs

@@ -11,7 +11,7 @@
             [frontend.fs :as fs]
             [frontend.fs.nfs :as nfs]
             [frontend.handler.common :as common-handler]
-            [frontend.handler.extract :as extract-handler]
+            [logseq.graph-parser.extract :as extract]
             [frontend.handler.ui :as ui-handler]
             [frontend.state :as state]
             [frontend.util :as util]
@@ -123,11 +123,15 @@
            file-content [{:file/path file}]
            tx (if (contains? gp-config/mldoc-support-formats format)
                 (let [[pages blocks]
-                      (extract-handler/extract-blocks-pages
-                       repo-url file content
+                      (extract/extract-blocks-pages
+                       file
+                       content
                        {:user-config (state/get-config)
                         :date-formatter (state/get-date-formatter)
-                        :page-name-order (state/page-name-order)})
+                        :page-name-order (state/page-name-order)
+                        :block-pattern (config/get-block-pattern format)
+                        :supported-formats (config/supported-formats)
+                        :db (db/get-db (state/get-current-repo))})
                       first-page (first pages)
                       delete-blocks (->
                                      (concat
@@ -149,7 +153,7 @@
                                           (seq))
                       ;; To prevent "unique constraint" on datascript
                       block-ids (set/union (set block-ids) (set block-refs-ids))
-                      pages (extract-handler/with-ref-pages pages blocks)
+                      pages (extract/with-ref-pages pages blocks)
                       pages-index (map #(select-keys % [:block/name]) pages)]
                   ;; does order matter?
                   (concat file-content pages-index delete-blocks pages block-ids blocks))

src/main/logseq/graph_parser/block.cljc

@@ -581,3 +581,73 @@
       (map #(dissoc % :block/meta) result))
     (catch :default e
       (log/error :extract-blocks-failure e))))
+
+(defn with-parent-and-left
+  [page-id blocks]
+  (loop [blocks (map (fn [block] (assoc block :block/level-spaces (:block/level block))) blocks)
+         parents [{:page/id page-id     ; db id or a map {:block/name "xxx"}
+                   :block/level 0
+                   :block/level-spaces 0}]
+         result []]
+    (if (empty? blocks)
+      (map #(dissoc % :block/level-spaces) result)
+      (let [[block & others] blocks
+            level-spaces (:block/level-spaces block)
+            {:block/keys [uuid level parent] :as last-parent} (last parents)
+            parent-spaces (:block/level-spaces last-parent)
+            [blocks parents result]
+            (cond
+              (= level-spaces parent-spaces)        ; sibling
+              (let [block (assoc block
+                                 :block/parent parent
+                                 :block/left [:block/uuid uuid]
+                                 :block/level level)
+                    parents' (conj (vec (butlast parents)) block)
+                    result' (conj result block)]
+                [others parents' result'])
+
+              (> level-spaces parent-spaces)         ; child
+              (let [parent (if uuid [:block/uuid uuid] (:page/id last-parent))
+                    block (cond->
+                            (assoc block
+                                  :block/parent parent
+                                  :block/left parent)
+                            ;; fix block levels with wrong order
+                            ;; For example:
+                            ;;   - a
+                            ;; - b
+                            ;; What if the input indentation is two spaces instead of 4 spaces
+                            (>= (- level-spaces parent-spaces) 1)
+                            (assoc :block/level (inc level)))
+                    parents' (conj parents block)
+                    result' (conj result block)]
+                [others parents' result'])
+
+              (< level-spaces parent-spaces)
+              (cond
+                (some #(= (:block/level-spaces %) (:block/level-spaces block)) parents) ; outdent
+                (let [parents' (vec (filter (fn [p] (<= (:block/level-spaces p) level-spaces)) parents))
+                      left (last parents')
+                      blocks (cons (assoc (first blocks)
+                                          :block/level (dec level)
+                                          :block/left [:block/uuid (:block/uuid left)])
+                                   (rest blocks))]
+                  [blocks parents' result])
+
+                :else
+                (let [[f r] (split-with (fn [p] (<= (:block/level-spaces p) level-spaces)) parents)
+                      left (first r)
+                      parent-id (if-let [block-id (:block/uuid (last f))]
+                                  [:block/uuid block-id]
+                                  page-id)
+                      block (cond->
+                              (assoc block
+                                     :block/parent parent-id
+                                     :block/left [:block/uuid (:block/uuid left)]
+                                     :block/level (:block/level left)
+                                     :block/level-spaces (:block/level-spaces left)))
+
+                      parents' (->> (concat f [block]) vec)
+                      result' (conj result block)]
+                  [others parents' result'])))]
+        (recur blocks parents result)))))

src/main/frontend/handler/extract.cljs → src/main/logseq/graph_parser/extract.cljc

@@ -1,17 +1,19 @@
-(ns frontend.handler.extract
-  "Extract helper."
+(ns ^:nbb-compatible logseq.graph-parser.extract
+  ;; Disable clj linters since we don't support clj
+  #?(:clj {:clj-kondo/config {:linters {:unresolved-namespace {:level :off}
+                                        :unresolved-symbol {:level :off}}}})
   (:require [clojure.set :as set]
             [clojure.string :as string]
             [clojure.walk :as walk]
             [datascript.core :as d]
-            [frontend.format.block :as block]
             [logseq.graph-parser.text :as text]
             [logseq.graph-parser.util :as gp-util]
             [logseq.graph-parser.mldoc :as gp-mldoc]
             [logseq.graph-parser.block :as gp-block]
             [logseq.graph-parser.property :as gp-property]
             [logseq.graph-parser.config :as gp-config]
-            [lambdaisland.glogi :as log]))
+            #?(:org.babashka/nbb [logseq.graph-parser.log :as log]
+               :default [lambdaisland.glogi :as log])))
 
 (defn- get-page-name
   [file ast page-name-order]
@@ -40,13 +42,13 @@
 
 ;; TODO: performance improvement
 (defn- extract-pages-and-blocks
-  #_:clj-kondo/ignore
-  [repo-url format ast properties file content date-formatter page-name-order]
+  [format ast properties file content {:keys [date-formatter page-name-order db] :as options}]
   (try
+    #_:clj-kondo/ignore ;;clj-kondo bug
     (let [page (get-page-name file ast page-name-order)
           [_original-page-name page-name _journal-day] (gp-block/convert-page-if-journal page date-formatter)
-          blocks (->> (block/extract-blocks ast content false format)
-                      (block/with-parent-and-left {:block/name page-name}))
+          blocks (->> (gp-block/extract-blocks ast content false format (dissoc options :page-name-order))
+                      (gp-block/with-parent-and-left {:block/name page-name}))
           ref-pages (atom #{})
           ref-tags (atom #{})
           blocks (map (fn [block]
@@ -91,7 +93,7 @@
                         (cond->
                          (gp-util/remove-nils
                           (assoc
-                           (block/page-name->map page false)
+                           (gp-block/page-name->map page false db true date-formatter)
                            :block/file {:file/path (gp-util/path-normalize file)}))
                          (seq properties)
                          (assoc :block/properties properties)
@@ -111,7 +113,7 @@
                             (when (text/namespace-page? page)
                               (->> (gp-util/split-namespace-pages page)
                                    (map (fn [page]
-                                          (-> (block/page-name->map page true)
+                                          (-> (gp-block/page-name->map page true db true date-formatter)
                                               (assoc :block/format format)))))))
           pages (->> (concat
                       [page-entity]
@@ -131,11 +133,11 @@
           blocks (->> (remove nil? blocks)
                       (map (fn [b] (dissoc b :block/title :block/body :block/level :block/children :block/meta :block/anchor))))]
       [pages blocks])
-    (catch js/Error e
+    (catch :default e
       (log/error :exception e))))
 
 (defn extract-blocks-pages
-  [repo-url file content {:keys [user-config date-formatter page-name-order]}]
+  [file content {:keys [user-config] :as options}]
   (if (string/blank? content)
     []
     (let [format (gp-util/get-format file)
@@ -161,11 +163,10 @@
                                        (string/replace (or v "") "\\" "")))
                              properties)))]
         (extract-pages-and-blocks
-         repo-url
          format ast properties
-         file content date-formatter page-name-order)))))
+         file content options)))))
 
-(defn with-block-uuid
+(defn- with-block-uuid
   [pages]
   (->> (gp-util/distinct-by :block/name pages)
        (map (fn [page]

src/test/frontend/modules/outliner/core_test.cljs

@@ -8,7 +8,7 @@
             [frontend.db :as db]
             [frontend.db.model :as db-model]
             [clojure.walk :as walk]
-            [frontend.format.block :as block]
+            [logseq.graph-parser.block :as gp-block]
             [datascript.core :as d]
             [frontend.test.helper :as helper]))
 
@@ -65,7 +65,7 @@
 
 (defn- build-blocks
   [tree]
-  (block/with-parent-and-left 1 (build-node-tree tree)))
+  (gp-block/with-parent-and-left 1 (build-node-tree tree)))
 
 (defn transact-tree!
   [tree]

src/test/frontend/handler/extract_test.cljs → src/test/logseq/graph_parser/extract_test.cljs

@@ -1,17 +1,17 @@
-(ns frontend.handler.extract-test
+(ns logseq.graph-parser.extract-test
   (:require [cljs.test :refer [async deftest is]]
-            [frontend.handler.extract :as extract]
-            [frontend.util :as util]
+            [logseq.graph-parser.extract :as extract]
+            [clojure.pprint :as pprint]
             [promesa.core :as p]))
 
 (defn- extract
   [text]
-  (p/let [result (extract/extract-blocks-pages "repo" "a.md" text {})
+  (p/let [result (extract/extract-blocks-pages "a.md" text {:block-pattern "-"})
           result (last result)
           lefts (map (juxt :block/parent :block/left) result)]
     (if (not= (count lefts) (count (distinct lefts)))
       (do
-        (util/pprint (map (fn [x] (select-keys x [:block/uuid :block/level :block/content :block/left])) result))
+        (pprint/pprint (map (fn [x] (select-keys x [:block/uuid :block/level :block/content :block/left])) result))
         (throw (js/Error. ":block/parent && :block/left conflicts")))
       (mapv :block/content result))))
 

src/test/logseq/graph_parser/nbb_test_runner.cljs

@@ -6,7 +6,8 @@
             [logseq.graph-parser.text-test]
             [logseq.graph-parser.mldoc-test]
             [logseq.graph-parser.block-test]
-            [logseq.graph-parser.property-test]))
+            [logseq.graph-parser.property-test]
+            [logseq.graph-parser.extract-test]))
 
 (defmethod cljs.test/report [:cljs.test/default :end-run-tests] [m]
   (when-not (cljs.test/successful? m)
@@ -21,4 +22,5 @@
   (t/run-tests 'logseq.graph-parser.mldoc-test
                'logseq.graph-parser.text-test
                'logseq.graph-parser.property-test
-               'logseq.graph-parser.block-test))
+               'logseq.graph-parser.block-test
+               'logseq.graph-parser.extract-test))