Migrate most of extract ns to nbb

src/main/frontend/components/block.cljs

@@ -981,7 +981,7 @@
                   show-brackets? (state/show-brackets?)]
               (if (and page
                        (when-let [ext (util/get-file-ext href)]
-                         (config/mldoc-support? ext)))
+                         (gp-config/mldoc-support? ext)))
                 [:span.page-reference
                  (when show-brackets? [:span.text-gray-500 "[["])
                  (page-cp config page)

src/main/frontend/components/content.cljs

@@ -36,7 +36,7 @@
 
 (defn- lazy-load
   [format]
-  (let [format (format/normalize format)]
+  (let [format (gp-util/normalize-format format)]
     (when-let [record (format/get-format-record format)]
       (when-not (protocol/loaded? record)
         (set-format-js-loading! format true)
@@ -388,7 +388,7 @@
                    :format format}
                   id
                   config)
-      (let [format (format/normalize format)
+      (let [format (gp-util/normalize-format format)
             loading? (get loading format)
             markup? (contains? config/html-render-formats format)
             on-click (fn [e]
@@ -446,5 +446,5 @@
   (if hiccup
     [:div
      (hiccup-content id option)]
-    (let [format (format/normalize format)]
+    (let [format (gp-util/normalize-format format)]
       (non-hiccup-content id content on-click on-hide config format))))

src/main/frontend/components/file.cljs

@@ -9,11 +9,11 @@
             [frontend.context.i18n :refer [t]]
             [frontend.date :as date]
             [frontend.db :as db]
-            [frontend.format :as format]
             [frontend.handler.export :as export-handler]
             [frontend.state :as state]
             [frontend.util :as util]
             [logseq.graph-parser.config :as gp-config]
+            [logseq.graph-parser.util :as gp-util]
             [goog.object :as gobj]
             [reitit.frontend.easy :as rfe]
             [rum.core :as rum]))
@@ -71,7 +71,7 @@
                    state)}
   [state]
   (let [path (get-path state)
-        format (format/get-format path)
+        format (gp-util/get-format path)
         original-name (db/get-file-page path)
         random-id (str (d/squuid))]
     [:div.file {:id (str "file-edit-wrapper-" random-id)}

src/main/frontend/config.cljs

@@ -97,14 +97,6 @@
   (set/union (text-formats)
              (img-formats)))
 
-;; TODO: rename
-(defonce mldoc-support-formats
-  #{:org :markdown :md})
-
-(defn mldoc-support?
-  [format]
-  (contains? mldoc-support-formats (keyword format)))
-
 (def mobile?
   (when-not util/node-test?
     (gp-util/safe-re-find #"Mobi" js/navigator.userAgent)))

src/main/frontend/db/model.cljs

@@ -12,7 +12,6 @@
             [frontend.db.conn :as conn]
             [frontend.db.react :as react]
             [frontend.db.utils :as db-utils]
-            [frontend.format :as format]
             [frontend.state :as state]
             [frontend.util :as util :refer [react]]
             [logseq.graph-parser.util :as gp-util]
@@ -294,7 +293,7 @@
       (:block/format page)
       (when-let [file (:block/file page)]
         (when-let [path (:file/path (db-utils/entity (:db/id file)))]
-          (format/get-format path)))))
+          (gp-util/get-format path)))))
    (state/get-preferred-format)
    :markdown))
 

src/main/frontend/format.cljs

@@ -4,6 +4,7 @@
             [frontend.format.protocol :as protocol]
             [logseq.graph-parser.mldoc :as gp-mldoc]
             [logseq.graph-parser.text :as text]
+            [logseq.graph-parser.util :as gp-util]
             [clojure.string :as string]))
 
 ;; TODO: Properly fix this circular dependency:
@@ -13,22 +14,9 @@
 (defonce mldoc-record (->MldocMode))
 (defonce adoc-record (->AdocMode))
 
-(defn normalize
-  [format]
-  (case (keyword format)
-    :md :markdown
-    :asciidoc :adoc
-    ;; default
-    (keyword format)))
-
-(defn get-format
-  [file]
-  (when file
-    (normalize (keyword (string/lower-case (last (string/split file #"\.")))))))
-
 (defn get-format-record
   [format]
-  (case (normalize format)
+  (case (gp-util/normalize-format format)
     :org
     mldoc-record
     :markdown

src/main/frontend/handler/export.cljs

@@ -17,6 +17,7 @@
             [frontend.util :as util]
             [frontend.format.mldoc :as mldoc]
             [logseq.graph-parser.mldoc :as gp-mldoc]
+            [logseq.graph-parser.util :as gp-util]
             [goog.dom :as gdom]
             [promesa.core :as p])
   (:import [goog.string StringBuffer]))
@@ -379,7 +380,7 @@
                                               [?e2 :block/file ?e]
                                               [?e2 :block/name ?n]
                                               [?e2 :block/original-name ?n2]] db path)
-                                :format (f/get-format path)})))))
+                                :format (gp-util/get-format path)})))))
 
 
 (defn export-repo-as-markdown!

src/main/frontend/handler/extract.cljs

@@ -3,22 +3,18 @@
   (:require [clojure.set :as set]
             [clojure.string :as string]
             [clojure.walk :as walk]
-            [frontend.config :as config]
-            [frontend.db :as db]
-            [frontend.format :as format]
+            [datascript.core :as d]
             [frontend.format.block :as block]
-            [frontend.format.mldoc :as mldoc]
-            [frontend.state :as state]
             [logseq.graph-parser.text :as text]
-            [frontend.util :as util]
             [logseq.graph-parser.util :as gp-util]
             [logseq.graph-parser.mldoc :as gp-mldoc]
             [logseq.graph-parser.block :as gp-block]
             [logseq.graph-parser.property :as gp-property]
+            [logseq.graph-parser.config :as gp-config]
             [lambdaisland.glogi :as log]))
 
-(defn get-page-name
-  [file ast]
+(defn- get-page-name
+  [file ast page-name-order]
   ;; headline
   (let [ast (map first ast)]
     (if (string/includes? file "pages/contents.")
@@ -33,11 +29,11 @@
                                     title))
             file-name (when-let [file-name (last (string/split file #"/"))]
                         (let [result (first (gp-util/split-last "." file-name))]
-                          (if (config/mldoc-support? (string/lower-case (util/get-file-ext file)))
-                            (util/url-decode (string/replace result "." "/"))
+                          (if (gp-config/mldoc-support? (string/lower-case (gp-util/get-file-ext file)))
+                            (js/decodeURIComponent (string/replace result "." "/"))
                             result)))]
         (or property-name
-            (if (= (state/page-name-order) "heading")
+            (if (= page-name-order "heading")
               (or first-block-name file-name)
               (or file-name first-block-name)))))))
 
@@ -45,10 +41,10 @@
 ;; TODO: performance improvement
 (defn- extract-pages-and-blocks
   #_:clj-kondo/ignore
-  [repo-url format ast properties file content]
+  [repo-url format ast properties file content date-formatter page-name-order]
   (try
-    (let [page (get-page-name file ast)
-          [_original-page-name page-name _journal-day] (gp-block/convert-page-if-journal page (state/get-date-formatter))
+    (let [page (get-page-name file ast page-name-order)
+          [_original-page-name page-name _journal-day] (gp-block/convert-page-if-journal page date-formatter)
           blocks (->> (block/extract-blocks ast content false format)
                       (block/with-parent-and-left {:block/name page-name}))
           ref-pages (atom #{})
@@ -70,13 +66,13 @@
           page-entity (let [alias (:alias properties)
                             alias (if (string? alias) [alias] alias)
                             aliases (and alias
-                                         (seq (remove #(or (= page-name (util/page-name-sanity-lc %))
+                                         (seq (remove #(or (= page-name (gp-util/page-name-sanity-lc %))
                                                            (string/blank? %)) ;; disable blank alias
                                                       alias)))
                             aliases (->>
                                      (map
                                       (fn [alias]
-                                        (let [page-name (util/page-name-sanity-lc alias)
+                                        (let [page-name (gp-util/page-name-sanity-lc alias)
                                               aliases (distinct
                                                        (conj
                                                         (remove #{alias} aliases)
@@ -84,7 +80,7 @@
                                               aliases (when (seq aliases)
                                                         (map
                                                          (fn [alias]
-                                                           {:block/name (util/page-name-sanity-lc alias)})
+                                                           {:block/name (gp-util/page-name-sanity-lc alias)})
                                                          aliases))]
                                           (if (seq aliases)
                                             {:block/name page-name
@@ -108,7 +104,7 @@
                                                   tags (if (string? tags) [tags] tags)
                                                   tags (remove string/blank? tags)]
                                               (swap! ref-tags set/union (set tags))
-                                              (map (fn [tag] {:block/name (util/page-name-sanity-lc tag)
+                                              (map (fn [tag] {:block/name (gp-util/page-name-sanity-lc tag)
                                                               :block/original-name tag})
                                                    tags)))))
           namespace-pages (let [page (:block/original-name page-entity)]
@@ -123,15 +119,15 @@
                       (map
                        (fn [page]
                          {:block/original-name page
-                          :block/name (util/page-name-sanity-lc page)})
+                          :block/name (gp-util/page-name-sanity-lc page)})
                        @ref-tags)
                       namespace-pages)
                      ;; remove block references
                      (remove vector?)
                      (remove nil?))
-          pages (util/distinct-by :block/name pages)
+          pages (gp-util/distinct-by :block/name pages)
           pages (remove nil? pages)
-          pages (map (fn [page] (assoc page :block/uuid (db/new-block-id))) pages)
+          pages (map (fn [page] (assoc page :block/uuid (d/squuid))) pages)
           blocks (->> (remove nil? blocks)
                       (map (fn [b] (dissoc b :block/title :block/body :block/level :block/children :block/meta :block/anchor))))]
       [pages blocks])
@@ -139,21 +135,22 @@
       (log/error :exception e))))
 
 (defn extract-blocks-pages
-  [repo-url file content]
+  [repo-url file content {:keys [user-config date-formatter page-name-order]}]
   (if (string/blank? content)
     []
-    (let [format (format/get-format file)
+    (let [format (gp-util/get-format file)
           _ (println "Parsing start: " file)
-          ast (mldoc/->edn content (gp-mldoc/default-config format
+          ast (gp-mldoc/->edn content (gp-mldoc/default-config format
                                                          ;; {:parse_outline_only? true}
-                                                         ))]
+                                                         )
+                           user-config)]
       (println "Parsing finished : " file)
       (let [first-block (ffirst ast)
             properties (let [properties (and (gp-property/properties-ast? first-block)
                                              (->> (last first-block)
                                                   (map (fn [[x y]]
                                                          [x (if (string? y)
-                                                              (text/parse-property format x y (state/get-config))
+                                                              (text/parse-property format x y user-config)
                                                               y)]))
                                                   (into {})
                                                   (walk/keywordize-keys)))]
@@ -166,15 +163,15 @@
         (extract-pages-and-blocks
          repo-url
          format ast properties
-         file content)))))
+         file content date-formatter page-name-order)))))
 
 (defn with-block-uuid
   [pages]
-  (->> (util/distinct-by :block/name pages)
+  (->> (gp-util/distinct-by :block/name pages)
        (map (fn [page]
               (if (:block/uuid page)
                 page
-                (assoc page :block/uuid (db/new-block-id)))))))
+                (assoc page :block/uuid (d/squuid)))))))
 
 (defn with-ref-pages
   [pages blocks]

src/main/frontend/handler/file.cljs

@@ -8,7 +8,6 @@
             [clojure.core.async :as async]
             [frontend.config :as config]
             [frontend.db :as db]
-            [frontend.format :as format]
             [frontend.fs :as fs]
             [frontend.fs.nfs :as nfs]
             [frontend.handler.common :as common-handler]
@@ -17,6 +16,7 @@
             [frontend.state :as state]
             [frontend.util :as util]
             [logseq.graph-parser.util :as gp-util]
+            [logseq.graph-parser.config :as gp-config]
             [lambdaisland.glogi :as log]
             [promesa.core :as p]
             [frontend.mobile.util :as mobile]
@@ -43,7 +43,7 @@
   [files formats]
   (filter
    (fn [file]
-     (let [format (format/get-format file)]
+     (let [format (gp-util/get-format file)]
        (contains? formats format)))
    files))
 
@@ -119,10 +119,15 @@
          file (gp-util/path-normalize file)
          new? (nil? (db/entity [:file/path file]))]
      (db/set-file-content! repo-url file content)
-     (let [format (format/get-format file)
+     (let [format (gp-util/get-format file)
            file-content [{:file/path file}]
-           tx (if (contains? config/mldoc-support-formats format)
-                (let [[pages blocks] (extract-handler/extract-blocks-pages repo-url file content)
+           tx (if (contains? gp-config/mldoc-support-formats format)
+                (let [[pages blocks]
+                      (extract-handler/extract-blocks-pages
+                       repo-url file content
+                       {:user-config (state/get-config)
+                        :date-formatter (state/get-date-formatter)
+                        :page-name-order (state/page-name-order)})
                       first-page (first pages)
                       delete-blocks (->
                                      (concat

src/main/frontend/handler/repo.cljs

@@ -5,7 +5,6 @@
             [frontend.context.i18n :refer [t]]
             [frontend.date :as date]
             [frontend.db :as db]
-            [frontend.format :as format]
             [frontend.fs :as fs]
             [frontend.fs.nfs :as nfs]
             [frontend.handler.common :as common-handler]
@@ -23,6 +22,7 @@
             [shadow.resource :as rc]
             [frontend.db.persist :as db-persist]
             [logseq.graph-parser.util :as gp-util]
+            [logseq.graph-parser.config :as gp-config]
             [electron.ipc :as ipc]
             [clojure.set :as set]
             [clojure.core.async :as async]
@@ -217,8 +217,8 @@
   [repo-url files delete-files delete-blocks file-paths db-encrypted? re-render? re-render-opts opts]
   (let [support-files (filter
                        (fn [file]
-                         (let [format (format/get-format (:file/path file))]
-                           (contains? (set/union #{:edn :css} config/mldoc-support-formats) format)))
+                         (let [format (gp-util/get-format (:file/path file))]
+                           (contains? (set/union #{:edn :css} gp-config/mldoc-support-formats) format)))
                        files)
         support-files (sort-by :file/path support-files)
         {journals true non-journals false} (group-by (fn [file] (string/includes? (:file/path file) "journals/")) support-files)

src/main/frontend/mobile/intent.cljs

@@ -11,6 +11,7 @@
             [frontend.util :as util]
             [frontend.config :as config]
             [logseq.graph-parser.mldoc :as gp-mldoc]
+            [logseq.graph-parser.config :as gp-config]
             ["path" :as path]
             [frontend.mobile.util :as mobile-util]
             [frontend.handler.notification :as notification]
@@ -109,7 +110,7 @@
           format (db/get-page-format page)
           application-type (last (string/split type "/"))
           content (cond
-                    (config/mldoc-support? application-type)
+                    (gp-config/mldoc-support? application-type)
                     (embed-text-file url title)
 
                     (contains? (set/union #{:pdf} config/media-formats) (keyword application-type))

src/main/frontend/search.cljs

@@ -1,7 +1,7 @@
 (ns frontend.search
   (:require [cljs-bean.core :as bean]
             [clojure.string :as string]
-            [frontend.config :as config]
+            [logseq.graph-parser.config :as gp-config]
             [frontend.db :as db]
             [frontend.regex :as regex]
             [frontend.search.browser :as search-browser]
@@ -143,7 +143,7 @@
   ([q limit]
    (let [q (clean-str q)]
      (when-not (string/blank? q)
-       (let [mldoc-exts (set (map name config/mldoc-support-formats))
+       (let [mldoc-exts (set (map name gp-config/mldoc-support-formats))
              files (->> (db/get-files (state/get-current-repo))
                         (map first)
                         (remove (fn [file]
@@ -237,4 +237,4 @@
 (defn cache-stale?
   [repo]
   (when-let [engine (get-engine repo)]
-    (protocol/cache-stale? engine repo)))
+    (protocol/cache-stale? engine repo)))

src/main/logseq/graph_parser/config.cljs

@@ -14,3 +14,11 @@
 (defn draw?
   [path]
   (string/starts-with? path default-draw-directory))
+
+;; TODO: rename
+(defonce mldoc-support-formats
+  #{:org :markdown :md})
+
+(defn mldoc-support?
+  [format]
+  (contains? mldoc-support-formats (keyword format)))

src/main/logseq/graph_parser/util.cljs

@@ -161,3 +161,27 @@
   (some->> (string/split s #" ")
            (map string/capitalize)
            (string/join " ")))
+
+(defn distinct-by
+  "Copy of frontend.util/distinct-by. Too basic to couple to main app"
+  [f col]
+  (reduce
+   (fn [acc x]
+     (if (some #(= (f x) (f %)) acc)
+       acc
+       (vec (conj acc x))))
+   []
+   col))
+
+(defn normalize-format
+  [format]
+  (case (keyword format)
+    :md :markdown
+    :asciidoc :adoc
+    ;; default
+    (keyword format)))
+
+(defn get-format
+  [file]
+  (when file
+    (normalize-format (keyword (string/lower-case (last (string/split file #"\.")))))))

src/test/frontend/handler/extract_test.cljs

@@ -6,7 +6,7 @@
 
 (defn- extract
   [text]
-  (p/let [result (extract/extract-blocks-pages "repo" "a.md" text)
+  (p/let [result (extract/extract-blocks-pages "repo" "a.md" text {})
           result (last result)
           lefts (map (juxt :block/parent :block/left) result)]
     (if (not= (count lefts) (count (distinct lefts)))