Setup CI jobs that are only for graph-parser

Also move all non graph-parser fns to frontend thanks to carve lint

.clj-kondo/config.edn

@@ -26,6 +26,7 @@
              frontend.db.query-react query-react
              frontend.util util
              frontend.util.property property
+             frontend.util.text text-util
              frontend.config config
              frontend.format.mldoc mldoc
              frontend.format.block block
@@ -40,7 +41,7 @@
              logseq.graph-parser.date-time-util date-time-util}}}
 
  :hooks {:analyze-call {rum.core/defc hooks.rum/defc
-                         rum.core/defcs hooks.rum/defcs}}
+                        rum.core/defcs hooks.rum/defcs}}
  :lint-as {promesa.core/let clojure.core/let
            promesa.core/loop clojure.core/loop
            promesa.core/recur clojure.core/recur

.github/workflows/graph-parser.yml

@@ -0,0 +1,56 @@
+name: logseq graph-parser CI
+
+on:
+  push:
+    branches: [master]
+    paths:
+      - 'deps/graph-parser/**'
+      - '!deps/graph-parser/**.md'
+  pull_request:
+    branches: [master]
+    paths:
+      - 'deps/graph-parser/**'
+      - '!deps/graph-parser/**.md'
+
+env:
+  CLOJURE_VERSION: '1.10.1.727'
+  # setup-java@v2 dropped support for legacy Java version syntax.
+  # This is the same as 1.8.
+  JAVA_VERSION: '8'
+  # This is the latest node version we can run.
+  NODE_VERSION: '16'
+  BABASHKA_VERSION: '0.8.1'
+
+jobs:
+
+  lint:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Set up Java
+        uses: actions/setup-java@v2
+        with:
+          distribution: 'zulu'
+          java-version: ${{ env.JAVA_VERSION }}
+
+      - name: Set up Clojure
+        uses: DeLaGuardo/setup-clojure@master
+        with:
+          cli: ${{ env.CLOJURE_VERSION }}
+
+      - name: Setup Babashka
+        uses: turtlequeue/[email protected]
+        with:
+          babashka-version: ${{ env.BABASHKA_VERSION }}
+
+      - name: Run clj-kondo lint
+        run: cd deps/graph-parser && clojure -M:clj-kondo --parallel --lint src test
+
+      - name: Lint for vars that are too large
+        run: scripts/large_vars.clj deps/graph-parser/src
+
+      - name: Carve lint for unused vars
+        run: cd deps/graph-parser && ../../scripts/carve.clj

deps/graph-parser/.carve/config.edn

@@ -1,5 +1,7 @@
 {:paths ["src"]
  :api-namespaces [
                   ;; carve doesn't detect nbb only usage
-                  logseq.graph-parser.log]
+                  logseq.graph-parser.log
+                  ;; Used by logseq but not worth splitting up
+                  logseq.graph-parser.db.schema]
  :report {:format :ignore}}

deps/graph-parser/.carve/ignore

@@ -0,0 +1,6 @@
+;; For CLI
+logseq.graph-parser.cli/parse
+;; For CLI
+logseq.graph-parser.db/start-conn
+;; For CLI
+logseq.graph-parser.mldoc/ast-export-markdown

deps/graph-parser/.clj-kondo/config.edn

@@ -0,0 +1,20 @@
+{:linters
+ {:unresolved-symbol {:exclude [;; TODO: Remove parse-* and update-* when https://github.com/clj-kondo/clj-kondo/issues/1694 is done
+                                parse-long
+                                parse-double
+                                parse-uuid
+                                update-keys
+                                update-vals]}
+
+  :consistent-alias
+  {:aliases {datascript.core d
+             logseq.graph-parser graph-parser
+             logseq.graph-parser.text text
+             logseq.graph-parser.block gp-block
+             logseq.graph-parser.mldoc gp-mldoc
+             logseq.graph-parser.util gp-util
+             logseq.graph-parser.property gp-property
+             logseq.graph-parser.config gp-config
+             logseq.graph-parser.date-time-util date-time-util}}}
+ :lint-as {promesa.core/let clojure.core/let}
+ :skip-comments true}

deps/graph-parser/.gitignore

@@ -0,0 +1 @@
+.clj-kondo/.cache

deps/graph-parser/deps.edn

@@ -7,4 +7,8 @@
   frankiesardo/linked {:mvn/version "1.3.0"}
   com.andrewmcveigh/cljs-time {:git/url "https://github.com/logseq/cljs-time" ;; fork
                                :sha     "5704fbf48d3478eedcf24d458c8964b3c2fd59a9"}
-  com.lambdaisland/glogi {:mvn/version "1.1.144"}}}
+  com.lambdaisland/glogi {:mvn/version "1.1.144"}}
+
+ :aliases
+ {:clj-kondo {:replace-deps {clj-kondo/clj-kondo {:mvn/version "2022.04.25"}}
+              :main-opts  ["-m" "clj-kondo.main"]}}}

deps/graph-parser/src/logseq/graph_parser/extract.cljc

@@ -183,10 +183,6 @@
          (map (partial apply merge))
          (with-block-uuid))))
 
-(defn extract-all-block-refs
-  [content]
-  (map second (re-seq #"\(\(([a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12})\)\)" content)))
-
 #?(:org.babashka/nbb
    (alter-var-root #'gp-mldoc/parse-property (constantly text/parse-property))
    :default

deps/graph-parser/src/logseq/graph_parser/text.cljs

@@ -51,8 +51,6 @@
 
 (def page-ref-re-without-nested #"\[\[([^\[\]]+)\]\]")
 
-(defonce between-re #"\(between ([^\)]+)\)")
-
 (defn page-ref-un-brackets!
   [s]
   (or (get-page-name s) s))
@@ -122,18 +120,6 @@
   [s]
   (string/split s #"(\"[^\"]*\")"))
 
-(def bilibili-regex #"^((?:https?:)?//)?((?:www).)?((?:bilibili.com))(/(?:video/)?)([\w-]+)(\S+)?$")
-(def loom-regex #"^((?:https?:)?//)?((?:www).)?((?:loom.com))(/(?:share/|embed/))([\w-]+)(\S+)?$")
-(def vimeo-regex #"^((?:https?:)?//)?((?:www).)?((?:player.vimeo.com|vimeo.com))(/(?:video/)?)([\w-]+)(\S+)?$")
-(def youtube-regex #"^((?:https?:)?//)?((?:www|m).)?((?:youtube.com|youtu.be|y2u.be|youtube-nocookie.com))(/(?:[\w-]+\?v=|embed/|v/)?)([\w-]+)([\S^\?]+)?$")
-
-(defn get-matched-video
-  [url]
-  (or (re-find youtube-regex url)
-      (re-find loom-regex url)
-      (re-find vimeo-regex url)
-      (re-find bilibili-regex url)))
-
 (def markdown-link #"\[([^\[]+)\](\(.*\))")
 
 (defn split-page-refs-without-brackets
@@ -228,16 +214,6 @@
        :else
        (remove-level-space-aux! text block-pattern space? trim-left?)))))
 
-(defn build-data-value
-  [col]
-  (let [items (map (fn [item] (str "\"" item "\"")) col)]
-    (gstring/format "[%s]"
-                 (string/join ", " items))))
-
-(defn media-link?
-  [media-formats s]
-  (some (fn [fmt] (gp-util/safe-re-find (re-pattern (str "(?i)\\." fmt "(?:\\?([^#]*))?(?:#(.*))?$")) s)) media-formats))
-
 (defn namespace-page?
   [p]
   (and (string? p)
@@ -246,100 +222,6 @@
        (not (string/starts-with? p "./"))
        (not (gp-util/url? p))))
 
-(defn add-timestamp
-  [content key value]
-  (let [new-line (str (string/upper-case key) ": " value)
-        lines (string/split-lines content)
-        new-lines (map (fn [line]
-                         (string/trim
-                          (if (string/starts-with? (string/lower-case line) key)
-                            new-line
-                            line)))
-                    lines)
-        new-lines (if (not= (map string/trim lines) new-lines)
-                    new-lines
-                    (cons (first new-lines) ;; title
-                          (cons
-                           new-line
-                           (rest new-lines))))]
-    (string/join "\n" new-lines)))
-
-(defn remove-timestamp
-  [content key]
-  (let [lines (string/split-lines content)
-        new-lines (filter (fn [line]
-                            (not (string/starts-with? (string/lower-case line) key)))
-                          lines)]
-    (string/join "\n" new-lines)))
-
-(defn get-current-line-by-pos
-  [s pos]
-  (let [lines (string/split-lines s)
-        result (reduce (fn [acc line]
-                         (let [new-pos (+ acc (count line))]
-                           (if (>= new-pos pos)
-                             (reduced line)
-                             (inc new-pos)))) 0 lines)]
-    (when (string? result)
-      result)))
-
-(defn get-string-all-indexes
-  "Get all indexes of `value` in the string `s`."
-  [s value]
-  (loop [acc []
-         i 0]
-    (if-let [i (string/index-of s value i)]
-      (recur (conj acc i) (+ i (count value)))
-      acc)))
-
-(defn surround-by?
-  "`pos` must be surrounded by `before` and `and` in string `value`, e.g. ((|))"
-  [value pos before end]
-  (let [start-pos (if (= :start before) 0 (- pos (count before)))
-        end-pos (if (= :end end) (count value) (+ pos (count end)))]
-    (when (>= (count value) end-pos)
-      (= (cond
-           (and (= :end end) (= :start before))
-           ""
-
-           (= :end end)
-           before
-
-           (= :start before)
-           end
-
-           :else
-           (str before end))
-         (subs value start-pos end-pos)))))
-
-(defn wrapped-by?
-  "`pos` must be wrapped by `before` and `and` in string `value`, e.g. ((a|b))"
-  [value pos before end]
-  (let [before-matches (->> (get-string-all-indexes value before)
-                            (map (fn [i] [i :before])))
-        end-matches (->> (get-string-all-indexes value end)
-                         (map (fn [i] [i :end])))
-        indexes (sort-by first (concat before-matches end-matches [[pos :between]]))
-        ks (map second indexes)
-        q [:before :between :end]]
-    (true?
-     (reduce (fn [acc k]
-               (if (= q (conj acc k))
-                 (reduced true)
-                 (vec (take-last 2 (conj acc k)))))
-             []
-             ks))))
-
-(defn get-graph-name-from-path
-  [path]
-  (when (string? path)
-    (let [parts (->> (string/split path #"/")
-                     (take-last 2))]
-      (-> (if (not= (first parts) "0")
-            (string/join "/" parts)
-            (last parts))
-          js/decodeURI))))
-
 (defonce non-parsing-properties
   (atom #{"background-color" "background_color"}))
 

deps/graph-parser/src/logseq/graph_parser/utf8.cljs

@@ -1,5 +1,4 @@
-(ns ^:nbb-compatible logseq.graph-parser.utf8
-  (:require [goog.object :as gobj]))
+(ns ^:nbb-compatible logseq.graph-parser.utf8)
 
 (defonce encoder
   (js/TextEncoder. "utf-8"))
@@ -22,7 +21,3 @@
    (if end
      (decode (.subarray arr start end))
      (decode (.subarray arr start)))))
-
-(defn length
-  [arr]
-  (gobj/get arr "length"))

deps/graph-parser/test/logseq/graph_parser/text_test.cljs

@@ -98,36 +98,6 @@
       "**foobar" "foobar"
       "*********************foobar" "foobar")))
 
-(deftest test-add-timestamp
-  []
-  (are [x y] (= x y)
-    (text/add-timestamp "LATER hello world\nhello"
-                        "scheduled"
-                        "<2021-08-25 Wed>")
-    "LATER hello world\nSCHEDULED: <2021-08-25 Wed>\nhello"
-
-    (text/add-timestamp "LATER hello world "
-                        "scheduled"
-                        "<2021-08-25 Wed>")
-    "LATER hello world\nSCHEDULED: <2021-08-25 Wed>"
-
-    (text/add-timestamp "LATER hello world\nfoo:: bar\ntest"
-                        "scheduled"
-                        "<2021-08-25 Wed>")
-    "LATER hello world\nSCHEDULED: <2021-08-25 Wed>\nfoo:: bar\ntest"))
-
-(deftest get-string-all-indexes
-  []
-  (are [x y] (= x y)
-    (text/get-string-all-indexes "[[hello]] [[world]]" "[[")
-    [0 10]
-
-    (text/get-string-all-indexes "abc abc ab" "ab")
-    [0 4 8]
-
-    (text/get-string-all-indexes "a.c a.c ab" "a.")
-    [0 4]))
-
 (deftest test-parse-property
   (testing "parse-property"
     (are [k v y] (= (text/parse-property k v {}) y)

src/main/frontend/components/block.cljs

@@ -52,6 +52,7 @@
             [frontend.util.clock :as clock]
             [frontend.util.property :as property]
             [frontend.util.drawer :as drawer]
+            [frontend.util.text :as text-util]
             [logseq.graph-parser.config :as gp-config]
             [logseq.graph-parser.util :as gp-util]
             [logseq.graph-parser.mldoc :as gp-mldoc]
@@ -824,7 +825,7 @@
         (nil? metadata-show)
         (or
          (gp-config/local-asset? s)
-         (text/media-link? media-formats s)))
+         (text-util/media-link? media-formats s)))
        (true? (boolean metadata-show))))
 
      ;; markdown
@@ -833,7 +834,7 @@
      ;; image http link
      (and (or (string/starts-with? full-text "http://")
               (string/starts-with? full-text "https://"))
-          (text/media-link? media-formats s)))))
+          (text-util/media-link? media-formats s)))))
 
 (defn- relative-assets-path->absolute-path
   [path]
@@ -1113,7 +1114,7 @@
 (defn- macro-vimeo-cp
   [_config arguments]
   (when-let [url (first arguments)]
-    (when-let [vimeo-id (nth (util/safe-re-find text/vimeo-regex url) 5)]
+    (when-let [vimeo-id (nth (util/safe-re-find text-util/vimeo-regex url) 5)]
       (when-not (string/blank? vimeo-id)
         (let [width (min (- (util/get-width) 96)
                          560)
@@ -1133,7 +1134,7 @@
     (when-let [id (cond
                     (<= (count url) 15) url
                     :else
-                    (nth (util/safe-re-find text/bilibili-regex url) 5))]
+                    (nth (util/safe-re-find text-util/bilibili-regex url) 5))]
       (when-not (string/blank? id)
         (let [width (min (- (util/get-width) 96)
                          560)
@@ -1154,7 +1155,7 @@
     (let [width (min (- (util/get-width) 96)
                      560)
           height (int (* width (/ 315 560)))
-          results (text/get-matched-video url)
+          results (text-util/get-matched-video url)
           src (match results
                      [_ _ _ (:or "youtube.com" "youtu.be" "y2u.be") _ id _]
                      (if (= (count id) 11) ["youtube-player" id] url)
@@ -1302,7 +1303,7 @@
         (when-let [youtube-id (cond
                                 (== 11 (count url)) url
                                 :else
-                                (nth (util/safe-re-find text/youtube-regex url) 5))]
+                                (nth (util/safe-re-find text-util/youtube-regex url) 5))]
           (when-not (string/blank? youtube-id)
             (youtube/youtube-video youtube-id))))
 
@@ -2316,7 +2317,7 @@
   (let [refs (model/get-page-names-by-ids
               (->> (map :db/id refs)
                    (remove nil?)))]
-    (text/build-data-value refs)))
+    (text-util/build-data-value refs)))
 
 (defn- get-children-refs
   [children]

src/main/frontend/components/journal.cljs

@@ -8,10 +8,10 @@
             [frontend.db.model :as model]
             [frontend.handler.page :as page-handler]
             [frontend.state :as state]
-            [logseq.graph-parser.text :as text]
             [logseq.graph-parser.util :as gp-util]
             [frontend.ui :as ui]
             [frontend.util :as util]
+            [frontend.util.text :as text-util]
             [goog.object :as gobj]
             [reitit.frontend.easy :as rfe]
             [rum.core :as rum]))
@@ -32,7 +32,7 @@
         page-entity (db/pull [:block/name (util/page-name-sanity-lc title)])
         data-page-tags (when (seq (:block/tags page-entity))
                          (let [page-names (model/get-page-names-by-ids (map :db/id (:block/tags page)))]
-                           (text/build-data-value page-names)))]
+                           (text-util/build-data-value page-names)))]
     [:div.flex-1.journal.page (cond-> {}
                                 data-page-tags
                                 (assoc :data-page-tags data-page-tags))

src/main/frontend/components/page.cljs

@@ -25,10 +25,10 @@
             [frontend.handler.route :as route-handler]
             [frontend.mixins :as mixins]
             [frontend.state :as state]
-            [logseq.graph-parser.text :as text]
             [frontend.search :as search]
             [frontend.ui :as ui]
             [frontend.util :as util]
+            [frontend.util.text :as text-util]
             [goog.object :as gobj]
             [reitit.frontend.easy :as rfe]
             [medley.core :as medley]
@@ -346,7 +346,7 @@
       [:div.flex-1.page.relative
        (merge (if (seq (:block/tags page))
                 (let [page-names (model/get-page-names-by-ids (map :db/id (:block/tags page)))]
-                  {:data-page-tags (text/build-data-value page-names)})
+                  {:data-page-tags (text-util/build-data-value page-names)})
                 {})
 
               {:key path-page-name

src/main/frontend/components/repo.cljs

@@ -14,7 +14,7 @@
             [reitit.frontend.easy :as rfe]
             [rum.core :as rum]
             [frontend.mobile.util :as mobile-util]
-            [logseq.graph-parser.text :as text]
+            [frontend.util.text :as text-util]
             [promesa.core :as p]
             [electron.ipc :as ipc]
             [goog.object :as gobj]
@@ -54,7 +54,7 @@
             [:div.flex.justify-between.mb-4 {:key id}
              (if local?
                (let [local-dir (config/get-local-dir url)
-                     graph-name (text/get-graph-name-from-path local-dir)]
+                     graph-name (text-util/get-graph-name-from-path local-dir)]
                  [:a {:title local-dir
                       :on-click #(state/pub-event! [:graph/switch url])}
                   graph-name])
@@ -89,7 +89,7 @@
         repo-links (mapv
                     (fn [{:keys [url]}]
                       (let [repo-path (db/get-repo-name url)
-                            short-repo-name (text/get-graph-name-from-path repo-path)]
+                            short-repo-name (text-util/get-graph-name-from-path repo-path)]
                         {:title short-repo-name
                          :hover-detail repo-path ;; show full path on hover
                          :options {:class "ml-1"

src/main/frontend/components/select.cljs

@@ -9,8 +9,8 @@
             [frontend.state :as state]
             [frontend.ui :as ui]
             [frontend.util :as util]
+            [frontend.util.text :as text-util]
             [frontend.db :as db]
-            [logseq.graph-parser.text :as text]
             [rum.core :as rum]
             [frontend.config :as config]
             [frontend.handler.repo :as repo-handler]
@@ -76,7 +76,7 @@
                            (or (config/demo-graph? url)
                                (= url (state/get-current-repo)))))
                  (map (fn [{:keys [url]}]
-                        {:value (text/get-graph-name-from-path
+                        {:value (text-util/get-graph-name-from-path
                                  ;; TODO: Use helper when a common one is refactored
                                  ;; from components.repo
                                  (if (config/local-db? url)
@@ -99,7 +99,7 @@
                      (remove (fn [{:keys [url]}]
                                (config/demo-graph? url)))
                      (map (fn [{:keys [url] :as original-graph}]
-                            {:value (text/get-graph-name-from-path
+                            {:value (text-util/get-graph-name-from-path
                                      ;; TODO: Use helper when a common one is refactored
                                      ;; from components.repo
                                      (if (config/local-db? url)

src/main/frontend/db/conn.cljs

@@ -5,6 +5,7 @@
             [frontend.mobile.util :as mobile-util]
             [frontend.state :as state]
             [frontend.config :as config]
+            [frontend.util.text :as text-util]
             [logseq.graph-parser.text :as text]
             [logseq.graph-parser.db :as gp-db]))
 
@@ -21,7 +22,7 @@
   [repo]
   (cond
     (mobile-util/native-platform?)
-    (text/get-graph-name-from-path repo)
+    (text-util/get-graph-name-from-path repo)
 
     (config/local-db? repo)
     (config/get-local-dir repo)

src/main/frontend/db/query_dsl.cljs

@@ -14,6 +14,7 @@
             [frontend.db.rules :as rules]
             [frontend.template :as template]
             [logseq.graph-parser.text :as text]
+            [frontend.util.text :as text-util]
             [frontend.util :as util]))
 
 
@@ -443,17 +444,18 @@ Some bindings in this fn:
   [s]
   (some-> s
           (string/replace text/page-ref-re "\"[[$1]]\"")
-          (string/replace text/between-re (fn [[_ x]]
-                                            (->> (string/split x #" ")
-                                                 (remove string/blank?)
-                                                 (map (fn [x]
-                                                        (if (or (contains? #{"+" "-"} (first x))
-                                                                (and (util/safe-re-find #"\d" (first x))
-                                                                     (some #(string/ends-with? x %) ["y" "m" "d" "h" "min"])))
-                                                          (keyword (name x))
-                                                          x)))
-                                                 (string/join " ")
-                                                 (util/format "(between %s)"))))))
+          (string/replace text-util/between-re
+                          (fn [[_ x]]
+                            (->> (string/split x #" ")
+                                 (remove string/blank?)
+                                 (map (fn [x]
+                                        (if (or (contains? #{"+" "-"} (first x))
+                                                (and (util/safe-re-find #"\d" (first x))
+                                                     (some #(string/ends-with? x %) ["y" "m" "d" "h" "min"])))
+                                          (keyword (name x))
+                                          x)))
+                                 (string/join " ")
+                                 (util/format "(between %s)"))))))
 
 (defn- add-bindings!
   [form q]

src/main/frontend/diff.cljs

@@ -6,7 +6,7 @@
             [cljs-bean.core :as bean]
             [frontend.util :as util]
             [logseq.graph-parser.util :as gp-util]
-            [logseq.graph-parser.text :as text]))
+            [frontend.util.text :as text-util]))
 
 (defn diff
   [s1 s2]
@@ -47,7 +47,7 @@
 
                       :else
                       (recur r1 t2 (inc i1) i2))))
-            current-line (text/get-current-line-by-pos markup pos)]
+            current-line (text-util/get-current-line-by-pos markup pos)]
         (cond
           (= (util/nth-safe markup pos)
              (util/nth-safe markup (inc pos))

src/main/frontend/fs/watcher_handler.cljs

@@ -4,12 +4,12 @@
             [frontend.db :as db]
             [frontend.db.model :as model]
             [frontend.handler.editor :as editor]
-            [logseq.graph-parser.extract :as extract]
             [frontend.handler.file :as file-handler]
             [frontend.handler.page :as page-handler]
             [frontend.handler.repo :as repo-handler]
             [frontend.handler.ui :as ui-handler]
             [logseq.graph-parser.util :as gp-util]
+            [frontend.util.text :as text-util]
             [lambdaisland.glogi :as log]
             [electron.ipc :as ipc]
             [promesa.core :as p]
@@ -21,7 +21,7 @@
 (defn- set-missing-block-ids!
   [content]
   (when (string? content)
-    (doseq [block-id (extract/extract-all-block-refs content)]
+    (doseq [block-id (text-util/extract-all-block-refs content)]
       (when-let [block (try
                          (model/get-block-by-uuid block-id)
                          (catch js/Error _e

src/main/frontend/handler/editor.cljs

@@ -48,6 +48,7 @@
             [frontend.util.priority :as priority]
             [frontend.util.property :as property]
             [frontend.util.thingatpt :as thingatpt]
+            [frontend.util.text :as text-util]
             [goog.dom :as gdom]
             [goog.dom.classes :as gdom-classes]
             [goog.object :as gobj]
@@ -814,7 +815,7 @@
               tail-len (count value)
               pos (max
                    (if original-content
-                     (utf8/length (utf8/encode original-content))
+                     (gobj/get (utf8/encode original-content) "length")
                      0)
                    0)]
           (edit-block! block pos id
@@ -934,8 +935,8 @@
     (when-let [block (db/pull [:block/uuid block-id])]
       (let [{:block/keys [content]} block
             content (or content (state/get-edit-content))
-            new-content (-> (text/remove-timestamp content key)
-                            (text/add-timestamp key value))]
+            new-content (-> (text-util/remove-timestamp content key)
+                            (text-util/add-timestamp key value))]
         (when (not= content new-content)
           (let [input-id (state/get-edit-input-id)]
             (if (and input-id
@@ -1566,7 +1567,7 @@
   (when input
     (let [value (gobj/get input "value")
           pos (cursor/pos input)]
-      (text/surround-by? value pos before end))))
+      (text-util/surround-by? value pos before end))))
 
 (defn wrapped-by?
   [input before end]
@@ -1574,7 +1575,7 @@
     (let [value (gobj/get input "value")
           pos (dec (cursor/pos input))]
       (when (>= pos 0)
-        (text/wrapped-by? value pos before end)))))
+        (text-util/wrapped-by? value pos before end)))))
 
 (defn get-matched-pages
   "Return matched page names"
@@ -2873,7 +2874,7 @@
 (defn wrap-macro-url
   [url]
   (cond
-    (boolean (text/get-matched-video url))
+    (boolean (text-util/get-matched-video url))
     (util/format "{{video %s}}" url)
 
     (string/includes? url "twitter.com")

src/main/frontend/mobile/deeplink.cljs

@@ -1,4 +1,4 @@
-(ns frontend.mobile.deeplink 
+(ns frontend.mobile.deeplink
   (:require
    [clojure.string :as string]
    [frontend.config :as config]
@@ -9,7 +9,7 @@
    [frontend.handler.user :as user-handler]
    [frontend.mobile.intent :as intent]
    [frontend.state :as state]
-   [logseq.graph-parser.text :as text]))
+   [frontend.util.text :as text-util]))
 
 (def *link-to-another-graph (atom false))
 
@@ -19,7 +19,7 @@
         pathname (.-pathname parsed-url)
         search-params (.-searchParams parsed-url)
         current-repo-url (state/get-current-repo)
-        get-graph-name-fn #(-> (text/get-graph-name-from-path %)
+        get-graph-name-fn #(-> (text-util/get-graph-name-from-path %)
                                (string/split "/")
                                last
                                string/lower-case)

src/main/frontend/mobile/intent.cljs

@@ -13,10 +13,10 @@
             [frontend.mobile.util :as mobile-util]
             [frontend.state :as state]
             [frontend.util :as util]
+            [frontend.util.text :as text-util]
             [lambdaisland.glogi :as log]
             [logseq.graph-parser.config :as gp-config]
             [logseq.graph-parser.mldoc :as gp-mldoc]
-            [logseq.graph-parser.text :as text]
             [promesa.core :as p]))
 
 (defn- handle-received-text [result]
@@ -34,7 +34,7 @@
                      (string/split url "\"\n"))
         text (some-> text (string/replace #"^\"" ""))
         url (and url
-                 (cond (boolean (text/get-matched-video url))
+                 (cond (boolean (text-util/get-matched-video url))
                        (util/format "{{video %s}}" url)
 
                        (and (string/includes? url "twitter.com")

src/main/frontend/util/text.cljs

@@ -0,0 +1,126 @@
+(ns frontend.util.text
+  (:require [clojure.string :as string]
+            [goog.string :as gstring]
+            [frontend.util :as util]))
+
+(defonce between-re #"\(between ([^\)]+)\)")
+
+(def bilibili-regex #"^((?:https?:)?//)?((?:www).)?((?:bilibili.com))(/(?:video/)?)([\w-]+)(\S+)?$")
+(def loom-regex #"^((?:https?:)?//)?((?:www).)?((?:loom.com))(/(?:share/|embed/))([\w-]+)(\S+)?$")
+(def vimeo-regex #"^((?:https?:)?//)?((?:www).)?((?:player.vimeo.com|vimeo.com))(/(?:video/)?)([\w-]+)(\S+)?$")
+(def youtube-regex #"^((?:https?:)?//)?((?:www|m).)?((?:youtube.com|youtu.be|y2u.be|youtube-nocookie.com))(/(?:[\w-]+\?v=|embed/|v/)?)([\w-]+)([\S^\?]+)?$")
+
+(defn get-matched-video
+  [url]
+  (or (re-find youtube-regex url)
+      (re-find loom-regex url)
+      (re-find vimeo-regex url)
+      (re-find bilibili-regex url)))
+
+(defn build-data-value
+  [col]
+  (let [items (map (fn [item] (str "\"" item "\"")) col)]
+    (gstring/format "[%s]"
+                 (string/join ", " items))))
+
+(defn media-link?
+  [media-formats s]
+  (some (fn [fmt] (util/safe-re-find (re-pattern (str "(?i)\\." fmt "(?:\\?([^#]*))?(?:#(.*))?$")) s)) media-formats))
+
+(defn add-timestamp
+  [content key value]
+  (let [new-line (str (string/upper-case key) ": " value)
+        lines (string/split-lines content)
+        new-lines (map (fn [line]
+                         (string/trim
+                          (if (string/starts-with? (string/lower-case line) key)
+                            new-line
+                            line)))
+                    lines)
+        new-lines (if (not= (map string/trim lines) new-lines)
+                    new-lines
+                    (cons (first new-lines) ;; title
+                          (cons
+                           new-line
+                           (rest new-lines))))]
+    (string/join "\n" new-lines)))
+
+(defn remove-timestamp
+  [content key]
+  (let [lines (string/split-lines content)
+        new-lines (filter (fn [line]
+                            (not (string/starts-with? (string/lower-case line) key)))
+                          lines)]
+    (string/join "\n" new-lines)))
+
+(defn get-current-line-by-pos
+  [s pos]
+  (let [lines (string/split-lines s)
+        result (reduce (fn [acc line]
+                         (let [new-pos (+ acc (count line))]
+                           (if (>= new-pos pos)
+                             (reduced line)
+                             (inc new-pos)))) 0 lines)]
+    (when (string? result)
+      result)))
+
+(defn surround-by?
+  "`pos` must be surrounded by `before` and `and` in string `value`, e.g. ((|))"
+  [value pos before end]
+  (let [start-pos (if (= :start before) 0 (- pos (count before)))
+        end-pos (if (= :end end) (count value) (+ pos (count end)))]
+    (when (>= (count value) end-pos)
+      (= (cond
+           (and (= :end end) (= :start before))
+           ""
+
+           (= :end end)
+           before
+
+           (= :start before)
+           end
+
+           :else
+           (str before end))
+         (subs value start-pos end-pos)))))
+
+(defn get-string-all-indexes
+  "Get all indexes of `value` in the string `s`."
+  [s value]
+  (loop [acc []
+         i 0]
+    (if-let [i (string/index-of s value i)]
+      (recur (conj acc i) (+ i (count value)))
+      acc)))
+
+(defn wrapped-by?
+  "`pos` must be wrapped by `before` and `and` in string `value`, e.g. ((a|b))"
+  [value pos before end]
+  (let [before-matches (->> (get-string-all-indexes value before)
+                            (map (fn [i] [i :before])))
+        end-matches (->> (get-string-all-indexes value end)
+                         (map (fn [i] [i :end])))
+        indexes (sort-by first (concat before-matches end-matches [[pos :between]]))
+        ks (map second indexes)
+        q [:before :between :end]]
+    (true?
+     (reduce (fn [acc k]
+               (if (= q (conj acc k))
+                 (reduced true)
+                 (vec (take-last 2 (conj acc k)))))
+             []
+             ks))))
+
+(defn get-graph-name-from-path
+  [path]
+  (when (string? path)
+    (let [parts (->> (string/split path #"/")
+                     (take-last 2))]
+      (-> (if (not= (first parts) "0")
+            (string/join "/" parts)
+            (last parts))
+          js/decodeURI))))
+
+(defn extract-all-block-refs
+  [content]
+  (map second (re-seq #"\(\(([a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12})\)\)" content)))

src/test/frontend/handler/repo_test.cljs

@@ -35,7 +35,8 @@
        (into {})))
 
 ;; Integration test that test parsing a large graph like docs
-(deftest ^:integration parse-and-load-files-to-db
+;; TODO: remove large-var and finish delete testing
+(deftest ^:large-vars/cleanup-todo ^:integration parse-and-load-files-to-db
   (let [graph-dir "src/test/docs"
         _ (docs-graph-helper/clone-docs-repo-if-not-exists graph-dir)
         files (docs-graph-helper/build-graph-files graph-dir)

src/test/frontend/util/text_test.cljs

@@ -0,0 +1,33 @@
+(ns frontend.util.text-test
+  (:require [cljs.test :refer [are deftest]]
+            [frontend.util.text :as text-util]))
+
+(deftest test-add-timestamp
+  []
+  (are [x y] (= x y)
+    (text-util/add-timestamp "LATER hello world\nhello"
+                        "scheduled"
+                        "<2021-08-25 Wed>")
+    "LATER hello world\nSCHEDULED: <2021-08-25 Wed>\nhello"
+
+    (text-util/add-timestamp "LATER hello world "
+                        "scheduled"
+                        "<2021-08-25 Wed>")
+    "LATER hello world\nSCHEDULED: <2021-08-25 Wed>"
+
+    (text-util/add-timestamp "LATER hello world\nfoo:: bar\ntest"
+                        "scheduled"
+                        "<2021-08-25 Wed>")
+    "LATER hello world\nSCHEDULED: <2021-08-25 Wed>\nfoo:: bar\ntest"))
+
+(deftest get-string-all-indexes
+  []
+  (are [x y] (= x y)
+    (text-util/get-string-all-indexes "[[hello]] [[world]]" "[[")
+    [0 10]
+
+    (text-util/get-string-all-indexes "abc abc ab" "ab")
+    [0 4 8]
+
+    (text-util/get-string-all-indexes "a.c a.c ab" "a.")
+    [0 4]))