fix: can't parse graphs because of links regular expressions

close #4308

src/main/frontend/format.cljs

@@ -2,8 +2,11 @@
   (:require [frontend.format.mldoc :refer [->MldocMode] :as mldoc]
             [frontend.format.adoc :refer [->AdocMode]]
             [frontend.format.protocol :as protocol]
+            [frontend.text :as text]
             [clojure.string :as string]))
 
+(set! mldoc/parse-property text/parse-property)
+
 (defonce mldoc-record (->MldocMode))
 (defonce adoc-record (->AdocMode))
 

src/main/frontend/format/block.cljs

@@ -150,7 +150,7 @@
 
 ;; TODO: we should move this to mldoc
 (defn extract-properties
-  [properties]
+  [format properties]
   (when (seq properties)
     (let [properties (seq properties)
           page-refs (->>
@@ -179,7 +179,7 @@
                                            (remove string/blank? v)
                                            (if (string/blank? v)
                                              nil
-                                             (text/parse-property k v)))
+                                             (text/parse-property format k v)))
                                        k (keyword k)
                                        v (if (and
                                               (string? v)
@@ -413,7 +413,7 @@
                     (if (or (:pre-block? block)
                             (= (:format block) :org))
                       content
-                      (text/remove-indentation-spaces content (inc (:level block)) false))))]
+                      (mldoc/remove-indentation-spaces content (inc (:level block)) false))))]
     (if (= format :org)
       content
       (property/->new-properties content))))
@@ -516,7 +516,7 @@
                     (recur headings (rest blocks) timestamps' properties last-pos last-level children (conj block-all-content block-content) body))
 
                   (property/properties-ast? block)
-                  (let [properties (extract-properties (second block))]
+                  (let [properties (extract-properties format (second block))]
                     (recur headings (rest blocks) timestamps properties last-pos last-level children (conj block-all-content block-content) body))
 
                   (heading-block? block)

src/main/frontend/format/mldoc.cljs

@@ -2,7 +2,6 @@
   (:require [cljs-bean.core :as bean]
             [clojure.string :as string]
             [frontend.format.protocol :as protocol]
-            [frontend.text :as text]
             [frontend.utf8 :as utf8]
             [frontend.util :as util]
             [goog.object :as gobj]
@@ -87,6 +86,18 @@
                      config
                      (or references default-references)))
 
+(defn remove-indentation-spaces
+  [s level remove-first-line?]
+  (let [lines (string/split-lines s)
+        [f & r] lines
+        body (map (fn [line]
+                    (if (string/blank? (util/safe-subs line 0 level))
+                      (util/safe-subs line level)
+                      line))
+               (if remove-first-line? lines r))
+        content (if remove-first-line? body (cons f body))]
+    (string/join "\n" content)))
+
 (defn- ->vec
   [s]
   (if (string? s) [s] s))
@@ -99,7 +110,7 @@
        (distinct)))
 
 (defn collect-page-properties
-  [ast]
+  [ast parse-property]
   (if (seq ast)
     (let [original-ast ast
           ast (map first ast)           ; without position meta
@@ -119,7 +130,7 @@
                              (let [k (keyword (string/lower-case k))
                                    v (if (contains? #{:title :description :filters :macro} k)
                                        v
-                                       (text/parse-property k v))]
+                                       (parse-property k v))]
                                [k v]))))
           properties (into (linked/map) properties)
           macro-properties (filter (fn [x] (= :macro (first x))) properties)
@@ -168,7 +179,7 @@
             (let [{:keys [start_pos end_pos]} pos-meta
                   content (utf8/substring content start_pos end_pos)
                   spaces (re-find #"^[\t ]+" (first (string/split-lines content)))
-                  content (if spaces (text/remove-indentation-spaces content (count spaces) true)
+                  content (if spaces (remove-indentation-spaces content (count spaces) true)
                               content)
                   block ["Src" (assoc (second block) :full_content content)]]
               [block pos-meta])
@@ -181,6 +192,8 @@
                "Hiccup"
                "Heading"} type))
 
+(def parse-property nil)
+
 (defn ->edn
   [content config]
   (if (string? content)
@@ -191,7 +204,7 @@
             (parse-json config)
             (util/json->clj)
             (update-src-full-content content)
-            (collect-page-properties)))
+            (collect-page-properties parse-property)))
       (catch js/Error e
         (js/console.error e)
         []))
@@ -203,7 +216,7 @@
     (if (string/blank? content)
       {}
       (let [[headers blocks] (-> content (parse-opml) (util/json->clj))]
-        [headers (collect-page-properties blocks)]))
+        [headers (collect-page-properties blocks parse-property)]))
     (catch js/Error e
       (log/error :edn/convert-failed e)
       [])))
@@ -246,3 +259,8 @@
   [ast typ]
   (and (contains? #{"Drawer"} (ffirst ast))
        (= typ (second (first ast)))))
+
+(defn link?
+  [format link]
+  (when (string? link)
+    (= "Link" (ffirst (inline->edn link (default-config format))))))

src/main/frontend/format/mldoc_test.cljs

@@ -1,2 +0,0 @@
-(ns frontend.format.mldoc-test
-  (:require [frontend.format.mldoc]))

src/main/frontend/handler/extract.cljs

@@ -140,6 +140,7 @@
   (if (string/blank? content)
     []
     (let [format (format/get-format file)
+          _ (println "Parsing start: " file)
           ast (mldoc/->edn content (mldoc/default-config format
                                                          ;; {:parse_outline_only? true}
                                                          ))]
@@ -149,7 +150,7 @@
                                              (->> (last first-block)
                                                   (map (fn [[x y]]
                                                          [x (if (string? y)
-                                                              (text/parse-property x y)
+                                                              (text/parse-property format x y)
                                                               y)]))
                                                   (into {})
                                                   (walk/keywordize-keys)))]

src/main/frontend/handler/link.cljs

@@ -1,38 +0,0 @@
-(ns frontend.handler.link
-  (:require [frontend.util :as util]))
-
-(def plain-link-re-string
-  "(?:http://www\\.|https://www\\.|http://|https://){1}[a-z0-9]+(?:[\\-\\.]{1}[a-z0-9]+)*\\.[a-z]{2,5}(?:.*)*")
-;; Based on https://orgmode.org/manual/Link-Format.html#Link-Format
-(def org-link-re-1 (re-pattern (util/format "\\[\\[(%s)\\]\\[(.+)\\]\\]" plain-link-re-string)))
-(def org-link-re-2 (re-pattern (util/format "\\[\\[(%s)\\]\\]" plain-link-re-string)))
-(def markdown-link-re (re-pattern (util/format "^\\[(.+)\\]\\((%s)\\)" plain-link-re-string)))
-
-(defn- org-link
-  [link]
-  (when-let [matches (or (re-matches org-link-re-1 link)
-                         (re-matches org-link-re-2 link))]
-    {:type "org-link"
-     :url (second matches)
-     :label (nth matches 2 nil)}))
-
-(defn- markdown-link
-  [link]
-  (when-let [matches (re-matches markdown-link-re link)]
-    {:type "markdown-link"
-     :url (nth matches 2)
-     :label (second matches)}))
-
-(defn- plain-link
-  [link]
-  (when (util/url? link)
-    {:type "plain-link"
-     :url link}))
-
-(defn link
-  "If the given string is an org, markdown or plain url, return a map indicating
-  the url type, url itself and the optional label."
-  [link]
-  (or (plain-link link)
-      (org-link link)
-      (markdown-link link)))

src/main/frontend/text.cljs

@@ -2,7 +2,7 @@
   (:require [frontend.config :as config]
             [frontend.util :as util]
             [clojure.string :as string]
-            [frontend.handler.link :as link]))
+            [frontend.format.mldoc :as mldoc]))
 
 (def page-ref-re-0 #"\[\[(.*)\]\]")
 (def org-page-ref-re #"\[\[(file:.*)\]\[.+?\]\]")
@@ -231,18 +231,6 @@
   [img-formats s]
   (some (fn [fmt] (util/safe-re-find (re-pattern (str "(?i)\\." fmt "(?:\\?([^#]*))?(?:#(.*))?$")) s)) img-formats))
 
-(defn remove-indentation-spaces
-  [s level remove-first-line?]
-  (let [lines (string/split-lines s)
-        [f & r] lines
-        body (map (fn [line]
-                    (if (string/blank? (util/safe-subs line 0 level))
-                      (util/safe-subs line level)
-                      line))
-               (if remove-first-line? lines r))
-        content (if remove-first-line? body (cons f body))]
-    (string/join "\n" content)))
-
 (defn namespace-page?
   [p]
   (and (string? p)
@@ -349,30 +337,32 @@
   (atom #{"background-color" "background_color"}))
 
 (defn parse-property
-  [k v]
-  (let [k (name k)
-        v (if (or (symbol? v) (keyword? v)) (name v) (str v))
-        v (string/trim v)]
-    (cond
-      (contains? #{"title" "filters"} k)
-      v
+  ([k v]
+   (parse-property :markdown k v))
+  ([format k v]
+   (let [k (name k)
+         v (if (or (symbol? v) (keyword? v)) (name v) (str v))
+         v (string/trim v)]
+     (cond
+       (contains? #{"title" "filters"} k)
+       v
 
-      (= v "true")
-      true
-      (= v "false")
-      false
+       (= v "true")
+       true
+       (= v "false")
+       false
 
-      (and (not= k "alias") (util/safe-re-find #"^\d+$" v))
-      (util/safe-parse-int v)
+       (and (not= k "alias") (util/safe-re-find #"^\d+$" v))
+       (util/safe-parse-int v)
 
-      (util/wrapped-by-quotes? v) ; wrapped in ""
-      v
+       (util/wrapped-by-quotes? v) ; wrapped in ""
+       v
 
-      (contains? @non-parsing-properties (string/lower-case k))
-      v
+       (contains? @non-parsing-properties (string/lower-case k))
+       v
 
-      (link/link v)
-      v
+       (mldoc/link? format v)
+       v
 
-      :else
-      (split-page-refs-without-brackets v))))
+       :else
+       (split-page-refs-without-brackets v)))))

src/test/frontend/format/block_test.cljs

@@ -3,7 +3,7 @@
             [cljs.test :refer [deftest are]]))
 
 (deftest test-extract-properties
-  (are [x y] (= (:properties (block/extract-properties x)) y)
+  (are [x y] (= (:properties (block/extract-properties :markdown x)) y)
     [["year" "1000"]] {:year 1000}
     [["year" "\"1000\""]] {:year "\"1000\""}
     [["background-color" "#000000"]] {:background-color "#000000"}
@@ -23,7 +23,7 @@
     [["foo" "bar, [[baz, test]]"]] {:foo #{"bar" "baz, test"}}
     [["foo" "bar, [[baz, test, [[nested]]]]"]] {:foo #{"bar" "baz, test, [[nested]]"}})
 
-  (are [x y] (= (vec (:page-refs (block/extract-properties x))) y)
+  (are [x y] (= (vec (:page-refs (block/extract-properties :markdown x))) y)
     [["year" "1000"]] []
     [["year" "\"1000\""]] []
     [["foo" "[[bar]] test"]] ["bar" "test"]

src/test/frontend/format/mldoc-test.cljs

@@ -0,0 +1,35 @@
+(ns frontend.format.mldoc-test
+  (:require [frontend.format.mldoc :as mldoc]
+            [cljs.test :refer [testing deftest are]]))
+
+(deftest test-link
+  (testing "non-link"
+    (are [x y] (= (mldoc/link? :markdown x) y)
+      "google.com" false))
+
+  (testing "plain links"
+    (are [x y] (= (mldoc/link? :markdown x) y)
+      "http://www.google.com" true
+      "http://google.com" true))
+
+  (testing "org links with labels"
+    (are [x y] (= (mldoc/link? :org x) y)
+      "[[http://www.google.com][google]]" true
+      "[[http://google.com][google]]" true
+      "[[https://www.google.com][google]]" true
+      "[[https://google.com][google]]" true))
+
+  (testing "org links without labels"
+    (are [x y] (= (mldoc/link? :org x) y)
+      "[[http://www.google.com]]" true
+      "[[https://www.google.com]]" true))
+
+  (testing "markdown links"
+    (are [x y] (= (mldoc/link? :markdown x) y)
+      "[google](http://www.google.com)" true
+      "[google](https://www.google.com)" true))
+
+  ;; https://github.com/logseq/logseq/issues/4308
+  (testing "parsing links should be finished"
+    (are [x y] (= (mldoc/link? :markdown x) y)
+      "[YouTube](https://www.youtube.com/watch?v=-8ym7pyUs9gL) - [Vimeo](https://vimeo.com/677920303) {{youtube https://www.youtube.com/watch?v=-8ym7pyUs9g}}" true)))

src/test/frontend/handler/link_test.cljs

@@ -1,49 +0,0 @@
-(ns frontend.handler.link-test
-  (:require [cljs.test :refer [are deftest testing]]
-            [frontend.handler.link :as link]))
-
-(deftest test-link
-  (testing "non-link"
-    (are [x y] (= (link/link x) y)
-      "google.com" nil
-      "[[google.com][google]]" nil
-      "[[google.com]]" nil
-      "[google](google.com)" nil))
-
-  (testing "plain links"
-    (are [x y] (= (link/link x) y)
-      "http://www.google.com"
-      {:type "plain-link" :url "http://www.google.com"}
-
-      "http://google.com"
-      {:type "plain-link" :url "http://google.com"}))
-
-  (testing "org links with labels"
-    (are [x y] (= (link/link x) y)
-      "[[http://www.google.com][google]]"
-      {:type "org-link" :url "http://www.google.com" :label "google"}
-
-      "[[http://google.com][google]]"
-      {:type "org-link" :url "http://google.com" :label "google"}
-
-      "[[https://www.google.com][google]]"
-      {:type "org-link" :url "https://www.google.com" :label "google"}
-
-      "[[https://google.com][google]]"
-      {:type "org-link" :url "https://google.com" :label "google"}))
-
-  (testing "org links without labels"
-    (are [x y] (= (link/link x) y)
-      "[[http://www.google.com]]"
-      {:type "org-link" :url "http://www.google.com" :label nil}
-
-      "[[https://www.google.com]]"
-      {:type "org-link" :url "https://www.google.com" :label nil}))
-
-  (testing "markdown links"
-    (are [x y] (= (link/link x) y)
-      "[google](http://www.google.com)"
-      {:type "markdown-link" :url "http://www.google.com" :label "google"}
-
-      "[google](https://www.google.com)"
-      {:type "markdown-link" :url "https://www.google.com" :label "google"})))