refactor: replace directives(front matter or org) with properties

deps/graph-parser/src/logseq/graph_parser/block.cljs

@@ -170,7 +170,7 @@
                               (keyword k))))
                    ;; get links ast
                    (map last)
-                   (mapcat text/extract-refs-from-mldoc-ast))
+                   (mapcat (or (:extract-refs-from-property-value-fn user-config) text/extract-refs-from-mldoc-ast)))
         page-refs-from-property-names (get-page-refs-from-property-names properties user-config)]
     (->> (concat page-refs page-refs-from-property-names)
          (remove string/blank?)
@@ -184,7 +184,10 @@
           *invalid-properties (atom #{})
           properties (->> properties
                           (map (fn [[k v mldoc-ast]]
-                                 (let [k (-> (string/lower-case (subs (str k) 1))
+                                 (let [k (if (or (keyword? k) (symbol? k))
+                                           (subs (str k) 1)
+                                           k)
+                                       k (-> (string/lower-case k)
                                              (string/replace " " "-")
                                              (string/replace "_" "-")
                                              (string/replace #"[\"|^|(|)|{|}]+" ""))]
@@ -409,7 +412,7 @@
       (d/squuid)))
 
 (defn get-page-refs-from-properties
-  [format properties db date-formatter user-config]
+  [properties db date-formatter user-config]
   (let [page-refs (get-page-ref-names-from-properties properties user-config)]
     (map (fn [page] (page-name->map page true db true date-formatter)) page-refs)))
 
@@ -503,7 +506,13 @@
                    (let [content (utf8/substring encoded-content 0 first-block-start-pos)
                          {:keys [properties properties-order invalid-properties]} pre-block-properties
                          id (get-custom-id-or-new-id {:properties properties})
-                         property-refs (->> (get-page-refs-from-properties format properties db date-formatter user-config)
+                         property-refs (->> (get-page-refs-from-properties
+                                             properties db date-formatter
+                                             (assoc user-config
+                                                    :extract-refs-from-property-value-fn
+                                                    (fn [refs]
+                                                      (when (coll? refs)
+                                                        refs))))
                                             (map :block/original-name))
                          block {:uuid id
                                 :content content

deps/graph-parser/src/logseq/graph_parser/extract.cljc

@@ -167,7 +167,6 @@
             properties (let [properties (and (gp-property/properties-ast? first-block)
                                              (->> (last first-block)
                                                   (map (fn [[x y mldoc-ast]]
-                                                         (prn {:mldoc-ast mldoc-ast})
                                                          [x (text/parse-property x y mldoc-ast user-config)]))
                                                   (into {})
                                                   (walk/keywordize-keys)))]
@@ -199,7 +198,3 @@
          vals
          (map (partial apply merge))
          (with-block-uuid))))
-
-;; TODO: Properly fix this circular dependency:
-;; mldoc/->edn > text/parse-property > mldoc/link? ->mldoc/inline->edn + mldoc/default-config
-(set! gp-mldoc/parse-property text/parse-property)

deps/graph-parser/src/logseq/graph_parser/mldoc.cljc

@@ -17,6 +17,7 @@
 (defonce parseJson (gobj/get Mldoc "parseJson"))
 (defonce parseInlineJson (gobj/get Mldoc "parseInlineJson"))
 (defonce astExportMarkdown (gobj/get Mldoc "astExportMarkdown"))
+(defonce getReferences (gobj/get Mldoc "getReferences"))
 
 (def default-references
   (js/JSON.stringify
@@ -40,6 +41,11 @@
   [text config]
   (parseInlineJson text config))
 
+(defn get-references
+  [text config]
+  (when-not (string/blank? text)
+    (getReferences text config)))
+
 (defn ast-export-markdown
   [ast config references]
   (astExportMarkdown ast
@@ -105,77 +111,27 @@
        (distinct)))
 
 (defn collect-page-properties
-  [ast parse-property config-state]
-  (prn "debug ast")
-  (frontend.util/pprint ast)
-  (if (seq ast)
+  [ast config]
+  (when (seq ast)
     (let [original-ast ast
           ast (map first ast)           ; without position meta
           directive? (fn [[item _]] (= "directive" (string/lower-case (first item))))
           grouped-ast (group-by directive? original-ast)
-          directive-ast (take-while directive? original-ast)
-          [properties-ast other-ast] (if (= "Property_Drawer" (ffirst ast))
-                                       [(last (first ast))
-                                        (rest original-ast)]
-                                       [(->> (map first directive-ast)
-                                             (map rest))
-                                        (get grouped-ast false)])
-          _ (prn "debug:"
-                 {:properties-ast properties-ast})
-          properties (->>
-                      properties-ast
-                      (map (fn [[k v mldoc-ast]]
-                             (let [k (keyword (string/lower-case k))
-                                   v (if (contains? #{:title :description :filters :macro} k)
-                                       v
-                                       (parse-property k v mldoc-ast config-state))]
-                               [k v]))))
-          properties (into (linked/map) properties)
-          macro-properties (filter (fn [x] (= :macro (first x))) properties)
-          macros (if (seq macro-properties)
-                   (->>
-                    (map
-                     (fn [[_ v]]
-                       (let [[k v] (gp-util/split-first " " v)]
-                         (mapv
-                          string/trim
-                          [k v])))
-                     macro-properties)
-                    (into {}))
-                   {})
-          properties (->> (remove (fn [x] (= :macro (first x))) properties)
-                          (into (linked/map)))
-          properties (cond-> properties
-                             (seq macros)
-                             (assoc :macros macros))
-          alias (:alias properties)
-          alias (when alias
-                  (if (coll? alias)
-                    (remove string/blank? alias)
-                    [alias]))
-          filetags (when-let [org-file-tags (:filetags properties)]
-                     (->> (string/split org-file-tags ":")
-                          (remove string/blank?)))
-          tags (:tags properties)
-          tags (->> (->vec-concat tags filetags)
-                    (remove string/blank?)
-                    vec)
-          properties (assoc properties :tags tags :alias alias)
-          properties (-> properties
-                         (update :filetags (constantly filetags)))
-          properties (into (linked/map)
-                           (remove (fn [[_k v]]
-                                     (or (nil? v) (and (coll? v) (empty? v))))
-                                   properties))]
+          [properties-ast other-ast] [(->> (get grouped-ast true)
+                                           (map first))
+                                      (get grouped-ast false)]
+          properties (map (fn [[_directive k v]]
+                            (let [kname (string/lower-case k)
+                                  k (keyword kname)
+                                  mldoc-ast (-> (get-references v config) gp-util/json->clj)]
+                              [k v mldoc-ast]))
+                       properties-ast)]
       (if (seq properties)
-        (cons [["Properties" properties] nil] other-ast)
-        original-ast))
-    ast))
-
-(def parse-property nil)
+        (cons [["Property_Drawer" properties] nil] other-ast)
+        original-ast))))
 
 (defn ->edn
-  [content config config-state]
+  [content config _config-state]
   (if (string? content)
     (try
       (if (string/blank? content)
@@ -184,7 +140,7 @@
             (parse-json config)
             (gp-util/json->clj)
             (update-src-full-content content)
-            (collect-page-properties parse-property config-state)))
+            (collect-page-properties config)))
       (catch :default e
         (log/error :unexpected-error e)
         []))

deps/graph-parser/src/logseq/graph_parser/text.cljs

@@ -36,131 +36,13 @@
   [s]
   (or (get-page-name s) s))
 
-;; E.g "Foo Bar"
-(defn sep-by-comma
-  [s]
-  (when s
-    (some->>
-     (string/split s #"[\,|，]{1}")
-     (remove string/blank?)
-     (map string/trim))))
-
-(defn sep-by-hashtag
-  [s]
-  (when s
-    (some->>
-     (string/split s #"#")
-     (remove string/blank?)
-     (map string/trim))))
-
-(defn- not-matched-nested-pages
-  [s]
-  (and (string? s)
-       (> (count (re-seq page-ref/left-brackets-re s))
-          (count (re-seq page-ref/right-brackets-re s)))))
-
-(defn- ref-matched?
-  [s]
-  (let [x (re-seq page-ref/left-brackets-re s)
-        y (re-seq page-ref/right-brackets-re s)]
-    (and (> (count x) 0) (= (count x) (count y)))))
-
 (defn get-nested-page-name
   [page-name]
   (when-let [first-match (re-find page-ref/page-ref-without-nested-re page-name)]
     (second first-match)))
 
-(defn- concat-nested-pages
-  [coll]
-  (first
-   (reduce (fn [[acc not-matched-s] s]
-             (cond
-               (and not-matched-s (= s right-brackets))
-               (let [s' (str not-matched-s s)]
-                 (if (ref-matched? s')
-                   [(conj acc s') nil]
-                   [acc s']))
-
-               not-matched-s
-               [acc (str not-matched-s s)]
-
-               (not-matched-nested-pages s)
-               [acc s]
-
-               :else
-               [(conj acc s) not-matched-s])) [[] nil] coll)))
-
-(defn- sep-by-quotes
-  [s]
-  (string/split s #"(\"[^\"]*\")"))
-
 (def markdown-link #"\[([^\[]+)\](\(.*\))")
 
-(defn split-page-refs-without-brackets
-  ([s]
-   (split-page-refs-without-brackets s {}))
-  ([s {:keys [un-brackets?]
-       :or {un-brackets? true}}]
-   (cond
-     (and (string? s) (gp-util/wrapped-by-quotes? s))
-     (gp-util/unquote-string s)
-
-     (and (string? s) (re-find markdown-link s))
-     s
-
-     (and (string? s)
-            ;; Either a page ref, a tag or a comma separated collection
-            (or (re-find page-ref/page-ref-re s)
-                (re-find #"[\,|，|#|\"]+" s)))
-     (let [result (->> (sep-by-quotes s)
-                       (mapcat
-                        (fn [s]
-                          (when-not (gp-util/wrapped-by-quotes? (string/trim s))
-                            (string/split s page-ref/page-ref-outer-capture-re))))
-                       (mapcat (fn [s]
-                                 (cond
-                                   (gp-util/wrapped-by-quotes? s)
-                                   nil
-
-                                   (string/includes? (string/trimr s)
-                                                     (str right-brackets ","))
-                                   (let [idx (string/index-of s (str right-brackets ","))]
-                                     [(subs s 0 idx)
-                                      right-brackets
-                                      (subs s (+ idx 3))])
-
-                                   :else
-                                   [s])))
-                       (remove #(= % ""))
-                       (mapcat (fn [s] (if (string/ends-with? s right-brackets)
-                                         [(subs s 0 (- (count s) 2))
-                                          right-brackets]
-                                         [s])))
-                       concat-nested-pages
-                       (remove string/blank?)
-                       (mapcat (fn [s]
-                                 (cond
-                                   (gp-util/wrapped-by-quotes? s)
-                                   nil
-
-                                   (page-ref/page-ref? s)
-                                   [(if un-brackets? (page-ref-un-brackets! s) s)]
-
-                                   :else
-                                   (->> (sep-by-comma s)
-                                        (mapcat sep-by-hashtag)))))
-                       (distinct))]
-       (if (or (coll? result)
-               (and (string? result)
-                    (string/starts-with? result "#")))
-         (let [result (if coll? result [result])
-               result (map (fn [s] (string/replace s #"^#+" "")) result)]
-           (set result))
-         (first result)))
-
-     :else
-     s)))
-
 (defn- remove-level-space-aux!
   [text pattern space? trim-left?]
   (let [pattern (gstring/format
@@ -242,16 +124,14 @@
             (second (first data))
 
             nil)))
-       (map string/trim)))
+       (map string/trim)
+       (set)))
 
 (defn parse-property
   "Property value parsing that takes into account built-in properties, and user config"
   [k v mldoc-ast config-state]
-  (prn {:k k
-        :v v
-        :mldoc-ast mldoc-ast})
   (let [refs (extract-refs-from-mldoc-ast mldoc-ast)
-        k (subs (str k) 1)
+        k (if (or (symbol? k) (keyword? k)) (subs (str k) 1) k)
         v (if (or (symbol? v) (keyword? v))
             (subs (str v) 1)
             (str v))
@@ -259,7 +139,7 @@
         non-string-property (parse-non-string-property-value v)]
     (cond
       (contains? (set/union
-                  #{"title" "filters"}
+                  #{"filters" "macro"}
                   (get config-state :ignored-page-references-keywords)) k)
       v
 

deps/graph-parser/test/logseq/graph_parser/text_test.cljs

@@ -35,37 +35,6 @@
     "[single bracket]" "[single bracket]"
     "no brackets" "no brackets"))
 
-(deftest sep-by-comma
-  []
-  (are [x y] (= (text/sep-by-comma x) y)
-    "foo,bar" ["foo" "bar"]
-    "foo, bar" ["foo" "bar"]
-    "foo bar" ["foo bar"]
-    "[[foo]] [[bar]]" ["[[foo]] [[bar]]"]
-    "[[foo]],[[bar]]" ["[[foo]]", "[[bar]]"]
-    "[[foo]], [[bar]]" ["[[foo]]", "[[bar]]"]
-    "[[foo]]" ["[[foo]]"]
-    "[[nested [[foo]]]]" ["[[nested [[foo]]]]"]))
-
-(deftest split-page-refs-without-brackets
-  []
-  (are [x y] (= (text/split-page-refs-without-brackets x) y)
-    "foobar" "foobar"
-    "foo bar" "foo bar"
-    "foo, bar" #{"foo" "bar"}
-    "[[foo]] [[bar]]" #{"foo" "bar"}
-    "[[foo]],[[bar]]" #{"foo", "bar"}
-    "[[foo]], [[bar]]" #{"foo", "bar"}
-    "[[foo]]" #{"foo"}
-    "[[nested [[foo]]]]" #{"nested [[foo]]"}
-    "[[nested [[foo]]]], [[foo]]" #{"nested [[foo]]" "foo"}
-    "[[nested [[foo]] [[bar]]]], [[foo]]" #{"nested [[foo]] [[bar]]" "foo"}
-    "[[nested [[foo]], [[bar]]]], [[foo]]" #{"nested [[foo]], [[bar]]" "foo"}
-    "#tag," #{"tag"}
-    "#tag" #{"tag"}
-    "#tag1,#tag2" #{"tag1" "tag2"}
-    "[[Jan 26th, 2021]], hello" #{"hello" "Jan 26th, 2021"}))
-
 (def block-patterns
   {:markdown "-"
    :org "*"})

deps/graph-parser/test/logseq/graph_parser_test.cljs

@@ -92,9 +92,7 @@
 
 (deftest quoted-property-values
   (testing "With default config"
-    (quoted-property-values-test {}))
-  (testing "With :rich-property-values config"
-    (quoted-property-values-test {:rich-property-values? true})))
+    (quoted-property-values-test {})))
 
 (deftest page-properties-persistence
   (testing "Non-string property values"
@@ -195,16 +193,7 @@
         :multi-link #{"Logseq" "is the fastest" "triples" "text editor"}
         :desc #{"This is a multiple sentence description. It has one" "link"}
         :comma-prop #{"one" "two" "three"}}
-       {}))
-
-    (testing "With :rich-property-values config"
-      (property-relationships-test
-       properties
-       {:single-link #{"bar"}
-        :multi-link #{"Logseq" "triples" "text editor"}
-        :desc #{"link"}
-        :comma-prop "one, two,three"}
-       {:rich-property-values? true}))))
+       {}))))
 
 (deftest invalid-properties
   (let [conn (ldb/start-conn)

package.json

@@ -106,7 +106,7 @@
         "ignore": "5.1.8",
         "is-svg": "4.3.0",
         "jszip": "3.5.0",
-        "mldoc": "1.4.1",
+        "mldoc": "1.4.6",
         "path": "0.12.7",
         "pixi-graph-fork": "0.2.0",
         "pixi.js": "6.2.0",

src/main/frontend/components/block.cljs

@@ -1904,18 +1904,13 @@
          [[:span.opacity-50 "Click here to start writing, type '/' to see all the commands."]])
        [tags])))))
 
-(rum/defc span-comma
-  []
-  [:span ", "])
-
 (rum/defc property-cp
   [config block k value]
   (let [date (and (= k :date) (date/get-locale-string (str value)))
         user-config (state/get-config)
-        ;; In this mode and when value is a set of refs, display full property text
+        ;; When value is a set of refs, display full property text
         ;; because :block/properties value only contains refs but user wants to see text
-        v (if (and (:rich-property-values? user-config)
-                   (coll? value)
+        v (if (and (coll? value) (seq value)
                    (not (contains? gp-property/editable-linkable-built-in-properties k)))
             (gp-property/property-value-from-content (name k) (:block/content block))
             value)
@@ -1935,15 +1930,6 @@
        date
        date
 
-       (coll? v)
-       (let [v (->> (remove string/blank? v)
-                    (filter string?))
-             vals (for [v-item v]
-                    (page-cp config {:block/name v-item}))
-             elems (interpose (span-comma) vals)]
-         (for [elem elems]
-           (rum/with-key elem (str (random-uuid)))))
-
        (and (string? v) (gp-util/wrapped-by-quotes? v))
        (gp-util/unquote-string v)
 

src/main/frontend/format/block.cljs

@@ -10,7 +10,8 @@
             ["@sentry/react" :as Sentry]
             [logseq.graph-parser.config :as gp-config]
             [logseq.graph-parser.property :as gp-property]
-            [logseq.graph-parser.mldoc :as gp-mldoc]))
+            [logseq.graph-parser.mldoc :as gp-mldoc]
+            [lambdaisland.glogi :as log]))
 
 (defn extract-blocks
   "Wrapper around logseq.graph-parser.block/extract-blocks that adds in system state
@@ -25,6 +26,7 @@ and handles unexpected failure."
                               :db (db/get-db (state/get-current-repo))
                               :date-formatter (state/get-date-formatter)})
     (catch :default e
+      (log/error :exception e)
       (Sentry/captureException e)
       (notification/show! "An unexpected error occurred during block extraction." :error)
       [])))

src/main/frontend/format/mldoc.cljs

@@ -40,12 +40,12 @@
                "Heading"} type))
 
 (defn opml->edn
-  [content]
+  [config content]
   (try
     (if (string/blank? content)
       {}
       (let [[headers blocks] (-> content (parse-opml) (gp-util/json->clj))]
-        [headers (gp-mldoc/collect-page-properties blocks gp-mldoc/parse-property (state/get-config))]))
+        [headers (gp-mldoc/collect-page-properties blocks config)]))
     (catch js/Error e
       (log/error :edn/convert-failed e)
       [])))

src/main/frontend/handler/editor.cljs

@@ -633,8 +633,7 @@
 (defn properties-block
   [properties format page]
   (let [content (property/insert-properties format "" properties)
-        refs (gp-block/get-page-refs-from-properties format
-                                                     properties
+        refs (gp-block/get-page-refs-from-properties properties
                                                      (db/get-db (state/get-current-repo))
                                                      (state/get-date-formatter)
                                                      (state/get-config))]
@@ -2726,11 +2725,11 @@
             (autopair input-id "(" format nil))
 
         ;; If you type `xyz`, the last backtick should close the first and not add another autopair
-        ;; If you type several backticks in a row, each one should autopair to accommodate multiline code (```)        
+        ;; If you type several backticks in a row, each one should autopair to accommodate multiline code (```)
         (contains? (set (keys autopair-map)) key)
         (let [curr (get-current-input-char input)
                   prev (util/nth-safe value (dec pos))]
-            (util/stop e) 
+            (util/stop e)
             (if (and (= key "`") (= "`" curr) (not= "`" prev))
               (cursor/move-cursor-forward input)
               (autopair input-id key format nil)))

src/main/frontend/handler/external.cljs

@@ -11,6 +11,7 @@
             [frontend.db :as db]
             [frontend.format.mldoc :as mldoc]
             [frontend.format.block :as block]
+            [logseq.graph-parser.mldoc :as gp-mldoc]
             [logseq.graph-parser.util :as gp-util]
             [logseq.graph-parser.date-time-util :as date-time-util]
             [frontend.handler.page :as page]
@@ -79,7 +80,8 @@
   [data finished-ok-handler]
   #_:clj-kondo/ignore
   (when-let [repo (state/get-current-repo)]
-    (let [[headers parsed-blocks] (mldoc/opml->edn data)
+    (let [config (gp-mldoc/default-config :markdown)
+          [headers parsed-blocks] (mldoc/opml->edn config data)
           parsed-blocks (->>
                          (block/extract-blocks parsed-blocks "" :markdown {})
                          (mapv editor/wrap-parse-block))

src/main/frontend/handler/page.cljs

@@ -76,8 +76,7 @@
    (let [p (common-handler/get-page-default-properties title)
          ps (merge p properties)
          content (page-property/insert-properties format "" ps)
-         refs (gp-block/get-page-refs-from-properties format
-                                                      properties
+         refs (gp-block/get-page-refs-from-properties properties
                                                       (db/get-db (state/get-current-repo))
                                                       (state/get-date-formatter)
                                                       (state/get-config))]

src/test/frontend/db/query_dsl_test.cljs

@@ -116,12 +116,6 @@ prop-d:: nada"}])
 (deftest block-property-queries
   (testing "block property tests with default config"
     (test-helper/with-config {}
-      (block-property-queries-test)))
-
-  (test-helper/start-test-db!) ;; reset db
-
-  (testing "block property tests with rich-property-values? config"
-    (test-helper/with-config {:rich-property-values? true}
       (block-property-queries-test))))
 
 (defn- page-property-queries-test
@@ -187,12 +181,6 @@ prop-d:: nada"}])
 (deftest page-property-queries
   (testing "page property tests with default config"
     (test-helper/with-config {}
-      (page-property-queries-test)))
-
-  (test-helper/start-test-db!) ;; reset db
-
-  (testing "page property tests with rich-property-values? config"
-    (test-helper/with-config {:rich-property-values? true}
       (page-property-queries-test))))
 
 (deftest task-queries

templates/config.edn

@@ -232,11 +232,6 @@
  ;; E.g.:property-pages/excludelist #{:duration :author}
  ;; :property-pages/excludelist
 
- ;; Enables property values to contain a mix of tags, page-refs, special
- ;; punctuation and free-form text.
- ;; Re-index current graph for config to take effect
- ;; :rich-property-values? true
-
  ;; logbook setup
  ;; :logbook/settings
  ;; {:with-second-support? false ;limit logbook to minutes, seconds will be eliminated

yarn.lock

@@ -4809,10 +4809,10 @@ mkdirp@^1.0.3:
   resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e"
   integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==
 
[email protected].1:
-  version "1.4.1"
-  resolved "https://registry.yarnpkg.com/mldoc/-/mldoc-1.4.1.tgz#a239c9cc79c5e10a5a1e1602703f96588f26ee5a"
-  integrity sha512-RUCmIVfT+6tQ+Z6uqVcjLP5xppTmYlLEGO17S5uxZKIe2NHEsluryYhWvf+5yhER72zRXsBgxFGjF4zxp9eTGA==
[email protected].6:
+  version "1.4.6"
+  resolved "https://registry.yarnpkg.com/mldoc/-/mldoc-1.4.6.tgz#b5c9f091ca3be107b21b3d6a8c673b7f1d59944d"
+  integrity sha512-oPRqtuqT4f1WZNUofmVOYVzFhIE+3gsPGFPRsBHGcJmxtoFWzBY8x6E/FQuanVhGyQGDPJg6SQsyzAVfOzjKGA==
   dependencies:
     yargs "^12.0.2"