Merge pull request #6024 from logseq/fix/remove-built-in-property-pages

Enhance: Property page enhancements

.github/workflows/graph-parser.yml

@@ -83,7 +83,7 @@ jobs:
         run: clojure -M:test
 
       - name: Run nbb-logseq tests
-        run: yarn nbb-logseq -cp src:test:../db/src -m logseq.graph-parser.nbb-test-runner/run-tests
+        run: yarn test
 
       # In this job because it depends on an npm package
       - name: Load namespaces into nbb-logseq

deps/graph-parser/.carve/ignore

@@ -2,3 +2,5 @@
 logseq.graph-parser.cli/parse-graph
 ;; For CLI
 logseq.graph-parser.mldoc/ast-export-markdown
+;; API
+logseq.graph-parser.property/register-built-in-properties

deps/graph-parser/README.md

@@ -49,7 +49,7 @@ To see available options that can run specific tests or namespaces: `clojure -M:
 
 To run nbb-logseq tests:
 ```
-yarn nbb-logseq -cp src:test -m logseq.graph-parser.nbb-test-runner/run-tests
+yarn test
 ```
 
 ### Managing dependencies

deps/graph-parser/package.json

@@ -7,5 +7,8 @@
   },
   "dependencies": {
     "mldoc": "^1.3.9"
+  },
+  "scripts": {
+    "test": "nbb-logseq -cp src:test:../db/src -m logseq.graph-parser.nbb-test-runner/run-tests"
   }
 }

deps/graph-parser/src/logseq/graph_parser/block.cljs

@@ -1,15 +1,16 @@
 (ns logseq.graph-parser.block
   "Block related code needed for graph-parser"
-  (:require [clojure.string :as string]
+  (:require [clojure.set :as set]
+            [clojure.string :as string]
             [clojure.walk :as walk]
             [datascript.core :as d]
-            [logseq.graph-parser.text :as text]
-            [logseq.graph-parser.utf8 :as utf8]
-            [logseq.graph-parser.property :as gp-property]
-            [logseq.graph-parser.util :as gp-util]
             [logseq.graph-parser.config :as gp-config]
+            [logseq.graph-parser.date-time-util :as date-time-util]
             [logseq.graph-parser.mldoc :as gp-mldoc]
-            [logseq.graph-parser.date-time-util :as date-time-util]))
+            [logseq.graph-parser.property :as gp-property]
+            [logseq.graph-parser.text :as text]
+            [logseq.graph-parser.utf8 :as utf8]
+            [logseq.graph-parser.util :as gp-util]))
 
 (defn heading-block?
   [block]
@@ -139,37 +140,53 @@
    (vector? block)
    (= "Timestamp" (first block))))
 
+(defn- get-page-refs-from-property-names
+  [properties {:property-pages/keys [enabled? excludelist]}]
+  (if (contains? #{true nil} enabled?)
+    (some->> properties
+             (map (comp name first))
+             (remove string/blank?)
+             (remove (set (map name excludelist)))
+             ;; Remove built-in properties as we don't want pages
+             ;; created for them by default
+             (remove (set (map name (into (gp-property/editable-built-in-properties)
+                                          (gp-property/hidden-built-in-properties)))))
+             distinct)
+    []))
+
 (defn- get-page-ref-names-from-properties
-  [format properties]
+  [format properties user-config]
   (let [page-refs (->>
-                     properties
-                     (remove (fn [[k _]]
-                               (contains? #{:background-color :background_color} (keyword k))))
-                     (map last)
-                     (map (fn [v]
-                            (cond
-                              (and (string? v)
-                                   (not (gp-mldoc/link? format v)))
-                              (let [v (string/trim v)
-                                    result (text/split-page-refs-without-brackets v {:un-brackets? false})]
-                                (if (coll? result)
-                                  (map text/page-ref-un-brackets! result)
-                                  []))
-
-                              (coll? v)
-                              (map (fn [s]
-                                     (when-not (and (string? v)
-                                                    (gp-mldoc/link? format v))
-                                       (text/page-ref-un-brackets! s))) v)
-
-                              :else
-                              nil)))
-                     (apply concat))
-        property-keys-page-refs (some->> properties
-                                         (map (comp name first))
-                                         (remove string/blank?)
-                                         (distinct))]
-    (->> (concat page-refs property-keys-page-refs)
+                   properties
+                   (remove (fn [[k _]]
+                             (contains?
+                              (set/union (apply disj
+                                           (gp-property/editable-built-in-properties)
+                                           gp-property/editable-linkable-built-in-properties)
+                                         (gp-property/hidden-built-in-properties))
+                              (keyword k))))
+                   (map last)
+                   (map (fn [v]
+                          (cond
+                            (and (string? v)
+                                 (not (gp-mldoc/link? format v)))
+                            (let [v (string/trim v)
+                                  result (text/split-page-refs-without-brackets v {:un-brackets? false})]
+                              (if (coll? result)
+                                (map text/page-ref-un-brackets! result)
+                                []))
+
+                            (coll? v)
+                            (map (fn [s]
+                                   (when-not (and (string? v)
+                                                  (gp-mldoc/link? format v))
+                                     (text/page-ref-un-brackets! s))) v)
+
+                            :else
+                            nil)))
+                   (apply concat))
+        page-refs-from-property-names (get-page-refs-from-property-names properties user-config)]
+    (->> (concat page-refs page-refs-from-property-names)
          (remove string/blank?)
          distinct)))
 
@@ -182,7 +199,7 @@
   (when (seq properties)
     (let [properties (seq properties)
           properties (into {} properties)
-          page-refs (get-page-ref-names-from-properties format properties)
+          page-refs (get-page-ref-names-from-properties format properties user-config)
           properties (->> properties
                           (map (fn [[k v]]
                                  (let [k (-> (string/lower-case (name k))
@@ -206,7 +223,7 @@
                                            k (keyword k)
                                            v (if (and
                                                   (string? v)
-                                                  (contains? #{:alias :aliases :tags} k))
+                                                  (contains? gp-property/editable-linkable-built-in-properties k))
                                                (set [v])
                                                v)
                                            v (if (coll? v) (set v) v)]
@@ -464,7 +481,7 @@
 
 (defn get-page-refs-from-properties
   [format properties db date-formatter]
-  (let [page-refs (get-page-ref-names-from-properties format properties)]
+  (let [page-refs (get-page-ref-names-from-properties format properties {})]
     (map (fn [page] (page-name->map page true db true date-formatter)) page-refs)))
 
 (defn- with-page-block-refs

deps/graph-parser/src/logseq/graph_parser/property.cljs

@@ -2,6 +2,7 @@
   "Property fns needed by graph-parser"
   (:require [logseq.graph-parser.util :as gp-util]
             [clojure.string :as string]
+            [clojure.set :as set]
             [goog.string :as gstring]
             [goog.string.format]))
 
@@ -12,6 +13,40 @@
    (contains? #{"Property_Drawer" "Properties"}
               (first block))))
 
+(def markers
+  #{"now" "later" "todo" "doing" "done" "wait" "waiting"
+    "canceled" "cancelled" "started" "in-progress"})
+
+;; Built-in properties are properties that logseq uses for its features. Most of
+;; these properties are hidden from the user but a few like the editable ones
+;; are visible for the user to edit.
+
+(def built-in-extended-properties (atom #{}))
+(defn register-built-in-properties
+  [props]
+  (reset! built-in-extended-properties (set/union @built-in-extended-properties props)))
+
+(def editable-linkable-built-in-properties
+  "Properties used by logseq that user can edit and that can have linkable property values"
+  #{:alias :aliases :tags})
+
+(defn editable-built-in-properties
+  "Properties used by logseq that user can edit"
+  []
+  (into #{:title :icon :template :template-including-parent :public :filters}
+        editable-linkable-built-in-properties))
+
+(defn hidden-built-in-properties
+  "Properties used by logseq that user can't edit or see"
+  []
+  (set/union
+   #{:id :custom-id :background-color :background_color :heading :collapsed
+     :created-at :updated-at :last-modified-at :created_at :last_modified_at
+     :query-table :query-properties :query-sort-by :query-sort-desc :ls-type
+     :hl-type :hl-page :hl-stamp :file-path}
+   (set (map keyword markers))
+   @built-in-extended-properties))
+
 (defonce properties-start ":PROPERTIES:")
 (defonce properties-end ":END:")
 (defonce properties-end-pattern

deps/graph-parser/src/logseq/graph_parser/test/docs_graph_helper.cljs

@@ -142,7 +142,7 @@
   ;; only increase over time as the docs graph rarely has deletions
   (testing "Counts"
     (is (= 211 (count files)) "Correct file count")
-    (is (= 44212 (count (d/datoms db :eavt))) "Correct datoms count")
+    (is (= 41290 (count (d/datoms db :eavt))) "Correct datoms count")
 
     (is (= 3600
            (ffirst

deps/graph-parser/test/logseq/graph_parser/block_test.cljs

@@ -1,36 +1,72 @@
 (ns logseq.graph-parser.block-test
   (:require [logseq.graph-parser.block :as gp-block]
-            [cljs.test :refer [deftest are]]))
+            [cljs.test :refer [deftest are testing is]]))
 
 (deftest test-extract-properties
   (are [x y] (= (:properties (gp-block/extract-properties :markdown x {})) y)
-    [["year" "1000"]] {:year 1000}
-    [["year" "\"1000\""]] {:year "\"1000\""}
-    [["background-color" "#000000"]] {:background-color "#000000"}
-    [["alias" "name/with space"]] {:alias #{"name/with space"}}
-    [["year" "1000"] ["alias" "name/with space"]] {:year 1000, :alias #{"name/with space"}}
-    [["year" "1000"] ["tags" "name/with space"]] {:year 1000, :tags #{"name/with space"}}
-    [["year" "1000"] ["tags" "name/with space, another"]] {:year 1000, :tags #{"name/with space" "another"}}
-    [["year" "1000"] ["alias" "name/with space, another"]] {:year 1000, :alias #{"name/with space" "another"}}
-    [["year" "1000"] ["alias" "name/with space, [[another [[nested]]]]"]] {:year 1000, :alias #{"name/with space" "another [[nested]]"}}
-    [["year" "1000"] ["alias" "name/with space, [[[[nested]] another]]"]] {:year 1000, :alias #{"name/with space" "[[nested]] another"}}
-    [["foo" "bar"]] {:foo "bar"}
-    [["foo" "bar, baz"]] {:foo #{"bar" "baz"}}
-    [["foo" "bar, [[baz]]"]] {:foo #{"bar" "baz"}}
-    [["foo" "[[bar]], [[baz]]"]] {:foo #{"bar" "baz"}}
-    [["foo" "[[bar]], [[nested [[baz]]]]"]] {:foo #{"bar" "nested [[baz]]"}}
-    [["foo" "[[bar]], [[nested [[baz]]]]"]] {:foo #{"bar" "nested [[baz]]"}}
-    [["foo" "bar, [[baz, test]]"]] {:foo #{"bar" "baz, test"}}
-    [["foo" "bar, [[baz, test, [[nested]]]]"]] {:foo #{"bar" "baz, test, [[nested]]"}}
-    [["file-path" "file:///home/x, y.pdf"]] {:file-path "file:///home/x, y.pdf"})
-
-  (are [x y] (= (vec (:page-refs (gp-block/extract-properties :markdown x {}))) y)
-    [["year" "1000"]] ["year"]
-    [["year" "\"1000\""]] ["year"]
-    [["foo" "[[bar]] test"]] ["bar" "test" "foo"]
-    [["foo" "[[bar]] test [[baz]]"]] ["bar" "test" "baz" "foo"]
-    [["foo" "[[bar]] test [[baz]] [[nested [[baz]]]]"]] ["bar" "test" "baz" "nested [[baz]]" "foo"]
-    [["foo" "#bar, #baz"]] ["bar" "baz" "foo"]
-    [["foo" "[[nested [[page]]]], test"]] ["nested [[page]]" "test" "foo"]))
-
-#_(cljs.test/run-tests)
+       ;; Built-in properties
+       [["background-color" "#000000"]] {:background-color "#000000"}
+       [["alias" "name/with space"]] {:alias #{"name/with space"}}
+       [["tags" "foo, bar"]] {:tags #{"foo" "bar"}}
+       [["tags" "'bar'"]] {:tags #{"'bar'"}}
+       [["file-path" "file:///home/x, y.pdf"]] {:file-path "file:///home/x, y.pdf"}
+
+       ;; User properties
+       [["year" "1000"]] {:year 1000}
+       [["year" "\"1000\""]] {:year "\"1000\""}
+       [["year" "1000"] ["alias" "name/with space"]] {:year 1000, :alias #{"name/with space"}}
+       [["year" "1000"] ["tags" "name/with space"]] {:year 1000, :tags #{"name/with space"}}
+       [["year" "1000"] ["tags" "name/with space, another"]] {:year 1000, :tags #{"name/with space" "another"}}
+       [["year" "1000"] ["alias" "name/with space, another"]] {:year 1000, :alias #{"name/with space" "another"}}
+       [["year" "1000"] ["alias" "name/with space, [[another [[nested]]]]"]] {:year 1000, :alias #{"name/with space" "another [[nested]]"}}
+       [["year" "1000"] ["alias" "name/with space, [[[[nested]] another]]"]] {:year 1000, :alias #{"name/with space" "[[nested]] another"}}
+       [["foo" "bar"]] {:foo "bar"}
+       [["foo" "bar, baz"]] {:foo #{"bar" "baz"}}
+       [["foo" "bar, [[baz]]"]] {:foo #{"bar" "baz"}}
+       [["foo" "[[bar]], [[baz]]"]] {:foo #{"bar" "baz"}}
+       [["foo" "[[bar]], [[nested [[baz]]]]"]] {:foo #{"bar" "nested [[baz]]"}}
+       [["foo" "[[bar]], [[nested [[baz]]]]"]] {:foo #{"bar" "nested [[baz]]"}}
+       [["foo" "bar, [[baz, test]]"]] {:foo #{"bar" "baz, test"}}
+       [["foo" "bar, [[baz, test, [[nested]]]]"]] {:foo #{"bar" "baz, test, [[nested]]"}})
+
+  (testing "page-refs"
+    (are [x y] (= (vec (:page-refs
+                        (gp-block/extract-properties :markdown x {:property-pages/enabled? true}))) y)
+         [["year" "1000"]] ["year"]
+         [["year" "\"1000\""]] ["year"]
+         [["year" "1000"] ["month" "12"]] ["year" "month"]
+         [["foo" "[[bar]] test"]] ["bar" "test" "foo"]
+         [["foo" "[[bar]] test [[baz]]"]] ["bar" "test" "baz" "foo"]
+         [["foo" "[[bar]] test [[baz]] [[nested [[baz]]]]"]] ["bar" "test" "baz" "nested [[baz]]" "foo"]
+         [["foo" "#bar, #baz"]] ["bar" "baz" "foo"]
+         [["foo" "[[nested [[page]]]], test"]] ["nested [[page]]" "test" "foo"])
+
+
+    (are [x y] (= (vec (:page-refs
+                        (gp-block/extract-properties :markdown x {:property-pages/enabled? false}))) y)
+         [["year" "1000"]] []
+         [["year" "1000"] ["month" "12"]] []
+         [["foo" "[[bar]] test"]] ["bar" "test"])
+
+    (is (= ["year"]
+           (:page-refs
+            (gp-block/extract-properties :markdown
+                                         [["year" "1000"] ["month" "12"]]
+                                         {:property-pages/enabled? true
+                                          :property-pages/excludelist #{:month :day}})))
+        ":property-pages/exclude-list excludes specified properties")
+
+    (is (= ["year"]
+           (:page-refs
+            (gp-block/extract-properties :markdown
+                                         [["year" "1000"]]
+                                         {})))
+        "Default to enabled when :property-pages/enabled? is not in config")
+
+    (is (= ["foo" "bar"]
+           (:page-refs
+            (gp-block/extract-properties :markdown
+                                         ;; tags is linkable and background-color is not
+                                         [["tags" "foo, bar"] ["background-color" "#008000"]]
+                                         {:property-pages/enabled? true})))
+        "Only editable linkable built-in properties have page-refs in property values")))

deps/graph-parser/test/logseq/graph_parser/cli_test.cljs

@@ -8,7 +8,7 @@
 (deftest ^:integration parse-graph
   (let [graph-dir "test/docs"
         _ (docs-graph-helper/clone-docs-repo-if-not-exists graph-dir)
-        {:keys [conn files asts]} (gp-cli/parse-graph graph-dir)]
+        {:keys [conn files asts]} (gp-cli/parse-graph graph-dir {:verbose false})]
 
     (docs-graph-helper/docs-graph-assertions @conn files)
 

e2e-tests/page-search.spec.ts

@@ -161,6 +161,6 @@ async function alias_test(page: Page, page_name: string, search_kws: string[]) {
   // TODO: search clicking (alias property)
 }
 
-test('page diacritic alias', async ({ page }) => {
+test.skip('page diacritic alias', async ({ page }) => {
   await alias_test(page, "ü", ["ü", "ü", "Ü"])
 })

src/main/frontend/components/block.cljs

@@ -1549,7 +1549,7 @@
   (and
    (or
     (empty? properties)
-    (property/properties-built-in? properties))
+    (property/properties-hidden? properties))
 
    (empty? title)
 
@@ -1805,9 +1805,12 @@
 
 (rum/defc property-cp
   [config block k v]
-  (let [date (and (= k :date) (date/get-locale-string (str v)))]
+  (let [date (and (= k :date) (date/get-locale-string (str v)))
+        property-pages-enabled? (contains? #{true nil} (:property-pages/enabled? (state/get-config)))]
     [:div
-     (page-cp (assoc config :property? true) {:block/name (subs (str k) 1)})
+     (if property-pages-enabled?
+       (page-cp (assoc config :property? true) {:block/name (subs (str k) 1)})
+       [:span.page-property-key.font-medium (name k)])
      [:span.mr-1 ":"]
      (cond
        (int? v)
@@ -2083,7 +2086,7 @@
           (timestamp-cp block "SCHEDULED" scheduled-ast)))
 
       (when (and (seq properties)
-                 (let [hidden? (property/properties-built-in? properties)]
+                 (let [hidden? (property/properties-hidden? properties)]
                    (not hidden?))
                  (not (and block-ref? (or (seq title) (seq body))))
                  (not (:slide? config)))

src/main/frontend/config.cljs

@@ -259,10 +259,6 @@
 
 (def config-default-content (rc/inline "config.edn"))
 
-(def markers
-  #{"now" "later" "todo" "doing" "done" "wait" "waiting"
-    "canceled" "cancelled" "started" "in-progress"})
-
 (defonce idb-db-prefix "logseq-db/")
 (defonce local-db-prefix "logseq_local_")
 (defonce local-handle "handle")

src/main/frontend/extensions/srs.cljs

@@ -3,6 +3,7 @@
             [frontend.db.query-dsl :as query-dsl]
             [frontend.db.query-react :as query-react]
             [frontend.util :as util]
+            [logseq.graph-parser.property :as gp-property]
             [frontend.util.property :as property]
             [frontend.util.drawer :as drawer]
             [frontend.util.persist-var :as persist-var]
@@ -707,12 +708,12 @@
 (component-macro/register query-macro-name cards)
 
 ;;; register builtin properties
-(property/register-built-in-properties #{card-last-interval-property
-                                         card-repeats-property
-                                         card-last-reviewed-property
-                                         card-next-schedule-property
-                                         card-last-easiness-factor-property
-                                         card-last-score-property})
+(gp-property/register-built-in-properties #{card-last-interval-property
+                                            card-repeats-property
+                                            card-last-reviewed-property
+                                            card-next-schedule-property
+                                            card-last-easiness-factor-property
+                                            card-last-score-property})
 
 ;;; register slash commands
 (commands/register-slash-command ["Cards"

src/main/frontend/util/property.cljs

@@ -11,26 +11,26 @@
             [logseq.graph-parser.text :as text]
             [frontend.util.cursor :as cursor]))
 
-(def built-in-extended-properties (atom #{}))
-(defn register-built-in-properties
-  [props]
-  (reset! built-in-extended-properties (set/union @built-in-extended-properties props)))
-
-(defn built-in-properties
+(defn hidden-properties
+  "These are properties hidden from user including built-in ones and ones
+  configured by user"
   []
   (set/union
-   #{:id :custom-id :background-color :heading :collapsed :created-at :updated-at :last-modified-at :created_at :last_modified_at :query-table :query-properties :query-sort-by :query-sort-desc
-     :ls-type :hl-type :hl-page :hl-stamp}
-   (set (map keyword config/markers))
-   (set (config/get-block-hidden-properties))
-   @built-in-extended-properties))
+   (gp-property/hidden-built-in-properties)
+   (set (config/get-block-hidden-properties))))
+
+;; TODO: Investigate if this behavior is correct for configured hidden
+;; properties and for editable built in properties
+(def built-in-properties
+  "Alias to hidden-properties to keep existing behavior"
+  hidden-properties)
 
-(defn properties-built-in?
+(defn properties-hidden?
   [properties]
   (and (seq properties)
        (let [ks (map (comp keyword string/lower-case name) (keys properties))
-             built-in-properties-set (built-in-properties)]
-         (every? built-in-properties-set ks))))
+             hidden-properties-set (hidden-properties)]
+         (every? hidden-properties-set ks))))
 
 (defn remove-empty-properties
   [content]

templates/config.edn

@@ -212,6 +212,13 @@
  ;; E.g. #{:created-at :updated-at}
  ;; :block-hidden-properties #{}
 
+ ;; Enable all your properties to have corresponding pages
+ :property-pages/enabled? true
+
+ ;; Properties to exclude from having property pages
+ ;; E.g. #{:duration :author}
+ ;; :property-pages/excludelist
+
  ;; logbook setup
  ;; :logbook/settings
  ;; {:with-second-support? false ;limit logbook to minutes, seconds will be eliminated