Merge pull request #8387 from logseq/ben/advanced-queries-relative-date-input

Enhancement: Add +/- syntax, (w)eek (m)onth (y)ear, and time support to query :inputs

deps/db/package.json

@@ -3,6 +3,6 @@
   "version": "1.0.0",
   "private": true,
   "devDependencies": {
-    "@logseq/nbb-logseq": "^1.1.157"
+    "@logseq/nbb-logseq": "^1.1.158"
   }
 }

deps/db/yarn.lock

@@ -2,10 +2,10 @@
 # yarn lockfile v1
 
 
-"@logseq/nbb-logseq@^1.1.157":
-  version "1.1.157"
-  resolved "https://registry.yarnpkg.com/@logseq/nbb-logseq/-/nbb-logseq-1.1.157.tgz#337be95156e5b22caf5533663ae8a5a79cc43fbd"
-  integrity sha512-cuutsKZDdg850qa6HquOTKKZ9WpWUjSozRdrfvI/2WIbAv2MVQKPQYtB03K55OW9i3D1K0jAwDM0xzGI2lWyFQ==
+"@logseq/nbb-logseq@^1.1.158":
+  version "1.1.158"
+  resolved "https://registry.yarnpkg.com/@logseq/nbb-logseq/-/nbb-logseq-1.1.158.tgz#ebff10bdb0998b43e52e69ad487be04236b84569"
+  integrity sha512-NT3w5BmYBUeyOPNDc3SRNUrC4EXiY75KXiTdDS24kYaRX4UR63UZiAcCrG2GLB3jS48N2xv23dWJusHFer/+sQ==
   dependencies:
     import-meta-resolve "^2.1.0"
 

deps/graph-parser/package.json

@@ -3,7 +3,7 @@
   "version": "1.0.0",
   "private": true,
   "devDependencies": {
-    "@logseq/nbb-logseq": "^1.1.157"
+    "@logseq/nbb-logseq": "^1.1.158"
   },
   "dependencies": {
     "mldoc": "^1.5.1"

deps/graph-parser/src/logseq/graph_parser/util/db.cljs

@@ -22,55 +22,139 @@ it will return 1622433600000, which is equivalent to Mon May 31 2021 00 :00:00."
   (parse-long
    (string/replace (date-time-util/ymd date) "/" "")))
 
+(defn old->new-relative-date-format [input]
+  (let [count (re-find #"^\d+" (name input))
+        plus-minus (if (re-find #"after" (name input)) "+" "-")
+        ms? (string/ends-with? (name input) "-ms")]
+    (keyword :today (str plus-minus count "d" (if ms? "-ms" "")))))
+
+(comment
+  (old->new-relative-date-format "1d")
+  (old->new-relative-date-format "1d-before")
+  (old->new-relative-date-format "1d-after")
+  (old->new-relative-date-format "1d-before-ms")
+  (old->new-relative-date-format "1d-after-ms")
+  (old->new-relative-date-format "1w-after-ms"))
+
+(defn get-relative-date [input]
+  (case (or (namespace input) "today")
+    "today" (t/today)))
+
+(defn get-offset-date [relative-date direction amount unit]
+  (let [offset-fn (case direction "+" t/plus "-" t/minus)
+        offset-amount (parse-long amount) 
+        offset-unit-fn (case unit
+                         "d" t/days
+                         "w" t/weeks
+                         "m" t/months
+                         "y" t/years)]
+    (offset-fn (offset-fn relative-date (offset-unit-fn offset-amount)))))
+
+(defn get-ts-units 
+  "There are currently several time suffixes being used in inputs:
+  - ms: milliseconds, will return a time relative to the direction the date is being adjusted
+  - start: will return the time at the start of the day [00:00:00.000]
+  - end: will return the time at the end of the day [23:59:59.999]
+  - HHMM: will return the specified time at the turn of the minute [HH:MM:00.000]
+  - HHMMSS: will return the specified time at the turm of the second [HH:MM:SS.000]
+  - HHMMSSmmm: will return the specified time at the turn of the millisecond [HH:MM:SS.mmm]
+  
+  The latter three will be capped to the maximum allowed for each unit so they will always be valid times"
+  [offset-direction offset-time]
+  (case offset-time 
+    "ms" (if (= offset-direction "+") [23 59 59 999] [0 0 0 0]) 
+    "start" [0 0 0 0] 
+    "end" [23 59 59 999] 
+    ;; if it's not a matching string, then assume it is HHMM
+    (let [[h1 h2 m1 m2 s1 s2 ms1 ms2 ms3] (str offset-time "000000000")]
+      [(min 23  (parse-long (str h1 h2))) 
+       (min 59  (parse-long (str m1 m2)))
+       (min 59  (parse-long (str s1 s2)))
+       (min 999 (parse-long (str ms1 ms2 ms3)))])))
+
+(defn keyword-input-dispatch [input]
+  (cond 
+    (#{:current-page :current-block :parent-block :today :yesterday :tomorrow :right-now-ms} input) input
+
+    (re-find #"^[+-]\d+[dwmy]?$" (name input)) :relative-date
+    (re-find #"^[+-]\d+[dwmy]-(ms|start|end|\d{2}|\d{4}|\d{6}|\d{9})?$" (name input)) :relative-date-time
+
+    (= :start-of-today-ms input) :today-time 
+    (= :end-of-today-ms input) :today-time
+    (re-find #"^today-(start|end|\d{2}|\d{4}|\d{6}|\d{9})$" (name input)) :today-time
+
+    (re-find #"^\d+d(-before|-after|-before-ms|-after-ms)?$" (name input)) :DEPRECATED-relative-date))
+
+(defmulti resolve-keyword-input (fn [_db input _opts] (keyword-input-dispatch input))) 
+
+(defmethod resolve-keyword-input :current-page [_ _ {:keys [current-page-fn]}]
+  (when current-page-fn
+    (some-> (current-page-fn) string/lower-case)))
+
+(defmethod resolve-keyword-input :current-block [db _ {:keys [current-block-uuid]}]
+  (when current-block-uuid
+    (:db/id (d/entity db [:block/uuid current-block-uuid]))))
+
+(defmethod resolve-keyword-input :parent-block [db _ {:keys [current-block-uuid]}]
+  (when current-block-uuid
+    (:db/id (:block/parent (d/entity db [:block/uuid current-block-uuid])))))
+
+(defmethod resolve-keyword-input :today [_ _ _]
+  (date->int (t/today)))
+
+(defmethod resolve-keyword-input :yesterday [_ _ _]
+  (date->int (t/minus (t/today) (t/days 1))))
+
+(defmethod resolve-keyword-input :tomorrow [_ _ _]
+  (date->int (t/plus (t/today) (t/days 1))))
+
+(defmethod resolve-keyword-input :right-now-ms [_ _ _]
+  (date-time-util/time-ms))
+
+;; today-time returns an epoch int 
+(defmethod resolve-keyword-input :today-time [_db input _opts]
+  (let [[hh mm ss ms] (case input 
+                        :start-of-today-ms [0 0 0 0]
+                        :end-of-today-ms [23 59 59 999]
+                        (get-ts-units nil (subs (name input) 6)))]
+    (date-at-local-ms (t/today) hh mm ss ms))) 
+
+;; relative-date returns a YYYMMDD string 
+(defmethod resolve-keyword-input :relative-date [_ input _]
+  (let [relative-to (get-relative-date input)
+        [_ offset-direction offset offset-unit] (re-find #"^([+-])(\d+)([dwmy])$" (name input))
+        offset-date (get-offset-date relative-to offset-direction offset offset-unit)]
+    (date->int offset-date)))
+
+;; relative-date-time returns an epoch int
+(defmethod resolve-keyword-input :relative-date-time [_ input _]
+  (let [relative-to (get-relative-date input)
+        [_ offset-direction offset offset-unit ts] (re-find #"^([+-])(\d+)([dwmy])-(ms|start|end|\d{2,9})$" (name input))
+        offset-date (get-offset-date relative-to offset-direction offset offset-unit)
+        [hh mm ss ms] (get-ts-units offset-direction ts)]
+    (date-at-local-ms offset-date hh mm ss ms))) 
+
+(defmethod resolve-keyword-input :DEPRECATED-relative-date [db input opts]
+  ;; This handles all of the cases covered by the following:
+  ;; :Xd, :Xd-before, :Xd-before-ms, :Xd-after, :Xd-after-ms
+  (resolve-keyword-input db (old->new-relative-date-format input) opts))
+
+(defmethod resolve-keyword-input :default [_ _ _]
+  nil)
+
 (defn resolve-input
   "Main fn for resolving advanced query :inputs"
   [db input {:keys [current-block-uuid current-page-fn]
              :or {current-page-fn (constantly nil)}}]
   (cond
-    ;; page and block inputs
-    (= :current-page input)
-    (some-> (current-page-fn) string/lower-case)
-    (and current-block-uuid (= :current-block input))
-    (:db/id (d/entity db [:block/uuid current-block-uuid]))
-    (and current-block-uuid (= :parent-block input))
-    (:db/id (:block/parent (d/entity db [:block/uuid current-block-uuid])))
-
-    ;; journal date inputs
-    (= :today input)
-    (date->int (t/today))
-    (= :yesterday input)
-    (date->int (t/minus (t/today) (t/days 1)))
-    (= :tomorrow input)
-    (date->int (t/plus (t/today) (t/days 1)))
-    ;; e.g. :3d-before
-    (and (keyword? input)
-         (re-find #"^\d+d(-before)?$" (name input)))
-    (let [input (name input)
-          days (parse-long (re-find #"^\d+" input))]
-      (date->int (t/minus (t/today) (t/days days))))
-    ;; e.g. :3d-after
-    (and (keyword? input)
-         (re-find #"^\d+d(-after)?$" (name input)))
-    (let [input (name input)
-          days (parse-long (re-find #"^\d+" input))]
-      (date->int (t/plus (t/today) (t/days days))))
-
-    ;; timestamp inputs
-    (= :right-now-ms input) (date-time-util/time-ms)
-    (= :start-of-today-ms input) (date-at-local-ms 0 0 0 0)
-    (= :end-of-today-ms input) (date-at-local-ms 24 0 0 0)
-    ;; e.g. :3d-before-ms
-    (and (keyword? input)
-         (re-find #"^\d+d-before-ms$" (name input)))
-    (let [input (name input)
-          days (parse-long (re-find #"^\d+" input))]
-      (date-at-local-ms (t/minus (t/today) (t/days days)) 0 0 0 0))
-    ;; e.g. :3d-after-ms
-    (and (keyword? input)
-         (re-find #"^\d+d-after-ms$" (name input)))
-    (let [input (name input)
-          days (parse-long (re-find #"^\d+" input))]
-      (date-at-local-ms (t/plus (t/today) (t/days days)) 24 0 0 0))
+    (keyword? input) 
+    (or
+     (resolve-keyword-input db input {:current-block-uuid current-block-uuid
+                                      :current-page-fn current-page-fn})
+     ;; The input is returned back unresolved if a resolver communicates it
+     ;; can't resolve it by returning nil. We may want to error if this is too
+     ;; subtle for the user
+     input)
 
     (and (string? input) (page-ref/page-ref? input))
     (-> (page-ref/get-page-name input)

deps/graph-parser/yarn.lock

@@ -2,10 +2,10 @@
 # yarn lockfile v1
 
 
-"@logseq/nbb-logseq@^1.1.157":
-  version "1.1.157"
-  resolved "https://registry.yarnpkg.com/@logseq/nbb-logseq/-/nbb-logseq-1.1.157.tgz#337be95156e5b22caf5533663ae8a5a79cc43fbd"
-  integrity sha512-cuutsKZDdg850qa6HquOTKKZ9WpWUjSozRdrfvI/2WIbAv2MVQKPQYtB03K55OW9i3D1K0jAwDM0xzGI2lWyFQ==
+"@logseq/nbb-logseq@^1.1.158":
+  version "1.1.158"
+  resolved "https://registry.yarnpkg.com/@logseq/nbb-logseq/-/nbb-logseq-1.1.158.tgz#ebff10bdb0998b43e52e69ad487be04236b84569"
+  integrity sha512-NT3w5BmYBUeyOPNDc3SRNUrC4EXiY75KXiTdDS24kYaRX4UR63UZiAcCrG2GLB3jS48N2xv23dWJusHFer/+sQ==
   dependencies:
     import-meta-resolve "^2.1.0"
 

src/test/frontend/db/query_react_test.cljs

@@ -22,6 +22,24 @@ adds rules that users often use"
   (when-let [result (query-custom/custom-query test-helper/test-db query opts)]
     (map first (deref result))))
 
+(defn- blocks-created-between-inputs [a b]
+   (sort
+     (map #(-> % :block/content string/split-lines first)
+          (custom-query {:inputs [a b]
+                         :query '[:find (pull ?b [*])
+                                  :in $ ?start ?end
+                                  :where
+                                  [?b :block/content]
+                                  [?b :block/created-at ?timestamp]
+                                  [(>= ?timestamp ?start)]
+                                  [(<= ?timestamp ?end)]]}))))
+
+(defn- blocks-journaled-between-inputs [a b]
+  (map :block/content (custom-query {:inputs [a b]
+                                     :query '[:find (pull ?b [*])
+                                              :in $ ?start ?end
+                                              :where (between ?b ?start ?end)]})))
+
 (deftest resolve-input-for-page-and-block-inputs
   (load-test-files [{:file/path "pages/page1.md"
                      :file/content
@@ -39,6 +57,16 @@ adds rules that users often use"
                                         [?bp :block/name ?current-page]]}))))
       ":current-page input resolves to current page name")
 
+  (is (= []
+         (map :block/content
+              (custom-query {:inputs [:current-page]
+                             :query '[:find (pull ?b [*])
+                                      :in $ ?current-page
+                                      :where [?b :block/page ?bp]
+                                      [?bp :block/name ?current-page]]}
+                            {:current-page-fn nil})))
+      ":current-page input doesn't resolve when not present")
+
   (is (= ["child 1" "child 2"]
          (let [block-uuid (-> (db-utils/q '[:find (pull ?b [:block/uuid])
                                             :where [?b :block/content "parent"]])
@@ -51,6 +79,24 @@ adds rules that users often use"
                                         :where [?b :block/parent ?current-block]]}
                               {:current-block-uuid block-uuid}))))
       ":current-block input resolves to current block's :db/id")
+
+  (is (= []
+         (map :block/content
+              (custom-query {:inputs [:current-block]
+                             :query '[:find (pull ?b [*])
+                                      :in $ ?current-block
+                                      :where [?b :block/parent ?current-block]]})))
+      ":current-block input doesn't resolve when current-block-uuid is not provided")
+
+  (is (= []
+         (map :block/content
+              (custom-query {:inputs [:current-block]
+                             :query '[:find (pull ?b [*])
+                                      :in $ ?current-block
+                                      :where [?b :block/parent ?current-block]]}
+                            {:current-block-uuid :magic})))
+      ":current-block input doesn't resolve when current-block-uuid is invalid")
+
   (is (= ["parent"]
          (let [block-uuid (-> (db-utils/q '[:find (pull ?b [:block/uuid])
                                             :where [?b :block/content "child 1"]])
@@ -91,41 +137,136 @@ adds rules that users often use"
 ;; These tests rely on seeding timestamps with properties. If this ability goes
 ;; away we could still test page-level timestamps
 (deftest resolve-input-for-timestamp-inputs
-  (let [today-timestamp (db-util/date-at-local-ms 0 0 0 0)
-        next-week-timestamp (db-util/date-at-local-ms (t/plus (t/today) (t/days 7))
-                                                      0 0 0 0)]
-    (load-test-files [{:file/path "pages/page1.md"
-                       :file/content (gstring/format "foo::bar
-- yesterday
+  (load-test-files [{:file/path "pages/page1.md"
+                     :file/content (gstring/format "foo::bar
+- -1y
+created-at:: %s
+- -1m
+created-at:: %s
+- -1w
+created-at:: %s
+- -1d
 created-at:: %s
 - today
 created-at:: %s
-- next week
+- tonight
+created-at:: %s
+- +1d
+created-at:: %s
+- +1w
+created-at:: %s
+- +1m
+created-at:: %s
+- +1y
 created-at:: %s"
-                                                     (dec today-timestamp)
-                                                     (inc today-timestamp)
-                                                     next-week-timestamp)}])
+                                                   (db-util/date-at-local-ms (t/minus (t/today) (t/years 1)) 0 0 0 0)
+                                                   (db-util/date-at-local-ms (t/minus (t/today) (t/months 1)) 0 0 0 0)
+                                                   (db-util/date-at-local-ms (t/minus (t/today) (t/weeks 1)) 0 0 0 0)
+                                                   (db-util/date-at-local-ms (t/minus (t/today) (t/days 1)) 0 0 0 0)
+                                                   (db-util/date-at-local-ms (t/today) 12 0 0 0)
+                                                   (db-util/date-at-local-ms (t/today) 18 0 0 0)
+                                                   (db-util/date-at-local-ms (t/plus (t/today) (t/days 1)) 0 0 0 0)
+                                                   (db-util/date-at-local-ms (t/plus (t/today) (t/weeks 1)) 0 0 0 0)
+                                                   (db-util/date-at-local-ms (t/plus (t/today) (t/months 1)) 0 0 0 0)
+                                                   (db-util/date-at-local-ms (t/plus (t/today) (t/years 1)) 0 0 0 0))}])
+
+  (is (= ["today" "tonight"] (blocks-created-between-inputs :-0d-ms :+0d-ms))
+      ":+0d-ms and :-0d-ms resolve to correct datetime range")
+
+  (is (= ["+1d" "-1d" "today" "tonight"] (blocks-created-between-inputs :-1d-ms :+5d-ms))
+      ":-Xd-ms and :+Xd-ms resolve to correct datetime range")
+
+  (is (= ["+1d" "+1w" "-1d" "-1w" "today" "tonight"] (blocks-created-between-inputs :-1w-ms :+1w-ms))
+      ":-Xw-ms and :+Xw-ms resolve to correct datetime range")
+
+  (is (= ["+1d" "+1m" "+1w" "-1d" "-1m" "-1w" "today" "tonight"] (blocks-created-between-inputs :-1m-ms :+1m-ms))
+      ":-Xm-ms and :+Xm-ms resolve to correct datetime range")
+
+  (is (= ["+1d" "+1m" "+1w" "+1y" "-1d" "-1m" "-1w" "-1y" "today" "tonight"] (blocks-created-between-inputs :-1y-ms :+1y-ms))
+      ":-Xy-ms and :+Xy-ms resolve to correct datetime range")
+
+  (is (= ["today" "tonight"] (blocks-created-between-inputs :start-of-today-ms :end-of-today-ms))
+      ":start-of-today-ms and :end-of-today-ms resolve to correct datetime range")
+
+  (is (= ["+1d" "-1d" "today" "tonight"] (blocks-created-between-inputs :1d-before-ms :5d-after-ms)) 
+      ":Xd-before-ms and :Xd-after-ms resolve to correct datetime range")
+
+  (is (= ["today" "tonight"] (blocks-created-between-inputs :today-start :today-end))
+      ":today-start and :today-end resolve to correct datetime range")
+
+  (is (= ["+1d" "today" "tonight"] (blocks-created-between-inputs :-0d-start :+1d-end))
+      ":-XT-start and :+XT-end resolve to correct datetime range")
+
+  (is (= ["today"] (blocks-created-between-inputs :today-1159 :today-1201))
+      ":today-HHMM and :today-HHMM resolve to correct datetime range")
+
+  (is (= ["today"] (blocks-created-between-inputs :today-115959 :today-120001))
+      ":today-HHMMSS and :today-HHMMSS resolve to correct datetime range")
+
+  (is (= ["today"] (blocks-created-between-inputs :today-115959999 :today-120000001))
+      ":today-HHMMSSmmm and :today-HHMMSSmmm resolve to correct datetime range")
+
+  (is (= ["today" "tonight"] (blocks-created-between-inputs :today-1199 :today-9901))
+      ":today-HHMM and :today-HHMM resolve to valid datetime ranges")
+
+  (is (= ["+1d" "tonight"] (blocks-created-between-inputs :-0d-1201 :+1d-2359))
+      ":-XT-HHMM and :+XT-HHMM resolve to correct datetime range")
+
+  (is (= ["+1d" "tonight"] (blocks-created-between-inputs :-0d-120001 :+1d-235959))
+      ":-XT-HHMMSS and :+XT-HHMMSS resolve to correct datetime range")
+
+  (is (= ["+1d" "tonight"] (blocks-created-between-inputs :-0d-120000001 :+1d-235959999))
+      ":-XT-HHMMSSmmm and :+XT-HHMMSSmmm resolve to correct datetime range")
+
+  (is (= ["+1d" "tonight"] (blocks-created-between-inputs :-0d-1201 :+1d-2359))
+      ":-XT-HHMM and :+XT-HHMM resolve to correct datetime range")
+
+  (is (= [] (blocks-created-between-inputs :-0d-abcd :+1d-23.45))
+      ":-XT-HHMM and :+XT-HHMM will not reoslve with invalid time formats but will fail gracefully")) 
+        
+
+(deftest resolve-input-for-relative-date-queries 
+  (load-test-files [{:file/content "- -1y" :file/path "journals/2022_01_01.md"}
+                    {:file/content "- -1m" :file/path "journals/2022_12_01.md"}
+                    {:file/content "- -1w" :file/path "journals/2022_12_25.md"}
+                    {:file/content "- -1d" :file/path "journals/2022_12_31.md"}
+                    {:file/content "- now" :file/path "journals/2023_01_01.md"}
+                    {:file/content "- +1d" :file/path "journals/2023_01_02.md"}
+                    {:file/content "- +1w" :file/path "journals/2023_01_08.md"}
+                    {:file/content "- +1m" :file/path "journals/2023_02_01.md"}
+                    {:file/content "- +1y" :file/path "journals/2024_01_01.md"}])
+
+  (with-redefs [t/today (constantly (t/date-time 2023 1 1))]
+    (is (= ["now" "-1d" "-1w" "-1m" "-1y"] (blocks-journaled-between-inputs :-365d :today))
+        ":-365d and today resolve to correct journal range")
+    
+    (is (= ["now" "-1d" "-1w" "-1m" "-1y"] (blocks-journaled-between-inputs :-1y :today))
+        ":-1y and today resolve to correct journal range")
+    
+    (is (= ["now" "-1d" "-1w" "-1m"] (blocks-journaled-between-inputs :-1m :today))
+        ":-1m and today resolve to correct journal range")
+
+    (is (= ["now" "-1d" "-1w"] (blocks-journaled-between-inputs :-1w :today))
+        ":-1w and today resolve to correct journal range")
+
+    (is (= ["now" "-1d"] (blocks-journaled-between-inputs :-1d :today))
+        ":-1d and today resolve to correct journal range")
+
+    (is (= ["+1y" "+1m" "+1w" "+1d" "now"] (blocks-journaled-between-inputs :today :+365d))
+        ":+365d and today resolve to correct journal range")
+    
+    (is (= ["+1y" "+1m" "+1w" "+1d" "now"] (blocks-journaled-between-inputs :today :+1y))
+        ":+1y and today resolve to correct journal range")
+    
+    (is (= ["+1m" "+1w" "+1d" "now"] (blocks-journaled-between-inputs :today :+1m))
+        ":+1m and today resolve to correct journal range")
+
+    (is (= ["+1w" "+1d" "now"] (blocks-journaled-between-inputs :today :+1w))
+        ":+1w and today resolve to correct journal range")
+
+    (is (= ["+1d" "now"] (blocks-journaled-between-inputs :today :+1d))
+        ":+1d and today resolve to correct journal range")
+
+    (is (= ["+1d" "now"] (blocks-journaled-between-inputs :today :today/+1d))
+        ":today/+1d and today resolve to correct journal range")))
 
-    (is (= ["today"]
-           (map #(-> % :block/content string/split-lines first)
-                (custom-query {:inputs [:start-of-today-ms :end-of-today-ms]
-                               :query '[:find (pull ?b [*])
-                                        :in $ ?start ?end
-                                        :where
-                                        [?b :block/content]
-                                        [?b :block/created-at ?timestamp]
-                                        [(>= ?timestamp ?start)]
-                                        [(<= ?timestamp ?end)]]})))
-        ":start-of-today-ms and :end-of-today-ms resolve to correct datetime range")
-
-    (is (= ["yesterday" "today"]
-           (map #(-> % :block/content string/split-lines first)
-                (custom-query {:inputs [:1d-before-ms :5d-after-ms]
-                               :query '[:find (pull ?b [*])
-                                        :in $ ?start ?end
-                                        :where
-                                        [?b :block/content]
-                                        [?b :block/created-at ?timestamp]
-                                        [(>= ?timestamp ?start)]
-                                        [(<= ?timestamp ?end)]]})))
-        ":Xd-before-ms and :Xd-after-ms resolve to correct datetime range")))