Convert remaining query operators to use rules

- Removed counter and uniq-symbol as those are no longer needed
- build-query cleanly maps conditions to their fns
- Query dsl tests have been fully decoupled

src/main/frontend/db/query_dsl.cljs

@@ -2,7 +2,6 @@
   (:require [cljs-time.coerce :as tc]
             [cljs-time.core :as t]
             [cljs.reader :as reader]
-            [clojure.core]
             [clojure.set :as set]
             [clojure.string :as string]
             [clojure.walk :as walk]
@@ -44,25 +43,10 @@
 
 ;; (sort-by last_modified_at asc)
 
-(defonce remove-nil? (partial remove nil?))
-
-(defn query-wrapper
-  [where blocks?]
-  (when where
-    (let [q (if blocks?                   ; FIXME: it doesn't need to be either blocks or pages
-              `[:find (~'pull ~'?b ~model/block-attrs)
-                :in ~'$ ~'%
-                :where]
-              '[:find (pull ?p [*])
-                :in $ %
-                :where])
-          result (if (coll? (first where))
-                   (apply conj q where)
-                   (conj q where))]
-      (prn "Datascript query: " result)
-      result)))
-
 ;; (between -7d +7d)
+
+;; Time helpers
+;; ============
 (defn- ->journal-day-int [input]
   (let [input (string/lower-case (name input))]
     (cond
@@ -126,13 +110,8 @@
                  t/days)]
         (tc/to-long (t/plus (t/today) (tf duration)))))))
 
-(defn uniq-symbol
-  [counter prefix]
-  (let [result (symbol (str prefix (when-not (zero? @counter)
-                                     @counter)))]
-    (swap! counter inc)
-    result))
-
+;; Boolean operator utils: and, or, not
+;; ======================
 (defn- collect-vars
   [l]
   (let [vars (atom #{})]
@@ -144,34 +123,6 @@
      l)
     @vars))
 
-;; TODO: Convert ->*query fns to rules
-(defn ->property-query
-  ([k v]
-   (->property-query k v '?v))
-  ([k v sym]
-   [['?b :block/properties '?prop]
-    [(list 'missing? '$ '?b :block/name)]
-    [(list 'get '?prop (keyword k)) sym]
-    (list
-     'or
-     [(list '= sym v)]
-     [(list 'contains? sym v)]
-     ;; For integer pages that aren't strings
-     [(list 'contains? sym (str v))])]))
-
-(defn- build-property-two-arg
-  [e current-filter counter]
-  (let [k (string/replace (name (nth e 1)) "_" "-")
-        v (nth e 2)
-        v (if-not (nil? v)
-            (text/parse-property k v)
-            v)
-        v (if (coll? v) (first v) v)
-        sym (if (= current-filter 'or)
-              '?v
-              (uniq-symbol counter "?v"))]
-    {:query (->property-query k v sym)}))
-
 (defn- build-and-or-not-result
   [fe clauses current-filter nested-and?]
   (cond
@@ -215,6 +166,8 @@
 
 (declare build-query)
 
+(defonce remove-nil? (partial remove nil?))
+
 (defn- build-and-or-not
   [repo e {:keys [current-filter vars] :as env} level fe]
   (let [raw-clauses (map (fn [form]
@@ -253,6 +206,8 @@
         {:query query
          :rules (distinct (mapcat :rules raw-clauses))}))))
 
+;; build-query fns
+;; ===============
 (defn- build-between-two-arg
   [e]
   (let [start (->journal-day-int (nth e 1))
@@ -278,12 +233,41 @@
                      [(list '>= sym start)]
                      [(list '< sym end)]]}))))))
 
+(defn- build-between
+  [e]
+  (cond
+    (= 3 (count e))
+    (build-between-two-arg e)
+
+    ;; (between created_at -1d today)
+    (= 4 (count e))
+    (build-between-three-arg e)))
+
+(defn- build-property-two-arg
+  [e]
+  (let [k (string/replace (name (nth e 1)) "_" "-")
+        v (nth e 2)
+        v (if-not (nil? v)
+            (text/parse-property k v)
+            v)
+        v (if (coll? v) (first v) v)]
+    {:query (list 'property '?b (keyword k) v)
+     :rules [:property]}))
+
 (defn- build-property-one-arg
   [e]
   (let [k (string/replace (name (nth e 1)) "_" "-")]
     {:query (list 'has-property '?b (keyword k))
      :rules [:has-property]}))
 
+(defn- build-property [e]
+  (cond
+    (= 3 (count e))
+    (build-property-two-arg e)
+
+    (= 2 (count e))
+    (build-property-one-arg e)))
+
 (defn- build-task
   [e]
   (let [markers (if (coll? (first (rest e)))
@@ -340,7 +324,7 @@
     nil))
 
 (defn- build-sort-by
-  [e sort-by]
+  [e sort-by_]
   (let [[k order] (rest e)
              order (if (and order (contains? #{:asc :desc}
                                              (keyword (string/lower-case (name order)))))
@@ -358,11 +342,11 @@
                          :else
                          #(get-in % [:block/properties k]))
              comp (if (= order :desc) >= <=)]
-         (reset! sort-by
+         (reset! sort-by_
                  (fn [result]
                    (->> result
                         flatten
-                        (clojure.core/sort-by get-value comp))))
+                        (sort-by get-value comp))))
          nil))
 
 (defn- build-page
@@ -384,23 +368,25 @@
   [e]
   (let [page-name (-> (text/page-ref-un-brackets! e)
                       (util/page-name-sanity-lc))]
-    {:query [['?b :block/path-refs [:block/name page-name]]]}))
+    {:query (list 'page-ref '?b page-name)
+     :rules [:page-ref]}))
 
 (defn- build-block-content [e]
   {:query (list 'block-content '?b e)
    :rules [:block-content]})
 
 (defn build-query
-  "This fn converts a list in a query e.g. `(operator arg1 arg2)` to its datalog
+  "This fn converts a form/list in a query e.g. `(operator arg1 arg2)` to its datalog
   equivalent. This fn is called recursively on sublists for boolean operators
-  `and`, `or` and `not`. Some bindings in this fn:
+  `and`, `or` and `not`. This fn should return a map with :query and :rules or nil.
+
+Some bindings in this fn:
 
 * e - the list being processed
 * fe - the query operator e.g. `property`"
   ([repo e env]
    (build-query repo e (assoc env :vars (atom {})) 0))
-  ([repo e {:keys [sort-by blocks? sample counter current-filter] :as env} level]
-   ;; TODO: replace with multi-methods for extensibility.
+  ([repo e {:keys [sort-by blocks? sample] :as env} level]
    (let [fe (first e)
          fe (when fe (symbol (string/lower-case (name fe))))
          page-ref? (text/page-ref? e)]
@@ -422,22 +408,11 @@
        (contains? #{'and 'or 'not} fe)
        (build-and-or-not repo e env level fe)
 
-       (and (= 'between fe)
-            (= 3 (count e)))
-       (build-between-two-arg e)
-
-       ;; (between created_at -1d today)
-       (and (= 'between fe)
-            (= 4 (count e)))
-       (build-between-three-arg e)
+       (= 'between fe)
+       (build-between e)
 
-       (and (= 'property fe)
-            (= 3 (count e)))
-       (build-property-two-arg e current-filter counter)
-
-       (and (= 'property fe)
-            (= 2 (count e)))
-       (build-property-one-arg e)
+       (= 'property fe)
+       (build-property e)
 
        ;; task is the new name and todo is the old one
        (or (= 'todo fe) (= 'task fe))
@@ -470,6 +445,9 @@
        :else
        nil))))
 
+;; parse fns
+;; =========
+
 (defn- pre-transform
   [s]
   (some-> s
@@ -528,47 +506,64 @@
   [repo s]
   (when (and (string? s)
              (not (string/blank? s)))
-    (let [counter (atom 0)]
-      (try
-        (let [s (if (= \# (first s)) (util/format "[[%s]]" (subs s 1)) s)
-              form (some-> s
-                           (pre-transform)
-                           (reader/read-string))]
-          (if (symbol? form)
-            (str form)
-            (let [sort-by (atom nil)
-                  blocks? (atom nil)
-                  sample (atom nil)
-                  {result :query rules :rules}
-                  (when form (build-query repo form {:sort-by sort-by
-                                                     :blocks? blocks?
-                                                     :counter counter
-                                                     :sample sample}))]
-              (cond
-                (and (nil? result) (string? form))
-                form
-
-                (string? result)
-                result
-
-                :else
-                (let [result (when (seq result)
-                               (let [key (if (coll? (first result))
-                                           (keyword (ffirst result))
-                                           (keyword (first result)))
-                                     result (case key
-                                              :and
-                                              (rest result)
-
-                                              result)]
-                                 (add-bindings! form result)))]
-                  {:query result
-                   :rules (mapv rules/query-dsl-rules rules)
-                   :sort-by @sort-by
-                   :blocks? (boolean @blocks?)
-                   :sample sample})))))
-        (catch js/Error e
-          (log/error :query-dsl/parse-error e))))))
+    (try
+      (let [s (if (= \# (first s)) (util/format "[[%s]]" (subs s 1)) s)
+            form (some-> s
+                         (pre-transform)
+                         (reader/read-string))]
+        (if (symbol? form)
+          (str form)
+          (let [sort-by (atom nil)
+                blocks? (atom nil)
+                sample (atom nil)
+                {result :query rules :rules}
+                (when form (build-query repo form {:sort-by sort-by
+                                                   :blocks? blocks?
+                                                   :sample sample}))]
+            (cond
+              (and (nil? result) (string? form))
+              form
+
+              (string? result)
+              result
+
+              :else
+              (let [result (when (seq result)
+                             (let [key (if (coll? (first result))
+                                         (keyword (ffirst result))
+                                         (keyword (first result)))
+                                   result (case key
+                                            :and
+                                            (rest result)
+
+                                            result)]
+                               (add-bindings! form result)))]
+                {:query result
+                 :rules (mapv rules/query-dsl-rules rules)
+                 :sort-by @sort-by
+                 :blocks? (boolean @blocks?)
+                 :sample sample})))))
+      (catch js/Error e
+        (log/error :query-dsl/parse-error e)))))
+
+;; Main fns
+;; ========
+
+(defn query-wrapper
+  [where blocks?]
+  (when where
+    (let [q (if blocks?                   ; FIXME: it doesn't need to be either blocks or pages
+              `[:find (~'pull ~'?b ~model/block-attrs)
+                :in ~'$ ~'%
+                :where]
+              '[:find (pull ?p [*])
+                :in $ %
+                :where])
+          result (if (coll? (first where))
+                   (apply conj q where)
+                   (conj q where))]
+      (prn "Datascript query: " result)
+      result)))
 
 (defn query
   ([query-string]

src/main/frontend/db/rules.cljs

@@ -121,4 +121,21 @@
    :namespace
    '[(namespace ?p ?namespace)
      [?p :block/namespace ?parent]
-     [?parent :block/name ?namespace]]})
+     [?parent :block/name ?namespace]]
+
+   :property
+   '[(property ?b ?key ?val)
+     [?b :block/properties ?prop]
+     [(missing? $ ?b :block/name)]
+     [(get ?prop ?key) ?v]
+     (or-join [?v]
+              [(= ?v ?val)]
+              [(contains? ?v ?val)]
+              ;; For integer pages that aren't strings
+              (and
+               [(str ?val) ?str-val]
+               [(contains? ?v ?str-val)]))]
+
+   :page-ref
+   '[(page-ref ?b ?page-name)
+     [?b :block/path-refs [:block/name ?page-name]]]})

src/test/frontend/db/query_dsl_test.cljs

@@ -64,9 +64,9 @@ prop-d:: [[no-space-link]]
 prop-d:: nada"}])
 
   (testing "Blocks have given property value"
-    (is (= ["b1" "b2"]
-           (map (comp first str/split-lines :block/content)
-                (dsl-query "(property prop-a val-a)"))))
+    (is (= #{"b1" "b2"}
+           (set (map (comp first str/split-lines :block/content)
+                 (dsl-query "(property prop-a val-a)")))))
 
     (is (= ["b2"]
            (map (comp first str/split-lines :block/content)
@@ -345,19 +345,52 @@ tags: other
            "[[page-not-exist]]" empty-result
            "[[another-page-not-exist]]" empty-result))))
 
-(defn- load-test-files-for-parse
+(deftest page-ref-and-boolean-queries
+  (load-test-files [{:file/path "pages/page1.md"
+                     :file/content "foo:: bar
+- b1 [[page 1]] #tag2
+- b2 [[page 2]] #tag1
+- b3"}])
+
+  (testing "page-ref queries"
+
+    (is (= ["b2 [[page 2]] #tag1"]
+           (map :block/content (dsl-query "[[page 2]]"))))
+
+    (is (= []
+           (map :block/content (dsl-query "[[blarg]]")))
+        "Correctly returns no results"))
+
+  (testing "basic boolean queries"
+    (is (= ["b2 [[page 2]] #tag1"]
+           (map :block/content
+                (dsl-query "(and [[tag1]] [[page 2]])")))
+        "AND query")
+
+    (is (= ["b1 [[page 1]] #tag2" "b2 [[page 2]] #tag1"]
+           (map :block/content
+                (dsl-query "(or [[tag2]] [[page 2]])")))
+        "OR query")
+
+    (is (= ["foo:: bar\n" "b1 [[page 1]] #tag2" "b3"]
+           (map :block/content
+                ;; ANDed page1 to not clutter results with blocks in default db
+                (dsl-query "(and (page page1) (not [[page 2]]))")))
+        "NOT query")))
+
+(defn- load-test-files-with-timestamps
   []
   (let [files [{:file/path "journals/2020_12_26.md"
                 :file/content "---
 title: Dec 26th, 2020
 ---
-- DONE 26-b1 [[page 1]]
+- DONE 26-b1
 created-at:: 1608968448113
 last-modified-at:: 1608968448113
-- LATER 26-b2-modified-later [[page 2]] #tag1
+- LATER 26-b2-modified-later
 created-at:: 1608968448114
 last-modified-at:: 1608968448120
-- DONE [#A] 26-b3 [[page 1]]
+- DONE 26-b3
 created-at:: 1608968448115
 last-modified-at:: 1608968448115
 "}
@@ -365,16 +398,16 @@ last-modified-at:: 1608968448115
                 :file/content "---
 title: Dec 27th, 2020
 ---
-- NOW [#A] b1 [[page 1]]
+- NOW b1
 created-at:: 1609052958714
 last-modified-at:: 1609052958714
-- LATER [#B] b2-modified-later [[page 2]]
+- LATER b2-modified-later
 created-at:: 1609052959376
 last-modified-at:: 1609052974285
-- b3 [[page 1]]
+- b3
 created-at:: 1609052959954
 last-modified-at:: 1609052959954
-- b4 [[page 2]]
+- b4
 created-at:: 1609052961569
 last-modified-at:: 1609052961569
 - b5
@@ -384,80 +417,40 @@ last-modified-at:: 1609052963089"}
                 :file/content "---
 title: Dec 28th, 2020
 ---
-- 28-b1 [[page 1]]
+- 28-b1
 created-at:: 1609084800000
 last-modified-at:: 1609084800000
-- 28-b2-modified-later [[page 2]]
+- 28-b2-modified-later
 created-at:: 1609084800001
 last-modified-at:: 1609084800020
-- 28-b3 [[page 1]]
+- 28-b3
 created-at:: 1609084800002
 last-modified-at:: 1609084800002"}]]
-    (repo-handler/parse-files-and-load-to-db! test-db files {:re-render? false})))
-
-(deftest test-parse
-  (load-test-files-for-parse)
-
-  (testing "Single page query"
-    (are [x y] (= (q-count x) y)
-         "[[page 1]]"
-         {:query '[[?b :block/path-refs [:block/name "page 1"]]]
-          :count 6}
-
-         "[[page 2]]"
-         {:query '[[?b :block/path-refs [:block/name "page 2"]]]
-          :count 4}))
-
-  ;; boolean queries
-  (testing "AND queries"
-    (are [x y] (= (q-count x) y)
-         "(and [[tag1]] [[page 2]])"
-         {:query '([?b :block/path-refs [:block/name "tag1"]]
-                   [?b :block/path-refs [:block/name "page 2"]])
-          :count 1})
-
-    (are [x y] (= (q-count x) y)
-         "(and [[tag1]] [[page 2]])"
-         {:query '([?b :block/path-refs [:block/name "tag1"]]
-                   [?b :block/path-refs [:block/name "page 2"]])
-          :count 1}))
-
-  (testing "OR queries"
-    (are [x y] (= (q-count x) y)
-         "(or [[tag1]] [[page 2]])"
-         {:query '(or
-                   (and [?b :block/path-refs [:block/name "tag1"]])
-                   (and [?b :block/path-refs [:block/name "page 2"]]))
-          :count 4}))
-
-  (testing "NOT queries"
-    (are [x y] (= (q-count x) y)
-         "(not [[page 1]])"
-         {:query '([?b :block/uuid]
-                   (not [?b :block/path-refs [:block/name "page 1"]]))
-          :count 28}))
-
-  (testing "Between query"
-    (are [x y] (= (count-only x) y)
-         "(and (task now later done) (between [[Dec 26th, 2020]] tomorrow))"
-         5
-
-         ;; between with journal pages
-         "(and (task now later done) (between [[Dec 27th, 2020]] [[Dec 28th, 2020]]))"
-         2
-
-         ;; ;; between with created-at
-         ;; "(and (task now later done) (between created-at [[Dec 26th, 2020]] tomorrow))"
-         ;; 5
-
-         ;; ;; between with last-modified-at
-         ;; "(and (task now later done) (between last-modified-at [[Dec 26th, 2020]] tomorrow))"
-         ;; 5
-         ))
+    (load-test-files files)))
+
+(deftest between-queries
+  (load-test-files-with-timestamps)
+
+  (are [x y] (= (count-only x) y)
+       "(and (task now later done) (between [[Dec 26th, 2020]] tomorrow))"
+       5
+
+       ;; between with journal pages
+       "(and (task now later done) (between [[Dec 27th, 2020]] [[Dec 28th, 2020]]))"
+       2
+
+       ;; ;; between with created-at
+       ;; "(and (task now later done) (between created-at [[Dec 26th, 2020]] tomorrow))"
+       ;; 5
+
+       ;; ;; between with last-modified-at
+       ;; "(and (task now later done) (between last-modified-at [[Dec 26th, 2020]] tomorrow))"
+       ;; 5
+       )
   )
 
 #_(deftest sort-by-queries
-    (load-test-files-for-parse)
+    (load-test-files-with-timestamps)
     ;; (testing "sort-by (created-at defaults to desc)"
     ;;   (db/clear-query-state!)
     ;;   (let [result (->> (q "(and (task now later done)
@@ -528,9 +521,9 @@ last-modified-at:: 1609084800002"}]]
 (comment
  (require '[clojure.pprint :as pprint])
  (config/start-test-db!)
- (import-test-data!)
+ (load-test-files-with-timestamps)
 
- (dsl/query test-db "(all-page-tags)")
+ (dsl/query test-db "(task done)")
 
  ;; Useful for debugging
  (prn