More query dsl improvements

- Add tests for 5 operators that weren't tested
- Converted all operators to rules except two

src/main/frontend/db/query_dsl.cljs

@@ -159,7 +159,7 @@
      ;; For integer pages that aren't strings
      [(list 'contains? sym (str v))])]))
 
-(defn- build-query-property-two-arg
+(defn- build-property-two-arg
   [e current-filter counter]
   (let [k (string/replace (name (nth e 1)) "_" "-")
         v (nth e 2)
@@ -170,9 +170,9 @@
         sym (if (= current-filter 'or)
               '?v
               (uniq-symbol counter "?v"))]
-    (->property-query k v sym)))
+    {:query (->property-query k v sym)}))
 
-(defn- build-query-and-or-not-result
+(defn- build-and-or-not-result
   [fe clauses current-filter nested-and?]
   (cond
     (= fe 'not)
@@ -215,7 +215,7 @@
 
 (declare build-query)
 
-(defn- build-query-and-or-not
+(defn- build-and-or-not
   [repo e {:keys [current-filter vars] :as env} level fe]
   (let [raw-clauses (map (fn [form]
                            (build-query repo form (assoc env :current-filter fe) (inc level)))
@@ -226,7 +226,7 @@
                      (distinct))
         nested-and? (and (= fe 'and) (= current-filter 'and))]
     (when (seq clauses)
-      (let [result (build-query-and-or-not-result
+      (let [result (build-and-or-not-result
                     fe clauses current-filter nested-and?)
             vars' (set/union (set @vars) (collect-vars result))
             query (cond
@@ -253,26 +253,38 @@
         {:query query
          :rules (distinct (mapcat :rules raw-clauses))}))))
 
-(defn- build-query-between-two-arg
+(defn- build-between-two-arg
   [e]
   (let [start (->journal-day-int (nth e 1))
          end (->journal-day-int (nth e 2))
          [start end] (sort [start end])]
-    [['?b :block/page '?p]
-     ['?p :block/journal? true]
-     ['?p :block/journal-day '?d]
-     [(list '>= '?d start)]
-     [(list '<= '?d end)]]))
+    {:query (list 'between '?b start end)
+     :rules [:between]}))
 
-(defn- build-query-property-one-arg
+(defn- build-between-three-arg
+  [e]
+  (let [k (-> (second e)
+              (name)
+              (string/lower-case)
+              (string/replace "-" "_"))]
+    (when (contains? #{"created_at" "last_modified_at"} k)
+      (let [start (->timestamp (nth e 2))
+             end (->timestamp (nth e 3))]
+        (when (and start end)
+          (let [[start end] (sort [start end])
+                sym '?v]
+            {:query [['?b :block/properties '?prop]
+                     [(list 'get '?prop k) sym]
+                     [(list '>= sym start)]
+                     [(list '< sym end)]]}))))))
+
+(defn- build-property-one-arg
   [e]
   (let [k (string/replace (name (nth e 1)) "_" "-")]
-    [['?b :block/properties '?prop]
-     [(list 'missing? '$ '?b :block/name)]
-     [(list 'get '?prop (keyword k)) '?prop-v]
-     [true]]))
+    {:query (list 'has-property '?b (keyword k))
+     :rules [:has-property]}))
 
-(defn- build-query-todo
+(defn- build-task
   [e]
   (let [markers (if (coll? (first (rest e)))
                   (first (rest e))
@@ -282,7 +294,7 @@
         {:query (list 'task '?b markers)
          :rules [:task]}))))
 
-(defn- build-query-priority
+(defn- build-priority
   [e]
   (let [priorities (if (coll? (first (rest e)))
                      (first (rest e))
@@ -304,7 +316,7 @@
       {:query (list 'has-page-property '?p (keyword k))
        :rules [:has-page-property]})))
 
-(defn- build-query-page-tags
+(defn- build-page-tags
   [e]
   (let [tags (if (coll? (first (rest e)))
                (first (rest e))
@@ -315,7 +327,19 @@
         {:query (list 'page-tags '?p tags)
          :rules [:page-tags]}))))
 
-(defn- build-query-sort-by
+(defn- build-all-page-tags
+  []
+  {:query (list 'all-page-tags '?p)
+   :rules [:all-page-tags]} )
+
+(defn- build-sample
+  [e sample]
+  (when-let [num (second e)]
+    (when (integer? num)
+      (reset! sample num))
+    nil))
+
+(defn- build-sort-by
   [e sort-by]
   (let [[k order] (rest e)
              order (if (and order (contains? #{:asc :desc}
@@ -341,7 +365,38 @@
                         (clojure.core/sort-by get-value comp))))
          nil))
 
-(defn ^:large-vars/cleanup-todo build-query
+(defn- build-page
+  [e]
+  (let [page-name (text/page-ref-un-brackets! (str (first (rest e))))
+        page-name (util/page-name-sanity-lc page-name)]
+    {:query (list 'page '?b page-name)
+     :rules [:page]}))
+
+(defn- build-namespace
+  [e]
+  (let [page-name (text/page-ref-un-brackets! (str (first (rest e))))
+        page (util/page-name-sanity-lc page-name)]
+    (when-not (string/blank? page)
+      {:query (list 'namespace '?p page)
+       :rules [:namespace]})))
+
+(defn- build-page-ref
+  [e]
+  (let [page-name (-> (text/page-ref-un-brackets! e)
+                      (util/page-name-sanity-lc))]
+    {:query [['?b :block/path-refs [:block/name page-name]]]}))
+
+(defn- build-block-content [e]
+  {:query (list 'block-content '?b e)
+   :rules [:block-content]})
+
+(defn build-query
+  "This fn converts a list in a query e.g. `(operator arg1 arg2)` to its datalog
+  equivalent. This fn is called recursively on sublists for boolean operators
+  `and`, `or` and `not`. Some bindings in this fn:
+
+* e - the list being processed
+* fe - the query operator e.g. `property`"
   ([repo e env]
    (build-query repo e (assoc env :vars (atom {})) 0))
   ([repo e {:keys [sort-by blocks? sample counter current-filter] :as env} level]
@@ -351,94 +406,66 @@
          page-ref? (text/page-ref? e)]
      (when (or (and page-ref?
                     (not (contains? #{'page-property 'page-tags} (:current-filter env))))
-               (contains? #{'between 'property 'todo 'task 'priority 'sort-by 'page} fe))
+               (contains? #{'between 'property 'todo 'task 'priority 'sort-by 'page} fe)
+               (and (not page-ref?) (string? e)))
        (reset! blocks? true))
      (cond
        (nil? e)
        nil
 
        page-ref?
-       (let [page-name (-> (text/page-ref-un-brackets! e)
-                           (util/page-name-sanity-lc))]
-         {:query [['?b :block/path-refs [:block/name page-name]]]})
+       (build-page-ref e)
 
        (string? e)                      ; block content full-text search, could be slow
-       (do
-         (reset! blocks? true)
-         {:query
-          [['?b :block/content '?content]
-           [(list 'clojure.string/includes? '?content e)]]})
+       (build-block-content e)
 
        (contains? #{'and 'or 'not} fe)
-       (build-query-and-or-not repo e env level fe)
+       (build-and-or-not repo e env level fe)
 
        (and (= 'between fe)
             (= 3 (count e)))
-       {:query (build-query-between-two-arg e)}
+       (build-between-two-arg e)
 
        ;; (between created_at -1d today)
        (and (= 'between fe)
             (= 4 (count e)))
-       (let [k (-> (second e)
-                   (name)
-                   (string/lower-case)
-                   (string/replace "-" "_"))]
-         (when (contains? #{"created_at" "last_modified_at"} k)
-           (let [start (->timestamp (nth e 2))
-                  end (->timestamp (nth e 3))]
-             (when (and start end)
-               (let [[start end] (sort [start end])
-                     sym '?v]
-                 {:query [['?b :block/properties '?prop]
-                          [(list 'get '?prop k) sym]
-                          [(list '>= sym start)]
-                          [(list '< sym end)]]})))))
+       (build-between-three-arg e)
 
        (and (= 'property fe)
             (= 3 (count e)))
-       {:query (build-query-property-two-arg e current-filter counter)}
+       (build-property-two-arg e current-filter counter)
 
        (and (= 'property fe)
             (= 2 (count e)))
-       {:query (build-query-property-one-arg e)}
+       (build-property-one-arg e)
 
+       ;; task is the new name and todo is the old one
        (or (= 'todo fe) (= 'task fe))
-       (build-query-todo e)
+       (build-task e)
 
        (= 'priority fe)
-       (build-query-priority e)
+       (build-priority e)
 
        (= 'sort-by fe)
-       (build-query-sort-by e sort-by)
+       (build-sort-by e sort-by)
 
        (= 'page fe)
-       (let [page-name (text/page-ref-un-brackets! (str (first (rest e))))
-             page-name (util/page-name-sanity-lc page-name)]
-         {:query [['?b :block/page [:block/name page-name]]]})
+       (build-page e)
 
-       (and (= 'namespace fe)
-            (= 2 (count e)))
-       (let [page-name (text/page-ref-un-brackets! (str (first (rest e))))
-             page (util/page-name-sanity-lc page-name)]
-         (when-not (string/blank? page)
-           {:query [['?p :block/namespace '?parent]
-                    ['?parent :block/name page]]}))
+       (= 'namespace fe)
+       (build-namespace e)
 
        (= 'page-property fe)
        (build-page-property e)
 
        (= 'page-tags fe)
-       (build-query-page-tags e)
+       (build-page-tags e)
 
        (= 'all-page-tags fe)
-       {:query (list 'all-page-tags '?p)
-        :rules [:all-page-tags]}
+       (build-all-page-tags)
 
        (= 'sample fe)
-       (when-let [num (second e)]
-         (when (integer? num)
-           (reset! sample num))
-         nil)
+       (build-sample e sample)
 
        :else
        nil))))

src/main/frontend/db/rules.cljs

@@ -59,7 +59,9 @@
     ])
 
 (def query-dsl-rules
-  "Rules used by frontend.db.query-dsl"
+  "Rules used by frontend.db.query-dsl. The symbols ?b and ?p respectively refer
+  to block and page. Do not alter them as they are programatically built by the
+  query-dsl ns"
   {:page-property
    '[(page-property ?p ?key ?val)
      [?p :block/name]
@@ -91,4 +93,32 @@
 
    :all-page-tags
    '[(all-page-tags ?p)
-     [?e :block/tags ?p]]})
+     [?e :block/tags ?p]]
+
+   :between
+   '[(between ?b ?start ?end)
+     [?b :block/page ?p]
+     [?p :block/journal? true]
+     [?p :block/journal-day ?d]
+     [(>= ?d ?start)]
+     [(<= ?d ?end)]]
+
+   :has-property
+   '[(has-property ?b ?prop)
+     [?b :block/properties ?bp]
+     [(missing? $ ?b :block/name)]
+     [(get ?bp ?prop)]]
+
+   :block-content
+   '[(block-content ?b ?query)
+     [?b :block/content ?content]
+     [(clojure.string/includes? ?content ?query)]]
+
+   :page
+   '[(page ?b ?page-name)
+     [?b :block/page [:block/name ?page-name]]]
+
+   :namespace
+   '[(namespace ?p ?namespace)
+     [?p :block/namespace ?parent]
+     [?parent :block/name ?namespace]]})

src/test/frontend/db/query_dsl_test.cljs

@@ -19,7 +19,8 @@
 (defn- dsl-query
   [s]
   (db/clear-query-state!)
-  (map first (deref (dsl/query test-db s))))
+  (when-let [result (dsl/query test-db s)]
+    (map first (deref result))))
 
 (def parse (partial dsl/parse test-db))
 
@@ -43,8 +44,6 @@
   [s]
   (:count (q-count s)))
 
-(defonce empty-result {:query nil :result nil})
-
 ;; Tests
 ;; =====
 
@@ -108,7 +107,12 @@ prop-d:: nada"}])
   (is (= ["b3"]
          (map (comp first str/split-lines :block/content)
               (dsl-query "(property prop-d no-space-link)")))
-      "Blocks have property value with no space"))
+      "Blocks have property value with no space")
+
+  (is (= ["b3" "b4"]
+         (map (comp first str/split-lines :block/content)
+              (dsl-query "(property prop-d)")))
+      "Blocks that have a property"))
 
 (deftest page-property-queries
   (load-test-files [{:file/path "pages/page1.md"
@@ -186,6 +190,19 @@ prop-d:: nada"}])
               (dsl-query "(or (todo now) (and (todo later) (priority a)))")))
       "Multiple boolean operators with todo and priority operators"))
 
+(deftest sample-queries
+  (load-test-files [{:file/path "pages/page1.md"
+                     :file/content "foo:: bar
+- TODO b1
+- TODO b2"}])
+
+  (is (= 1
+         (count (dsl-query "(and (task todo) (sample 1))")))
+      "Correctly limits results")
+  (is (= 2
+         (count (dsl-query "(and (task todo) (sample blarg))")))
+      "Non-integer arg is ignored"))
+
 (deftest priority-queries
   (load-test-files [{:file/path "pages/page1.md"
                      :file/content "foo:: bar
@@ -274,6 +291,60 @@ tags: other
        "(all-page-tags)"
        ["page-tag-1" "page-tag-2" "page-tag-3" "other"]))
 
+(deftest block-content-query
+  (load-test-files [{:file/path "pages/page1.md"
+                     :file/content "- b1 Hit\n- b2 Another"}])
+
+  (is (= ["b1 Hit"]
+         (map :block/content (dsl-query "\"Hit\""))))
+
+  (is (= []
+         (map :block/content (dsl-query "\"miss\"")))
+      "Correctly returns no results"))
+
+(deftest page-queries
+  (load-test-files [{:file/path "pages/page1.md"
+                     :file/content "foo"}
+                    {:file/path "pages/page2.md"
+                     :file/content "bar"}])
+
+  (is (= ["page1"]
+         (map #(get-in % [:block/page :block/name])
+              (dsl-query "(page page1)"))))
+
+  (is (= []
+         (map #(get-in % [:block/page :block/name])
+              (dsl-query "(page nope)")))
+      "Correctly returns no results"))
+
+(deftest namespace-queries
+  (load-test-files [{:file/path "pages/ns1.page1.md"
+                     :file/content "foo"}
+                    {:file/path "pages/ns1.page2.md"
+                     :file/content "bar"}
+                    {:file/path "pages/ns2.page1.md"
+                     :file/content "baz"}])
+
+  (is (= #{"ns1/page1" "ns1/page2"}
+         (set (map :block/name (dsl-query "(namespace ns1)")))))
+
+  (is (= #{}
+         (set (map :block/name (dsl-query "(namespace blarg)"))))
+      "Correctly returns no results"))
+
+(deftest empty-queries
+  (let [empty-result {:query nil :result nil}]
+    (testing "nil or blank strings should be ignored"
+      (are [x y] (= (q x) y)
+           nil empty-result
+           "" empty-result
+           " " empty-result))
+
+    (testing "Non exists page should be ignored"
+      (are [x y] (nil? (:result (q x)))
+           "[[page-not-exist]]" empty-result
+           "[[another-page-not-exist]]" empty-result))))
+
 (defn- load-test-files-for-parse
   []
   (let [files [{:file/path "journals/2020_12_26.md"
@@ -325,20 +396,8 @@ last-modified-at:: 1609084800002"}]]
     (repo-handler/parse-files-and-load-to-db! test-db files {:re-render? false})))
 
 (deftest test-parse
-  []
   (load-test-files-for-parse)
 
-  (testing "nil or blank strings should be ignored"
-    (are [x y] (= (q x) y)
-         nil empty-result
-         "" empty-result
-         " " empty-result))
-
-  (testing "Non exists page should be ignored"
-    (are [x y] (nil? (:result (q x)))
-         "[[page-not-exist]]" empty-result
-         "[[another-page-not-exist]]" empty-result))
-
   (testing "Single page query"
     (are [x y] (= (q-count x) y)
          "[[page 1]]"
@@ -398,66 +457,67 @@ last-modified-at:: 1609084800002"}]]
   )
 
 #_(deftest sort-by-queries
-  ;; (testing "sort-by (created-at defaults to desc)"
-  ;;   (db/clear-query-state!)
-  ;;   (let [result (->> (q "(and (task now later done)
-  ;;                              (sort-by created-at))")
-  ;;                     :result
-  ;;                     deref
-  ;;                     (map #(get-in % [:block/properties "created-at"])))]
-  ;;     (is (= result
-  ;;            '(1609052959376 1609052958714 1608968448115 1608968448114 1608968448113)))))
-
-  ;; (testing "sort-by (created-at desc)"
-  ;;   (db/clear-query-state!)
-  ;;   (let [result (->> (q "(and (todo now later done)
-  ;;                              (sort-by created-at desc))")
-  ;;                     :result
-  ;;                     deref
-  ;;                     (map #(get-in % [:block/properties "created-at"])))]
-  ;;     (is (= result
-  ;;            '(1609052959376 1609052958714 1608968448115 1608968448114 1608968448113)))))
-
-  ;; (testing "sort-by (created-at asc)"
-  ;;   (db/clear-query-state!)
-  ;;   (let [result (->> (q "(and (todo now later done)
-  ;;                              (sort-by created-at asc))")
-  ;;                     :result
-  ;;                     deref
-  ;;                     (map #(get-in % [:block/properties "created-at"])))]
-  ;;     (is (= result
-  ;;            '(1608968448113 1608968448114 1608968448115 1609052958714 1609052959376)))))
-
-  ;; (testing "sort-by (last-modified-at defaults to desc)"
-  ;;   (db/clear-query-state!)
-  ;;   (let [result (->> (q "(and (todo now later done)
-  ;;                              (sort-by last-modified-at))")
-  ;;                     :result
-  ;;                     deref
-  ;;                     (map #(get-in % [:block/properties "last-modified-at"])))]
-  ;;     (is (= result
-  ;;            '(1609052974285 1609052958714 1608968448120 1608968448115 1608968448113)))))
-
-  ;; (testing "sort-by (last-modified-at desc)"
-  ;;   (db/clear-query-state!)
-  ;;   (let [result (->> (q "(and (todo now later done)
-  ;;                              (sort-by last-modified-at desc))")
-  ;;                     :result
-  ;;                     deref
-  ;;                     (map #(get-in % [:block/properties "last-modified-at"])))]
-  ;;     (is (= result
-  ;;            '(1609052974285 1609052958714 1608968448120 1608968448115 1608968448113)))))
-
-  ;; (testing "sort-by (last-modified-at desc)"
-  ;;   (db/clear-query-state!)
-  ;;   (let [result (->> (q "(and (todo now later done)
-  ;;                              (sort-by last-modified-at asc))")
-  ;;                     :result
-  ;;                     deref
-  ;;                     (map #(get-in % [:block/properties "last-modified-at"])))]
-  ;;     (is (= result
-  ;;            '(1608968448113 1608968448115 1608968448120 1609052958714 1609052974285)))))
-  )
+    (load-test-files-for-parse)
+    ;; (testing "sort-by (created-at defaults to desc)"
+    ;;   (db/clear-query-state!)
+    ;;   (let [result (->> (q "(and (task now later done)
+    ;;                              (sort-by created-at))")
+    ;;                     :result
+    ;;                     deref
+    ;;                     (map #(get-in % [:block/properties "created-at"])))]
+    ;;     (is (= result
+    ;;            '(1609052959376 1609052958714 1608968448115 1608968448114 1608968448113)))))
+
+    ;; (testing "sort-by (created-at desc)"
+    ;;   (db/clear-query-state!)
+    ;;   (let [result (->> (q "(and (todo now later done)
+    ;;                              (sort-by created-at desc))")
+    ;;                     :result
+    ;;                     deref
+    ;;                     (map #(get-in % [:block/properties "created-at"])))]
+    ;;     (is (= result
+    ;;            '(1609052959376 1609052958714 1608968448115 1608968448114 1608968448113)))))
+
+    ;; (testing "sort-by (created-at asc)"
+    ;;   (db/clear-query-state!)
+    ;;   (let [result (->> (q "(and (todo now later done)
+    ;;                              (sort-by created-at asc))")
+    ;;                     :result
+    ;;                     deref
+    ;;                     (map #(get-in % [:block/properties "created-at"])))]
+    ;;     (is (= result
+    ;;            '(1608968448113 1608968448114 1608968448115 1609052958714 1609052959376)))))
+
+    ;; (testing "sort-by (last-modified-at defaults to desc)"
+    ;;   (db/clear-query-state!)
+    ;;   (let [result (->> (q "(and (todo now later done)
+    ;;                              (sort-by last-modified-at))")
+    ;;                     :result
+    ;;                     deref
+    ;;                     (map #(get-in % [:block/properties "last-modified-at"])))]
+    ;;     (is (= result
+    ;;            '(1609052974285 1609052958714 1608968448120 1608968448115 1608968448113)))))
+
+    ;; (testing "sort-by (last-modified-at desc)"
+    ;;   (db/clear-query-state!)
+    ;;   (let [result (->> (q "(and (todo now later done)
+    ;;                              (sort-by last-modified-at desc))")
+    ;;                     :result
+    ;;                     deref
+    ;;                     (map #(get-in % [:block/properties "last-modified-at"])))]
+    ;;     (is (= result
+    ;;            '(1609052974285 1609052958714 1608968448120 1608968448115 1608968448113)))))
+
+    ;; (testing "sort-by (last-modified-at desc)"
+    ;;   (db/clear-query-state!)
+    ;;   (let [result (->> (q "(and (todo now later done)
+    ;;                              (sort-by last-modified-at asc))")
+    ;;                     :result
+    ;;                     deref
+    ;;                     (map #(get-in % [:block/properties "last-modified-at"])))]
+    ;;     (is (= result
+    ;;            '(1608968448113 1608968448115 1608968448120 1609052958714 1609052974285)))))
+    )
 
 (use-fixtures :each
               {:before config/start-test-db!
@@ -466,20 +526,28 @@ last-modified-at:: 1609084800002"}]]
 #_(cljs.test/run-tests)
 
 (comment
-  (require '[clojure.pprint :as pprint])
-  (config/start-test-db!)
-  (import-test-data!)
-
-  (dsl/query test-db "(all-page-tags)")
-
-  ;; (or (priority a) (not (priority a)))
-  ;; FIXME: Error: Insufficient bindings: #{?priority} not bound in [(contains? #{"A"} ?priority)]
-  (pprint/pprint
-   (d/q
-     '[:find (pull ?b [*])
-       :where
-       [?b :block/uuid]
-       (or (and [?b :block/priority ?priority] [(contains? #{"A"} ?priority)])
-           (not [?b :block/priority #{"A"}]
-                [(contains? #{"A"} ?priority)]))]
-     (frontend.db/get-conn test-db))))
+ (require '[clojure.pprint :as pprint])
+ (config/start-test-db!)
+ (import-test-data!)
+
+ (dsl/query test-db "(all-page-tags)")
+
+ ;; Useful for debugging
+ (prn
+  (datascript.core/q
+   '[:find (pull ?b [*])
+     :where
+     [?b :block/name]]
+   (frontend.db/get-conn test-db)))
+
+ ;; (or (priority a) (not (priority a)))
+ ;; FIXME: Error: Insufficient bindings: #{?priority} not bound in [(contains? #{"A"} ?priority)]
+ (pprint/pprint
+  (d/q
+   '[:find (pull ?b [*])
+     :where
+     [?b :block/uuid]
+     (or (and [?b :block/priority ?priority] [(contains? #{"A"} ?priority)])
+         (not [?b :block/priority #{"A"}]
+              [(contains? #{"A"} ?priority)]))]
+   (frontend.db/get-conn test-db))))