chore: convert query-dsl tests to db graphs

Able to remove some dead code and add :build.test/title which
is reusable for future tests

.github/workflows/build.yml

@@ -78,11 +78,8 @@ jobs:
       - name: Build test asset
         run: clojure -M:test compile test
 
-      - name: Run some ClojureScript tests against DB version
-        run: DB_GRAPH=1 node static/tests.js -r frontend.db.query-dsl-test
-
-      - name: Run ClojureScript query tests against DB version with basic query type
-        run: DB_GRAPH=1 DB_QUERY_TYPE=basic node static/tests.js -r frontend.db.query-dsl-test
+      - name: Run ClojureScript query tests against basic query type
+        run: DB_QUERY_TYPE=basic node static/tests.js -r frontend.db.query-dsl-test
 
       - name: Run ClojureScript tests
         run: node static/tests.js -e fix-me

.github/workflows/graph-parser.yml

@@ -79,7 +79,7 @@ jobs:
         run: yarn install --frozen-lockfile
 
       - name: Run ClojureScript tests
-        run: REPEATABLE_IDENTS=true clojure -M:test
+        run: clojure -M:test
 
       - name: Run nbb-logseq tests
         run: yarn test

deps/db/src/logseq/db/frontend/db_ident.cljc

@@ -75,8 +75,7 @@
   (assert (not (re-find #"^(logseq|block)(\.|$)" (name user-namespace)))
           "New ident is not allowed to use an internal namespace")
   (if #?(:org.babashka/nbb true
-         :cljs             (and (exists? js/process)
-                                (or js/process.env.REPEATABLE_IDENTS js/process.env.DB_GRAPH))
+         :cljs             (exists? js/process)
          :default          false)
      ;; Used for contexts where we want repeatable idents e.g. tests and CLIs
     (keyword user-namespace (normalize-ident-name-part name-string))

deps/graph-parser/README.md

@@ -54,7 +54,7 @@ $ yarn test -i focus
 
 ClojureScript tests use https://github.com/Olical/cljs-test-runner. To run tests:
 ```
-REPEATABLE_IDENTS=true clojure -M:test
+clojure -M:test
 ```
 
 To see available options that can run specific tests or namespaces: `clojure -M:test --help`

src/test/frontend/db/query_dsl_test.cljs

@@ -5,15 +5,14 @@
             [frontend.db.query-dsl :as query-dsl]
             [frontend.db.react :as react]
             [frontend.test.helper :as test-helper :include-macros true :refer [load-test-files load-test-files-for-db-graph]]
-            [frontend.util :as util]
-            [logseq.common.util.page-ref :as page-ref]))
+            [frontend.util :as util]))
 
 ;; TODO: quickcheck
 ;; 1. generate query filters
 ;; 2. find illegal queries which can't be executed by datascript
 ;; 3. find filters combinations which might break the current query implementation
 
-(use-fixtures :each {:before test-helper/start-test-db!
+(use-fixtures :each {:before #(test-helper/start-test-db! {:db-graph? true})
                      :after test-helper/destroy-test-db!})
 
 ;; Test helpers
@@ -63,25 +62,17 @@
       (with-redefs [query-dsl/db-block-attrs db-block-attrs]
         (apply query-dsl/query args)))))
 
-(defn- ->smart-query
-  "Updates to file version if js/process.env.DB_GRAPH is not set"
-  [query]
-  (if js/process.env.DB_GRAPH
-    (some-> query
-            (string/replace "(page-tags" "(tags"))
-    query))
-
 (defn- dsl-query
   [s]
   (react/clear-query-state!)
-  (when-let [result (dsl-query* test-helper/test-db (->smart-query s))]
+  (when-let [result (dsl-query* test-helper/test-db-name-db-version s)]
     (map first (deref result))))
 
 (defn- custom-query
   [query]
   (react/clear-query-state!)
   (when-let [result (with-redefs [query-dsl/db-block-attrs db-block-attrs]
-                      (query-dsl/custom-query test-helper/test-db query {}))]
+                      (query-dsl/custom-query test-helper/test-db-name-db-version query {}))]
     (map first (deref result))))
 
 ;; Tests
@@ -120,20 +111,15 @@
 
 (defn- block-property-queries-test
   []
-  (load-test-files [{:file/path "journals/2022_02_28.md"
-                     :file/content "a:: b
-- b1
-prop-a:: val-a
-prop-num:: 2000
-- b2
-prop-a:: val-a
-prop-b:: val-b
-- b3
-prop-d:: [[no-space-link]]
-prop-c:: [[page a]], [[page b]], [[page c]]
-prop-linked-num:: [[3000]]
-- b4
-prop-d:: [[nada]]"}])
+  (load-test-files
+   [{:page {:build/journal 20220228, :build/properties {:a "b"}}
+     :blocks [{:block/title "b1", :build/properties {:prop-a "val-a", :prop-num 2000}}
+              {:block/title "b2", :build/properties {:prop-a "val-a", :prop-b "val-b"}}
+              {:block/title "b3"
+               :build/properties {:prop-d #{[:build/page {:block/title "no-space-link"}]}
+                                  :prop-c #{[:build/page {:block/title "page a"}] [:build/page {:block/title "page b"}] [:build/page {:block/title "page c"}]}
+                                  :prop-linked-num #{[:build/page {:block/title "3000"}]}}}
+              {:block/title "b4", :build/properties {:prop-d #{[:build/page {:block/title "nada"}]}}}]}])
 
   (testing "Blocks have given property value"
     (is (= #{"b1" "b2"}
@@ -190,32 +176,31 @@ prop-d:: [[nada]]"}])
     (test-helper/with-config {}
       (block-property-queries-test))))
 
-(when js/process.env.DB_GRAPH
-  (deftest db-only-block-property-queries
-    (load-test-files-for-db-graph
-     {:properties
-      {:zzz {:logseq.property/type :default
-             :block/title "zzz name!"}}
-      :pages-and-blocks
-      [{:page {:block/title "page1"}
-        :blocks [{:block/title "b1"
-                  :build/properties {:Foo "bar"}}
-                 {:block/title "b2"
-                  :build/properties {:foo "bar"}}
-                 {:block/title "b3"
-                  :build/properties {:zzz "bar"}}]}]})
+(deftest db-only-block-property-queries
+  (load-test-files-for-db-graph
+   {:properties
+    {:zzz {:logseq.property/type :default
+           :block/title "zzz name!"}}
+    :pages-and-blocks
+    [{:page {:block/title "page1"}
+      :blocks [{:block/title "b1"
+                :build/properties {:Foo "bar"}}
+               {:block/title "b2"
+                :build/properties {:foo "bar"}}
+               {:block/title "b3"
+                :build/properties {:zzz "bar"}}]}]})
 
-    (is (= ["b1"]
-           (map :block/title (dsl-query "(property Foo)")))
-        "filter is case sensitive")
-    (is (= ["b2"]
-           (map :block/title (dsl-query "(property :user.property/foo)")))
-        "filter can handle qualified keyword properties")
-    (is (= ["b3"]
-           (map :block/title (dsl-query "(property \"zzz name!\")")))
-        "filter can handle property name")))
+  (is (= ["b1"]
+         (map :block/title (dsl-query "(property Foo)")))
+      "filter is case sensitive")
+  (is (= ["b2"]
+         (map :block/title (dsl-query "(property :user.property/foo)")))
+      "filter can handle qualified keyword properties")
+  (is (= ["b3"]
+         (map :block/title (dsl-query "(property \"zzz name!\")")))
+      "filter can handle property name"))
 
-(when (and js/process.env.DB_GRAPH (not js/process.env.DB_QUERY_TYPE))
+(when (not js/process.env.DB_QUERY_TYPE)
   (deftest property-default-type-default-value-queries
     (load-test-files-for-db-graph
      {:properties
@@ -304,11 +289,11 @@ prop-d:: [[nada]]"}])
 (deftest block-property-query-performance
   (let [pages (->> (repeat 10 {:tags ["tag1" "tag2"]})
                    (map-indexed (fn [idx {:keys [tags]}]
-                                  {:file/path (str "pages/page" idx ".md")
-                                   :file/content (if (seq tags)
-                                                   (str "page-prop:: b\n- block for page" idx
-                                                        "\ntagz:: " (string/join ", " (map page-ref/->page-ref tags)))
-                                                   "")})))
+                                  {:page {:block/title (str "page" idx)
+                                          :build/properties {:page-prop "b"}}
+                                   :blocks [{:block/title (str "block for page" idx)
+                                             :build/properties {:tagz (set (map #(vector :build/page {:block/title %}) tags))}}]}))
+                   vec)
         _ (load-test-files pages)
         {:keys [result time]}
         (util/with-time (dsl-query "(and (property tagz tag1) (property tagz tag2))"))]
@@ -319,209 +304,197 @@ prop-d:: [[nada]]"}])
 
 (defn- page-property-queries-test
   []
-  (load-test-files [{:file/path "pages/page1.md"
-                     :file/content "parent:: [[child page 1]], [[child-no-space]]\ninteresting:: true\nfoo:: baz"}
-                    {:file/path "pages/page2.md"
-                     :file/content "foo:: bar\ninteresting:: false"}
-                    {:file/path "pages/page3.md"
-                     :file/content "parent:: [[child page 1]], [[child page 2]]\nfoo:: bar\ninteresting:: false"}
-                    {:file/path "pages/page4.md"
-                     :file/content "parent:: [[child page 2]]\nfoo:: baz"}])
+  (load-test-files
+   [{:page {:block/title "page1"
+            :build/properties {:parent #{[:build/page {:block/title "child page 1"}] [:build/page {:block/title "child-no-space"}]}, :interesting true, :foo "baz"}}}
+    {:page {:block/title "page2", :build/properties {:foo "bar", :interesting false}}}
+    {:page {:block/title "page3"
+            :build/properties {:parent #{[:build/page {:block/title "child page 1"}] [:build/page {:block/title "child page 2"}]}, :foo "bar", :interesting false}}}
+    {:page {:block/title "page4"
+            :build/properties {:parent #{[:build/page {:block/title "child page 2"}]}, :foo "baz"}}}])
   (is (= ["page1" "page3" "page4"]
-         (map :block/name (dsl-query "(page-property parent)")))
+         (map :block/name (dsl-query "(property parent)")))
       "Pages have given property")
 
   (is (= #{"page1" "page3"}
-         (set (map :block/name (dsl-query "(page-property parent [[child page 1]])"))))
+         (set (map :block/name (dsl-query "(property parent [[child page 1]])"))))
       "Pages have property value that is a page and query is a page")
 
   (is (= #{"page1" "page3"}
-         (set (map :block/name (dsl-query "(page-property parent \"child page 1\")"))))
+         (set (map :block/name (dsl-query "(property parent \"child page 1\")"))))
       "Pages have property value that is a page and query is a string")
 
   (is (= ["page1"]
-         (map :block/name (dsl-query "(page-property parent [[child-no-space]])")))
+         (map :block/name (dsl-query "(property parent [[child-no-space]])")))
       "Pages have property value that is a page with no spaces")
 
   (is (= ["page3"]
          (map
           :block/name
-          (dsl-query "(and (page-property parent [[child page 1]]) (page-property parent [[child page 2]]))")))
+          (dsl-query "(and (property parent [[child page 1]]) (property parent [[child page 2]]))")))
       "Page property queries ANDed")
 
   (is (= #{"page1" "page3" "page4"}
          (set
           (map
            :block/name
-           (dsl-query "(or (page-property parent [[child page 1]]) (page-property parent [[child page 2]]))"))))
+           (dsl-query "(or (property parent [[child page 1]]) (property parent [[child page 2]]))"))))
       "Page property queries ORed")
 
   (is (= ["page1" "page3"]
          (map :block/name
-              (dsl-query "(and (page-property parent [[child page 1]]) (or (page-property foo baz) (page-property parent [[child page 2]])))"))))
+              (dsl-query "(and (property parent [[child page 1]]) (or (property foo baz) (property parent [[child page 2]])))"))))
 
   (is (= ["page4"]
          (map
           :block/name
-          (dsl-query "(and (page-property parent [[child page 2]]) (not (page-property foo bar)))")))
+          (dsl-query "(and (property parent [[child page 2]]) (not (property foo bar)))")))
       "Page property queries nested NOT in second clause")
 
   (is (= ["page4"]
          (map
           :block/name
-          (dsl-query "(and (not (page-property foo bar)) (page-property parent [[child page 2]]))")))
+          (dsl-query "(and (not (property foo bar)) (property parent [[child page 2]]))")))
       "Page property queries nested NOT in first clause")
 
   (testing "boolean values"
     (is (= ["page1"]
-           (map :block/name (dsl-query "(page-property interesting true)")))
+           (map :block/name (dsl-query "(property interesting true)")))
         "Boolean true")
 
     (is (= #{"page2" "page3"}
-           (set (map :block/name (dsl-query "(page-property interesting false)"))))
+           (set (map :block/name (dsl-query "(property interesting false)"))))
         "Boolean false")))
 
-(when-not js/process.env.DB_GRAPH
-  (deftest page-property-queries
-    (testing "page property tests with default config"
-      (test-helper/with-config {}
-        (page-property-queries-test)))))
+(deftest page-property-queries
+  (testing "page property tests with default config"
+    (test-helper/with-config {}
+      (page-property-queries-test))))
 
 (deftest task-queries
-  (load-test-files [{:file/path "pages/page1.md"
-                     :file/content "foo:: bar
-- DONE b1
-- TODO b2
-- DOING b3
-- DOING b4 [[A]]
-- DOING b5 [[B]]"}])
+  (load-test-files
+   [{:page {:block/title "page1"}
+     :blocks [{:build.test/title "DONE b1"}
+              {:build.test/title "TODO b2"}
+              {:build.test/title "DOING b3"}
+              {:build.test/title "DOING b4 [[A]]"}
+              {:build.test/title "DOING b5 [[B]]"}]}])
 
   (testing "Lowercase query"
-    (is (= ["DONE b1"]
+    (is (= ["b1"]
            (map testable-content (dsl-query "(task done)"))))
 
-    (is (= #{"DOING b3" "DOING b4" "DOING b5"}
+    (is (= #{"b3" "b4" "b5"}
            (set (map testable-content (dsl-query "(task doing)"))))))
 
-  (is (= #{"DOING b3" "DOING b4" "DOING b5"}
+  (is (= #{"b3" "b4" "b5"}
          (set (map testable-content (dsl-query "(task DOING)"))))
       "Uppercase query")
 
   (testing "Multiple specified tasks results in ORed results"
-    (is (= #{"DONE b1" "DOING b3" "DOING b4" "DOING b5"}
+    (is (= #{"b1" "b3" "b4" "b5"}
            (set (map testable-content (dsl-query "(task done doing)")))))
 
-    (is (= #{"DONE b1" "DOING b3" "DOING b4" "DOING b5"}
+    (is (= #{"b1" "b3" "b4" "b5"}
            (set (map testable-content (dsl-query "(task [done doing])"))))
         "Multiple arguments specified with vector notation"))
 
-  (is (= ["DONE b1" "DOING b4"]
+  (is (= ["b1" "b4"]
          (map testable-content
               (dsl-query "(or (task done) (and (task doing) [[A]]))")))
       "Multiple boolean operators with todo and priority operators")
 
-  (is (= ["DOING b4" "DOING b5"]
+  (is (= ["b4" "b5"]
          (map testable-content
               (dsl-query "(and (task doing) (or [[A]] [[B]]))")))))
 
-(when js/process.env.DB_GRAPH
-
-  ;; Ensure some filters work when no data with relevant properties exist
-  (deftest queries-with-no-data
-    (load-test-files [])
-    (is (= [] (dsl-query "(task todo)")))
-    (is (= [] (dsl-query "(priority high)")))))
+;; Ensure some filters work when no data with relevant properties exist
+(deftest queries-with-no-data
+  (load-test-files {:pages-and-blocks []})
+  (is (= [] (dsl-query "(task todo)")))
+  (is (= [] (dsl-query "(priority high)"))))
 
 (deftest sample-queries
-  (load-test-files [{:file/path "pages/page1.md"
-                     :file/content "foo:: bar
-- TODO b1
-- TODO b2"}])
+  (load-test-files
+   [{:page {:block/title "page1", :build/properties {:foo "bar"}}
+     :blocks [{:build.test/title "TODO b1"}
+              {:build.test/title "TODO b2"}]}])
   (is (= 1
          (count (dsl-query "(and (task todo) (sample 1))")))
       "Correctly limits block results")
   (is (= 1
-         (count (dsl-query (if js/process.env.DB_GRAPH "(and (property foo) (sample 1))"  "(and (page-property foo) (sample 1))"))))
+         (count (dsl-query "(and (property foo) (sample 1))")))
       "Correctly limits page results"))
 
 (deftest priority-queries
-  (load-test-files (if js/process.env.DB_GRAPH
-                     [{:page {:block/title "page1"}
-                       :blocks [{:block/title "[#A] b1"
-                                 :build/properties {:logseq.property/priority :logseq.property/priority.high}}
-                                {:block/title "[#B] b2"
-                                 :build/properties {:logseq.property/priority :logseq.property/priority.medium}}
-                                {:block/title "[#A] b3"
-                                 :build/properties {:logseq.property/priority :logseq.property/priority.high}}]}]
-
-                     [{:file/path "pages/page1.md"
-                       :file/content "foo:: bar
-- [#A] b1
-- [#B] b2
-- [#A] b3"}]))
+  (load-test-files [{:page {:block/title "page1"}
+                     :blocks [{:block/title "[#A] b1"
+                               :build/properties {:logseq.property/priority :logseq.property/priority.high}}
+                              {:block/title "[#B] b2"
+                               :build/properties {:logseq.property/priority :logseq.property/priority.medium}}
+                              {:block/title "[#A] b3"
+                               :build/properties {:logseq.property/priority :logseq.property/priority.high}}]}])
 
   (testing "one arg queries"
     (is (= #{"[#A] b1" "[#A] b3"}
            (set (map :block/title
-                     (dsl-query (if js/process.env.DB_GRAPH "(priority high)" "(priority a)"))))))
+                     (dsl-query "(priority high)")))))
     (is (= #{"[#A] b1" "[#A] b3"}
            (set (map :block/title
-                     (dsl-query (if js/process.env.DB_GRAPH "(priority high)" "(priority a)")))))))
+                     (dsl-query "(priority high)"))))))
 
   (testing "two arg queries"
     (is (= #{"[#A] b1" "[#B] b2" "[#A] b3"}
            (set (map :block/title
-                     (dsl-query (if js/process.env.DB_GRAPH "(priority high medium)" "(priority a b)"))))))
+                     (dsl-query "(priority high medium)")))))
     (is (= #{"[#A] b1" "[#B] b2" "[#A] b3"}
            (set (map :block/title
-                     (dsl-query (if js/process.env.DB_GRAPH "(priority [high medium])" "(priority [a b])")))))
+                     (dsl-query "(priority [high medium])"))))
         "Arguments with vector notation"))
 
   (is (= #{"[#A] b1" "[#B] b2" "[#A] b3"}
          (set (map :block/title
-                   (dsl-query (if js/process.env.DB_GRAPH "(priority high medium low)" "(priority a b c)")))))
+                   (dsl-query "(priority high medium low)"))))
       "Three arg queries and args that have no match"))
 
 (deftest nested-boolean-queries
-  (load-test-files [{:file/path "pages/page1.md"
-                     :file/content "foo:: bar
-- DONE b1 [[page 1]] [[page 3]]
-- DONE b2Z [[page 1]]"}
-                    {:file/path "pages/page2.md"
-                     :file/content "foo:: bar
-- NOW b3 [[page 1]]
-- LATER b4Z [[page 2]]
-"}])
-
-  (let [task-filter (if js/process.env.DB_GRAPH "(task doing todo)" "(task now later)")]
+  (load-test-files
+   [{:page {:block/title "page1", :build/properties {:foo "bar"}}
+     :blocks [{:build.test/title "DONE b1 [[page 1]] [[page 3]]"}
+              {:build.test/title "DONE b2Z [[page 1]]"}]}
+    {:page {:block/title "page2", :build/properties {:foo "bar"}}
+     :blocks [{:build.test/title "NOW b3 [[page 1]]"}
+              {:build.test/title "LATER b4Z [[page 2]]"}]}])
+
+  (let [task-filter "(task doing todo)"]
     (is (= []
            (dsl-query "(and (task done) (not [[page 1]]))")))
 
-    (is (= ["DONE b1"]
+    (is (= ["b1"]
            (map testable-content
                 (dsl-query "(and [[page 1]] (and [[page 3]] (not (task todo))))")))
         "Nested not")
 
-    (is (= ["NOW b3" "LATER b4Z"]
+    (is (= ["b3" "b4Z"]
            (map testable-content
                 (dsl-query (str "(and " task-filter " (or [[page 1]] [[page 2]]))")))))
 
-    (is (= #{"NOW b3"
-             "LATER b4Z"
-             "DONE b1"
-             "DONE b2Z"}
+    (is (= #{"b3"
+             "b4Z"
+             "b1"
+             "b2Z"}
            (set (map testable-content
                      (dsl-query (str "(and "
-                                     (if js/process.env.DB_GRAPH "(task doing todo done)" "(task now later done)")
+                                     "(task doing todo done)"
                                      " (or [[page 1]] (not [[page 1]])))"))))))
 
-    (is (= (if js/process.env.DB_GRAPH #{"bar" "DONE b1" "DONE b2Z"} #{"foo:: bar" "DONE b1" "DONE b2Z"})
+    (is (= #{"bar" "b1" "b2Z"}
            (->> (dsl-query (str "(not (and " task-filter " (or [[page 1]] [[page 2]])))"))
                 (keep testable-content)
                 (remove (fn [s] (db/page? (db/get-page s))))
                 set)))
 
-    (is (= #{"DONE b2Z" "LATER b4Z"}
+    (is (= #{"b2Z" "b4Z"}
            (->> (dsl-query "(and \"Z\" (or \"b2\" \"b4\"))")
                 (keep testable-content)
                 set))
@@ -535,33 +508,31 @@ prop-d:: [[nada]]"}])
 
 (deftest tags-queries
   (load-test-files
-   [{:file/path "pages/page1.md"
-     :file/content "tags:: [[page-tag-1]], [[page-tag-2]]"}
-    {:file/path "pages/page2.md"
-     :file/content "tags:: [[page-tag-2]], [[page-tag-3]]"}
-    {:file/path "pages/page3.md"
-     :file/content "tags:: [[other]]"}])
+   [{:page {:block/title "page1" :build/tags [:page-tag-1 :page-tag-2]}}
+    {:page {:block/title "page2" :build/tags [:page-tag-2 :page-tag-3]}}
+    {:page {:block/title "page3" :build/tags [:other]}}])
 
   (are [x y] (= (set y) (set (map :block/name (dsl-query x))))
 
-    "(page-tags [[page-tag-1]])"
+    "(tags [[page-tag-1]])"
     ["page1"]
 
-    "(page-tags page-tag-2)"
+    "(tags page-tag-2)"
     ["page1" "page2"]
 
-    "(page-tags page-tag-1 page-tag-2)"
+    "(tags page-tag-1 page-tag-2)"
     ["page1" "page2"]
 
-    "(page-tags page-TAG-1 page-tag-2)"
+    "(tags page-TAG-1 page-tag-2)"
     ["page1" "page2"]
 
-    "(page-tags [page-tag-1 page-tag-2])"
+    "(tags [page-tag-1 page-tag-2])"
     ["page1" "page2"]))
 
 (deftest block-content-query
-  (load-test-files [{:file/path "pages/page1.md"
-                     :file/content "- b1 Hit\n- b2 Another"}])
+  (load-test-files [{:page {:block/title "page1"}
+                     :blocks [{:block/title "b1 Hit"}
+                              {:block/title "b2 Another"}]}])
 
   (is (= ["b1 Hit"]
          (map :block/title (dsl-query "\"Hit\""))))
@@ -571,10 +542,10 @@ prop-d:: [[nada]]"}])
       "Correctly returns no results"))
 
 (deftest page-queries
-  (load-test-files [{:file/path "pages/page1.md"
-                     :file/content "foo"}
-                    {:file/path "pages/page2.md"
-                     :file/content "bar"}])
+  (load-test-files [{:page {:block/title "page1"}
+                     :blocks [{:block/title "foo"}]}
+                    {:page {:block/title "page2"}
+                     :blocks [{:block/title "bar"}]}])
 
   (is (= ["page1"]
          (map (fn [result]
@@ -596,12 +567,11 @@ prop-d:: [[nada]]"}])
       "\"\"")))
 
 (deftest page-ref-and-boolean-queries
-  (load-test-files [{:file/path "pages/page1.md"
-                     :file/content "foo:: bar
-- b1 [[page 1]] [[tag2]]
-- b2 [[page 2]] [[tag1]]
-  - b4 [[page 4]] [[tag4]]
-- b3"}])
+  (load-test-files
+   [{:page {:block/title "page1", :build/properties {:foo "bar"}}
+     :blocks [{:block/title "b1 [[page 1]] [[tag2]]"}
+              {:block/title "b2 [[page 2]] [[tag1]]"}
+              {:block/title "b3"}]}])
 
   (testing "page-ref queries"
 
@@ -628,7 +598,7 @@ prop-d:: [[nada]]"}])
         "OR query")
 
     (comment
-      ;; FIXME: load-test-files doesn't save `b4` to the db when DB_GRAPH=1
+      ;; FIXME: load-test-files doesn't save `b4`
       (is (= ["b1" "b4"]
              (map testable-content
                   (dsl-query "(or [[tag2]] [[page 4]])")))
@@ -639,28 +609,20 @@ prop-d:: [[nada]]"}])
                 (dsl-query "(or [[tag2]] [[page not exists]])")))
         "OR query with nonexistent page should return meaningful results")
 
-    (when js/process.env.DB_GRAPH
-      (is (= #{"b1" "bar" "b3"}
-             (->> (dsl-query "(not [[page 2]])")
+    (is (= #{"b1" "bar" "b3"}
+           (->> (dsl-query "(not [[page 2]])")
                 ;; Only filter to page1 to get meaningful results
-                  (filter #(= "page1" (get-in % [:block/page :block/name])))
-                  (map testable-content)
-                  (set)))
-          "NOT query"))))
+                (filter #(= "page1" (get-in % [:block/page :block/name])))
+                (map testable-content)
+                (set)))
+        "NOT query")))
 
 (deftest nested-page-ref-queries
-  (load-test-files (if js/process.env.DB_GRAPH
-                     [{:page {:block/title "page1"}
-                       :blocks [{:block/title "p1 [[Parent page]]"
-                                 :build/children [{:block/title "[[Child page]]"}]}
-                                {:block/title "p2 [[Parent page]]"
-                                 :build/children [{:block/title "Non linked content"}]}]}]
-                     [{:file/path "pages/page1.md"
-                       :file/content "foo:: bar
-- p1 [[Parent page]]
-  - [[Child page]]
-- p2 [[Parent page]]
-  - Non linked content"}]))
+  (load-test-files [{:page {:block/title "page1"}
+                     :blocks [{:block/title "p1 [[Parent page]]"
+                               :build/children [{:block/title "[[Child page]]"}]}
+                              {:block/title "p2 [[Parent page]]"
+                               :build/children [{:block/title "Non linked content"}]}]}])
   (is (= (set
           ["Non linked content"
            "p2"
@@ -670,19 +632,14 @@ prop-d:: [[nada]]"}])
                (dsl-query "(and [[Parent page]] (not [[Child page]]))"))))))
 
 (deftest between-queries
-  (load-test-files [{:file/path "journals/2020_12_26.md"
-                     :file/content "foo::bar
-- DONE 26-b1
-created-at:: 1608968448113
-- LATER 26-b2-modified-later
-created-at:: 1608968448114
-- DONE 26-b3
-created-at:: 1608968448115
-- 26-b4
-created-at:: 1608968448116
-"}])
-
-  (let [task-filter (if js/process.env.DB_GRAPH "(task todo done)" "(task later done)")]
+  (load-test-files
+   [{:page {:build/journal 20201226}
+     :blocks [{:build.test/title "DONE 26-b1", :block/created-at 1608968448113}
+              {:build.test/title "LATER 26-b2-modified-later", :block/created-at 1608968448114}
+              {:build.test/title "DONE 26-b3", :block/created-at 1608968448115}
+              {:block/title "26-b4", :block/created-at 1608968448116}]}])
+
+  (let [task-filter "(task todo done)"]
     (are [x y] (= (count (dsl-query x)) y)
       (str "(and " task-filter " (between [[Dec 26th, 2020]] tomorrow))")
       3
@@ -691,85 +648,25 @@ created-at:: 1608968448116
       (str "(and " task-filter " (between [[Dec 26th, 2020]] [[Dec 27th, 2020]]))")
       3)
 
-    (when js/process.env.DB_GRAPH
-      (is (= 3 (count (dsl-query "(and (task todo done) (between created-at [[Dec 26th, 2020]]))"))))
-      (is (= 3 (count (dsl-query "(and (task todo done) (between created-at [[Dec 26th, 2020]] +1d))")))))))
+    (is (= 3 (count (dsl-query "(and (task todo done) (between created-at [[Dec 26th, 2020]]))"))))
+    (is (= 3 (count (dsl-query "(and (task todo done) (between created-at [[Dec 26th, 2020]] +1d))"))))))
 
 (deftest custom-query-test
-  (load-test-files [{:file/path "pages/page1.md"
-                     :file/content "foo:: bar
-- NOW b1
-- TODO b2
-- LATER b3
-- b3"}])
-
-  (let [task-query (if js/process.env.DB_GRAPH
-                     '(task doing)
-                     '(task now))]
-    (is (= ["NOW b1"]
+  (load-test-files
+   [{:page {:block/title "page1", :build/properties {:foo "bar"}}
+     :blocks [{:build.test/title "NOW b1"}
+              {:build.test/title "TODO b2"}
+              {:build.test/title "LATER b3"}
+              {:block/title "b3"}]}])
+
+  (let [task-query '(task doing)]
+    (is (= ["b1"]
            (map :block/title (custom-query {:query task-query}))))
 
-    (is (= ["NOW b1"]
+    (is (= ["b1"]
            (map :block/title (custom-query {:query (list 'and task-query "b")})))
         "Query with rule that can't be derived from the form itself")))
 
-(def get-property-value #(get-in %1 [:block/properties %2]))
-
-(when-not js/process.env.DB_GRAPH
-  (deftest sort-by-queries
-    (load-test-files [{:file/path "journals/2020_02_25.md"
-                       :file/content "rating:: 10"}
-                      {:file/path "journals/2020_12_26.md"
-                       :file/content "rating:: 8
-- DONE 26-b1
-created-at:: 1608968448113
-fruit:: plum
-- LATER 26-b2-modified-later
-created-at:: 1608968448114
-fruit:: apple
-- DONE 26-b3 has no fruit to test sorting of absent property value
-created-at:: 1608968448115
-- 26-b4
-created-at:: 1608968448116
-"}])
-    (let [task-filter "(task later done)"]
-      (testing "sort-by user block property fruit"
-        (let [result (->> (dsl-query (str "(and " task-filter " (sort-by fruit))"))
-                          (map #(get-property-value % :fruit)))]
-          (is (= ["plum" "apple" nil]
-                 result)
-              "sort-by correctly defaults to desc"))
-
-        (let [result (->> (dsl-query (str "(and " task-filter " (sort-by fruit desc))"))
-                          (map #(get-property-value % :fruit)))]
-          (is (= ["plum" "apple" nil]
-                 result)
-              "sort-by desc"))
-
-        (let [result (->> (dsl-query (str "(and " task-filter " (sort-by fruit asc))"))
-                          (map #(get-property-value % :fruit)))]
-          (is (= [nil "apple" "plum"]
-                 result)
-              "sort-by asc")))
-
-      (testing "sort-by hidden, built-in block property created-at"
-        (let [result (->> (dsl-query (str "(and " task-filter " (sort-by created-at desc))"))
-                          (map #(get-property-value % :created-at)))]
-          (is (= [1608968448115 1608968448114 1608968448113]
-                 result))
-          "sorted-by desc")
-
-        (let [result (->> (dsl-query (str "(and " task-filter " (sort-by created-at asc))"))
-                          (map #(get-property-value % :created-at)))]
-          (is (= [1608968448113 1608968448114 1608968448115]
-                 result)
-              "sorted-by asc")))
-
-      (testing "user page property rating"
-        (is (= [10 8]
-               (->> (dsl-query "(and (page-property rating) (sort-by rating))")
-                    (map #(get-property-value % :rating)))))))))
-
 (deftest simplify-query
   (are [x y] (= (query-dsl/simplify-query x) y)
     '(and [[foo]])
@@ -814,20 +711,4 @@ created-at:: 1608968448116
       (or (and [?b :block/priority ?priority] [(contains? #{"A"} ?priority)])
           (not [?b :block/priority #{"A"}]
                [(contains? #{"A"} ?priority)]))]
-    (frontend.db/get-db test-helper/test-db))))
-
-(when-not js/process.env.DB_GRAPH
-  (deftest namespace-queries
-    (load-test-files [{:file/path "pages/ns1.page1.md"
-                       :file/content "foo"}
-                      {:file/path "pages/ns1.page2.md"
-                       :file/content "bar"}
-                      {:file/path "pages/ns2.page1.md"
-                       :file/content "baz"}])
-
-    (is (= #{"ns1/page1" "ns1/page2"}
-           (set (map :block/name (dsl-query "(namespace ns1)")))))
-
-    (is (= #{}
-           (set (map :block/name (dsl-query "(namespace blarg)"))))
-        "Correctly returns no results")))
+    (frontend.db/get-db test-helper/test-db))))

src/test/frontend/test/helper.cljs

@@ -17,8 +17,7 @@
             [logseq.db.sqlite.build :as sqlite-build]
             [logseq.db.sqlite.create-graph :as sqlite-create-graph]
             [logseq.db.sqlite.util :as sqlite-util]
-            [logseq.db.test.helper :as db-test]
-            [logseq.graph-parser.text :as text]))
+            [logseq.db.test.helper :as db-test]))
 
 (def bare-marker-pattern
   #"(NOW|LATER|TODO|DOING|DONE|WAITING|WAIT|CANCELED|CANCELLED|IN-PROGRESS){1}\s+")
@@ -50,41 +49,6 @@
   []
   (conn/destroy-all!))
 
-(defn- parse-property-value [value]
-  (if-let [refs (seq (map #(or (second %) (get % 2))
-                          (re-seq #"#(\S+)|\[\[(.*?)\]\]" value)))]
-    (set refs)
-    (if-some [new-val (text/parse-non-string-property-value value)]
-      new-val
-      value)))
-
-(defn- property-lines->properties
-  [property-lines]
-  (->> property-lines
-       (keep (fn [line]
-               (let [[k v] (string/split line #"::\s*" 2)]
-                 (when (string/includes? line "::")
-                   [(keyword k)
-                    (if (= "tags" k)
-                      (parse-property-value v)
-                      (let [val (parse-property-value v)]
-                        (if (coll? val)
-                          (set (map #(vector :build/page {:block/title %}) val))
-                          val)))]))))
-       (into {})))
-
-(defn- property-lines->attributes
-  "Converts markdown property lines e.g. `foo:: bar\nfoo:: baz` to properties
-  and attributes. All are treated as properties except for tags -> :block/tags
-  and created-at -> :block/created-at"
-  [lines]
-  (let [props (property-lines->properties lines)]
-    (cond-> {:build/properties (dissoc props :created-at :tags)}
-      (:tags props)
-      (assoc :build/tags (mapv keyword (:tags props)))
-      (:created-at props)
-      (assoc :block/created-at (:created-at props)))))
-
 (def file-to-db-statuses
   {"TODO" :logseq.property/status.todo
    "LATER" :logseq.property/status.todo
@@ -97,81 +61,41 @@
    "CANCELED" :logseq.property/status.canceled
    "CANCELLED" :logseq.property/status.canceled})
 
-(defn- parse-content
-  "Given a file's content as markdown, returns blocks and page attributes for the file
-   to be used with sqlite-build/build-blocks-tx"
-  [content*]
-  (let [blocks** (if (string/includes? content* "\n-")
-                   (->> (string/split content* #"\n-\s*")
-                        (mapv (fn [s]
-                                (let [[content & props] (string/split-lines s)]
-                                  (cond-> {:block/title content}
-                                    ;; If no property chars may accidentally parse child blocks
-                                    ;; so don't do property parsing
-                                    (and (string/includes? s ":: ") props)
-                                    (merge (property-lines->attributes props)))))))
-                   ;; only has a page pre-block
-                   [{:block/title content*}])
-        [page-attrs blocks*]
-        (if (string/includes? (:block/title (first blocks**)) "::")
-          [(property-lines->attributes (string/split-lines (:block/title (first blocks**))))
-           (rest blocks**)]
-          [nil blocks**])
-        blocks
-        (mapv #(if-let [status (some-> (second (re-find bare-marker-pattern (:block/title %)))
-                                       file-to-db-statuses)]
-                 (-> %
-                     (assoc :block/tags [{:db/ident :logseq.class/Task}])
-                     (update :build/properties merge {:logseq.property/status status}))
-                 %)
-              blocks*)]
-    {:blocks (mapv (fn [b] (update b :block/title #(string/replace-first % #"^-\s*" "")))
-                   blocks)
-     :page-attributes page-attrs}))
-
-(defn- build-blocks-tx-options
-  "Given arguments to load-test-files, parses and converts them to options for
-  sqlite-build/build-blocks-tx. Supports a limited set of markdown including
-  task keywords, page properties and block properties. See query-dsl-test for examples"
-  [options*]
-  (let [pages-and-blocks
-        (mapv (fn [{:file/keys [path content]}]
-                (let [{:keys [blocks page-attributes]} (parse-content content)
-                      unique-page-attrs
-                      (if (string/starts-with? path "journals")
-                        {:build/journal
-                         (or (some-> (second (re-find #"/([^/]+)\." path))
-                                     (string/replace "_" "")
-                                     parse-double)
-                             (throw (ex-info (str "Can't detect page name of file: " (pr-str path)) {})))}
-                        {:block/title
-                         (or (second (re-find #"/([^/]+)\." path))
-                             (throw (ex-info (str "Can't detect page name of file: " (pr-str path)) {})))})]
-                  {:page (cond-> unique-page-attrs
-                           (seq page-attributes)
-                           (merge page-attributes))
-                   :blocks blocks}))
-              options*)
-        options {:pages-and-blocks pages-and-blocks
-                 :auto-create-ontology? true}]
-    options))
+(defn- build-test-block
+  [block]
+  (if-let [status (some->> (:build.test/title block)
+                           (re-find bare-marker-pattern)
+                           second
+                           file-to-db-statuses)]
+    (-> {:block/title
+         (string/replace-first (:build.test/title block) bare-marker-pattern "")}
+        (assoc :block/tags [{:db/ident :logseq.class/Task}])
+        (update :build/properties merge {:logseq.property/status status}))
+    block))
 
 (defn load-test-files-for-db-graph
+  "Wrapper around sqlite-build/build-blocks-tx with frontend defaults. Also supports
+   the following special keys:
+   * :build.test/title - Only available to top-level blocks. Convenient for writing tasks quickly"
   [options*]
   (let [;; Builds options from markdown :file/content unless given explicit build-blocks config
         options (cond (:page (first options*))
                       {:pages-and-blocks options* :auto-create-ontology? true}
-                      (:pages-and-blocks options*)
-                      (assoc options* :auto-create-ontology? true)
                       :else
-                      (build-blocks-tx-options options*))
-        {:keys [init-tx block-props-tx] :as _txs} (sqlite-build/build-blocks-tx options)
+                      (assoc options* :auto-create-ontology? true))
+        options' (update options
+                         :pages-and-blocks
+                         (fn [pbs]
+                           (mapv (fn [m]
+                                   (update m :blocks
+                                           (fn [blocks]
+                                             (mapv build-test-block blocks))))
+                                 pbs)))
+        {:keys [init-tx block-props-tx] :as _txs} (sqlite-build/build-blocks-tx options')
         ;; Allow pages to reference each other via uuid and for unordered init-tx
         init-index (map #(select-keys % [:block/uuid]) init-tx)]
     ;; (cljs.pprint/pprint _txs)
-    (db/transact! test-db-name-db-version (concat init-index init-tx))
-    (when (seq block-props-tx)
-      (db/transact! test-db-name-db-version block-props-tx))))
+    (db/transact! test-db-name-db-version (concat init-index init-tx block-props-tx))))
 
 (defn load-test-files
   "Given a collection of file maps, loads them into the current test-db.