Fix parse-graph not ignoring correctly for relative dir

parse-graph now returns absolute file paths

deps/common/src/logseq/common/graph.cljs

@@ -66,7 +66,8 @@
       keyword))
 
 (defn get-files
-  "Given a graph's root dir, returns a list of all files that it recognizes"
+  "Given a graph's root dir, returns a list of all files that it recognizes.
+   Graph dir must be an absolute path in order for ignoring to work correctly"
   [graph-dir]
   (->> (readdir graph-dir)
        (remove (partial ignored-path? graph-dir))

deps/graph-parser/src/logseq/graph_parser/cli.cljs

@@ -24,14 +24,15 @@
     files))
 
 (defn- build-graph-files
-  "Given a graph directory, return allowed file paths and their contents in preparation
+  "Given a graph directory, return absolute, allowed file paths and their contents in preparation
    for parsing"
-  [dir config]
-  (->> (common-graph/get-files dir)
-       (map #(hash-map :file/path %))
-       graph-parser/filter-files
-       (remove-hidden-files dir config)
-       (mapv #(assoc % :file/content (slurp (:file/path %))))))
+  [dir* config]
+  (let [dir (path/resolve dir*)]
+    (->> (common-graph/get-files dir)
+        (map #(hash-map :file/path %))
+        graph-parser/filter-files
+        (remove-hidden-files dir config)
+        (mapv #(assoc % :file/content (slurp (:file/path %)))))))
 
 (defn- read-config
   "Reads repo-specific config from logseq/config.edn"

deps/graph-parser/src/logseq/graph_parser/test/docs_graph_helper.cljs

@@ -69,11 +69,11 @@
        ffirst))
 
 (defn- query-assertions
-  [db files]
+  [db graph-dir files]
   (testing "Query based stats"
     (is (= (->> files
                 ;; logseq files aren't saved under :block/file
-                (remove #(string/includes? % (str "/" gp-config/app-name "/")))
+                (remove #(string/includes? % (str graph-dir "/" gp-config/app-name "/")))
                 ;; edn files being listed in docs by parse-graph aren't graph files
                 (remove #(and (not (gp-config/whiteboard? %)) (string/ends-with? % ".edn")))
                 set)
@@ -148,7 +148,7 @@
   logseq app. It is important to run these in both contexts to ensure that the
   functionality in frontend.handler.repo and logseq.graph-parser remain the
   same"
-  [db files]
+  [db graph-dir files]
   ;; Counts assertions help check for no major regressions. These counts should
   ;; only increase over time as the docs graph rarely has deletions
   (testing "Counts"
@@ -168,4 +168,4 @@
                  db)))
         "Advanced query count"))
 
-  (query-assertions db files))
+  (query-assertions db graph-dir files))

deps/graph-parser/test/logseq/graph_parser/cli_test.cljs

@@ -1,16 +1,30 @@
-(ns logseq.graph-parser.cli-test
-  (:require [cljs.test :refer [deftest is testing]]
+(ns ^:node-only logseq.graph-parser.cli-test
+  (:require [cljs.test :refer [deftest is testing async use-fixtures]]
             [logseq.graph-parser.cli :as gp-cli]
             [logseq.graph-parser.test.docs-graph-helper :as docs-graph-helper]
-            [clojure.string :as string]))
+            [clojure.string :as string]
+            ["fs" :as fs]
+            ["process" :as process]
+            ["path" :as path]))
+
+(use-fixtures
+  :each
+  ;; Cleaning tmp/ before leaves last tmp/ after a test run for dev and debugging
+  {:before
+   #(async done
+           (if (fs/existsSync "tmp")
+             (fs/rm "tmp" #js {:recursive true} (fn [err]
+                                                  (when err (js/console.log err))
+                                                  (done)))
+             (done)))})
 
 ;; Integration test that test parsing a large graph like docs
 (deftest ^:integration parse-graph
   (let [graph-dir "test/docs-0.9.2"
         _ (docs-graph-helper/clone-docs-repo-if-not-exists graph-dir "v0.9.2")
-        {:keys [conn files asts]} (gp-cli/parse-graph graph-dir {:verbose false})] ;; legacy parsing
+        {:keys [conn files asts]} (gp-cli/parse-graph graph-dir {:verbose false})]
 
-    (docs-graph-helper/docs-graph-assertions @conn files)
+    (docs-graph-helper/docs-graph-assertions @conn graph-dir files)
 
     (testing "Asts"
       (is (seq asts) "Asts returned are non-zero")
@@ -25,3 +39,37 @@
                             (string/includes? (:file %) (str graph-dir "/logseq/")))
                           asts))
           "Parsed files shouldn't have empty asts"))))
+
+(defn- create-logseq-graph
+  "Creates a minimal mock graph"
+  [dir]
+  (fs/mkdirSync (path/join dir "logseq") #js {:recursive true})
+  (fs/mkdirSync (path/join dir "journals"))
+  (fs/mkdirSync (path/join dir "pages")))
+
+(deftest ^:focus build-graph-files
+  (create-logseq-graph "tmp/test-graph")
+  ;; Create files that are recognized
+  (fs/writeFileSync "tmp/test-graph/pages/foo.md" "")
+  (fs/writeFileSync "tmp/test-graph/journals/2023_05_09.md" "")
+  ;; Create file that are ignored and filtered out
+  (fs/writeFileSync "tmp/test-graph/pages/foo.json" "")
+  (fs/mkdirSync (path/join "tmp/test-graph" "logseq" "bak"))
+  (fs/writeFileSync "tmp/test-graph/logseq/bak/baz.md" "")
+
+  (testing "ignored files from common-graph"
+    (is (= (map #(path/join (process/cwd) "tmp/test-graph" %) ["journals/2023_05_09.md" "pages/foo.md"])
+           (map :file/path (#'gp-cli/build-graph-files (path/resolve "tmp/test-graph") {})))
+        "Correct paths returned for absolute dir")
+    (process/chdir "tmp/test-graph")
+    (is (= (map #(path/join (process/cwd) %) ["journals/2023_05_09.md" "pages/foo.md"])
+           (map :file/path (#'gp-cli/build-graph-files "." {})))
+        "Correct paths returned for relative current dir")
+    (process/chdir "../.."))
+
+  (testing ":hidden config"
+    (fs/mkdirSync (path/join "tmp/test-graph" "script"))
+    (fs/writeFileSync "tmp/test-graph/script/README.md" "")
+    (is (= (map #(path/join (process/cwd) "tmp/test-graph" %) ["journals/2023_05_09.md" "pages/foo.md"])
+           (map :file/path (#'gp-cli/build-graph-files "tmp/test-graph" {:hidden ["script"]})))
+        "Correct paths returned")))

src/test/frontend/handler/repo_conversion_test.cljs

@@ -17,11 +17,11 @@
                      :after test-helper/destroy-test-db!})
 
 (defn- query-assertions-v067
-  [db files]
+  [db graph-dir files]
   (testing "Query based stats"
     (is (= (->> files
                 ;; logseq files aren't saved under :block/file
-                (remove #(string/includes? % (str "/" gp-config/app-name "/")))
+                (remove #(string/includes? % (str graph-dir "/" gp-config/app-name "/")))
                 set)
            (->> (d/q '[:find (pull ?b [* {:block/file [:file/path]}])
                        :where [?b :block/name] [?b :block/file]]
@@ -93,7 +93,7 @@
   logseq app. It is important to run these in both contexts to ensure that the
   functionality in frontend.handler.repo and logseq.graph-parser remain the
   same"
-  [db files]
+  [db graph-dir files]
   ;; Counts assertions help check for no major regressions. These counts should
   ;; only increase over time as the docs graph rarely has deletions
   (testing "Counts"
@@ -113,7 +113,7 @@
                  db)))
         "Advanced query count"))
 
-  (query-assertions-v067 db files))
+  (query-assertions-v067 db graph-dir files))
 
 (defn- convert-to-triple-lowbar
   [path]
@@ -144,4 +144,4 @@
         db (conn/get-db test-helper/test-db)]
 
     ;; Result under new naming rule after conversion should be the same as the old one
-    (docs-graph-assertions-v067 db (map :file/path files))))
+    (docs-graph-assertions-v067 db graph-dir (map :file/path files))))

src/test/frontend/handler/repo_test.cljs

@@ -22,7 +22,7 @@
             (repo-handler/parse-files-and-load-to-db! test-helper/test-db files {:re-render? false :verbose false}))
         db (conn/get-db test-helper/test-db)]
 
-    (docs-graph-helper/docs-graph-assertions db (map :file/path files))))
+    (docs-graph-helper/docs-graph-assertions db graph-dir (map :file/path files))))
 
 (deftest parse-files-and-load-to-db-with-block-refs-on-reload
   (testing "Refs to blocks on a page are retained if that page is reloaded"