Split out graph fns and reuse them b/n gp-cli and electron

Also added test

deps/common/nbb.edn

@@ -0,0 +1,4 @@
+{:paths ["src"]
+ :deps
+ {io.github.nextjournal/nbb-test-runner
+  {:git/sha "60ed57aa04bca8d604f5ba6b28848bd887109347"}}}

deps/common/package.json

@@ -0,0 +1,11 @@
+{
+  "name": "@logseq/common",
+  "version": "1.0.0",
+  "private": true,
+  "devDependencies": {
+    "@logseq/nbb-logseq": "^1.2.173"
+  },
+  "scripts": {
+    "test": "yarn nbb-logseq -cp test -m nextjournal.test-runner"
+  }
+}

deps/common/src/logseq/common/graph.cljs

@@ -0,0 +1,73 @@
+(ns logseq.common.graph
+  "This ns provides common fns for a graph directory and only runs in a node environment"
+  (:require ["fs" :as fs]
+            ["path" :as node-path]
+            [clojure.string :as string]))
+
+(def ^:private win32?
+  "Copy of electron.utils/win32? . Too basic to couple the two libraries"
+  (= (.-platform js/process) "win32"))
+
+(defn- fix-win-path!
+  "Copy of electron.utils/fix-win-path!. Too basic to couple the two libraries"
+  [path]
+  (when (not-empty path)
+    (if win32?
+      (string/replace path "\\" "/")
+      path)))
+
+(defn readdir
+  "Reads given directory recursively and returns all filenames. Also applies
+  some graph-specific filtering e.g. symbolic links and files starting with '.'
+  are removed"
+  [root-dir]
+  (->> (tree-seq
+        (fn [[is-dir _fpath]]
+          is-dir)
+        (fn [[_is-dir dir]]
+          (let [files (fs/readdirSync dir #js {:withFileTypes true})]
+            (->> files
+                 (remove #(.isSymbolicLink ^js %))
+                 (remove #(string/starts-with? (.-name ^js %) "."))
+                 (map #(do
+                         [(.isDirectory %)
+                          (node-path/join dir (.-name %))])))))
+        [true root-dir])
+       (filter (complement first))
+       (map second)
+       (map fix-win-path!)
+       (vec)))
+
+(def ^:private allowed-formats
+  #{:org :markdown :md :edn :json :js :css :excalidraw :tldr})
+
+(defn- get-ext
+  [p]
+  (-> (node-path/extname p)
+      (subs 1)
+      keyword))
+
+(defn ignored-path?
+  "Given a graph directory and path, returns truthy value on whether the path is
+  ignored. Useful for contexts like reading a graph's directory and file watcher
+  notifications"
+  [dir path]
+  (when (string? path)
+    (or
+     (some #(string/starts-with? path (str dir "/" %))
+           ["." ".recycle" "node_modules" "logseq/bak" "version-files"])
+     (some #(string/includes? path (str "/" % "/"))
+           ["." ".recycle" "node_modules" "logseq/bak" "version-files"])
+     (some #(string/ends-with? path %)
+           [".DS_Store" "logseq/graphs-txid.edn"])
+     ;; hidden directory or file
+     (let [relpath (node-path/relative dir path)]
+       (or (re-find #"/\.[^.]+" relpath)
+           (re-find #"^\.[^.]+" relpath))))))
+
+(defn get-files
+  "Given a graph's root dir, returns a list of all files that it recognizes"
+  [graph-dir]
+  (->> (readdir graph-dir)
+       (remove (partial ignored-path? graph-dir))
+       (filter #(contains? allowed-formats (get-ext %)))))

deps/common/test/logseq/common/graph_test.cljs

@@ -0,0 +1,35 @@
+(ns logseq.common.graph-test
+  (:require [logseq.common.graph :as common-graph]
+            [cljs.test :refer [deftest is use-fixtures async]]
+            ["fs" :as fs]
+            ["path" :as node-path]))
+
+(use-fixtures
+  :each
+  ;; Cleaning tmp/ before leaves last tmp/ after a test run for dev and debugging
+  {:before
+   #(async done
+           (if (fs/existsSync "tmp")
+             (fs/rm "tmp" #js {:recursive true} (fn [err]
+                                                  (when err (js/console.log err))
+                                                  (done)))
+             (done)))})
+
+(defn- create-logseq-graph
+  "Creates a minimal mock graph"
+  [dir]
+  (fs/mkdirSync (node-path/join dir "logseq") #js {:recursive true})
+  (fs/mkdirSync (node-path/join dir "journals"))
+  (fs/mkdirSync (node-path/join dir "pages")))
+
+(deftest get-files
+  (create-logseq-graph "tmp/test-graph")
+  ;; Create files that are recognized
+  (fs/writeFileSync "tmp/test-graph/pages/foo.md" "")
+  (fs/writeFileSync "tmp/test-graph/journals/2023_05_09.md" "")
+  ;; Create files that are ignored
+  (fs/mkdirSync (node-path/join "tmp/test-graph" "logseq" "bak"))
+  (fs/writeFileSync "tmp/test-graph/logseq/bak/baz.md" "")
+  (fs/writeFileSync "tmp/test-graph/logseq/.gitignore" "")
+  (is (= ["tmp/test-graph/journals/2023_05_09.md" "tmp/test-graph/pages/foo.md"]
+         (common-graph/get-files "tmp/test-graph"))))

deps/common/yarn.lock

@@ -0,0 +1,15 @@
+# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+
+"@logseq/nbb-logseq@^1.2.173":
+  version "1.2.173"
+  resolved "https://registry.yarnpkg.com/@logseq/nbb-logseq/-/nbb-logseq-1.2.173.tgz#27a52c350f06ac9c337d73687738f6ea8b2fc3f3"
+  integrity sha512-ABKPtVnSOiS4Zpk9+UTaGcs5H6EUmRADr9FJ0aEAVpa0WfAyvUbX/NgkQGMe1kKRv3EbIuLwaxfy+txr31OtAg==
+  dependencies:
+    import-meta-resolve "^2.1.0"
+
+import-meta-resolve@^2.1.0:
+  version "2.2.2"
+  resolved "https://registry.yarnpkg.com/import-meta-resolve/-/import-meta-resolve-2.2.2.tgz#75237301e72d1f0fbd74dbc6cca9324b164c2cc9"
+  integrity sha512-f8KcQ1D80V7RnqVm+/lirO9zkOxjGxhaTC1IPrBGd3MEfNgmNG67tSUO9gTi2F3Blr2Az6g1vocaxzkVnWl9MA==

deps/graph-parser/deps.edn

@@ -3,8 +3,9 @@
  ;; External deps should be kept in sync with https://github.com/logseq/nbb-logseq/blob/main/bb.edn
  {com.andrewmcveigh/cljs-time {:git/url "https://github.com/logseq/cljs-time" ;; fork
                                :sha     "5704fbf48d3478eedcf24d458c8964b3c2fd59a9"}
-  ;; local dep
+  ;; local deps
   logseq/db                   {:local/root "../db"}
+  logseq/common               {:local/root "../common"}
   ;; stubbed in nbb
   com.lambdaisland/glogi      {:mvn/version "1.1.144"}
   ;; built in to nbb

deps/graph-parser/nbb.edn

@@ -2,5 +2,7 @@
  :deps
  {logseq/db
   {:local/root "../db"}
+  logseq/common
+  {:local/root "../common"}
   io.github.nextjournal/nbb-test-runner
   {:git/sha "60ed57aa04bca8d604f5ba6b28848bd887109347"}}}

deps/graph-parser/src/logseq/graph_parser/cli.cljs

@@ -3,38 +3,25 @@
   (:require ["fs" :as fs]
             ["child_process" :as child-process]
             [clojure.edn :as edn]
-            [clojure.string :as string]
+            [logseq.common.graph :as common-graph]
             [logseq.graph-parser :as graph-parser]
             [logseq.graph-parser.config :as gp-config]
             [logseq.graph-parser.util :as gp-util]
             [logseq.db :as ldb]))
 
-(defn slurp
+(defn- slurp
   "Return file contents like clojure.core/slurp"
   [file]
   (str (fs/readFileSync file)))
 
-(defn sh
-  "Run shell cmd synchronously and print to inherited streams by default. Aims
-    to be similar to babashka.tasks/shell
-TODO: Fail fast when process exits 1"
-  [cmd opts]
-  (child-process/spawnSync (first cmd)
-                           (clj->js (rest cmd))
-                           (clj->js (merge {:stdio "inherit"} opts))))
-
-(defn build-graph-files
-  "Given a git graph directory, returns allowed file paths and their contents in
-  preparation for parsing"
+(defn- build-graph-files
+  "Given a graph directory, return allowed file paths and their contents in preparation
+   for parsing"
   [dir]
-  ;; -z needed to avoid quoting unusual paths that cause slurp failures.
-  ;; See https://git-scm.com/docs/git-ls-files#_output for more
-  (let [files (->> (str (.-stdout (sh ["git" "ls-files" "-z"]
-                                      {:cwd dir :stdio nil})))
-                   (#(string/split % (re-pattern "\0")))
-                   (map #(hash-map :file/path (str dir "/" %)))
-                   graph-parser/filter-files)]
-    (mapv #(assoc % :file/content (slurp (:file/path %))) files)))
+  (->> (common-graph/get-files dir)
+       (map #(hash-map :file/path %))
+       graph-parser/filter-files
+       (mapv #(assoc % :file/content (slurp (:file/path %))))))
 
 (defn- read-config
   "Reads repo-specific config from logseq/config.edn"

deps/graph-parser/src/logseq/graph_parser/test/docs_graph_helper.cljs

@@ -152,8 +152,8 @@
   ;; Counts assertions help check for no major regressions. These counts should
   ;; only increase over time as the docs graph rarely has deletions
   (testing "Counts"
-    (is (= 306 (count files)) "Correct file count")
-    (is (= 69508 (count (d/datoms db :eavt))) "Correct datoms count")
+    (is (= 304 (count files)) "Correct file count")
+    (is (= 69502 (count (d/datoms db :eavt))) "Correct datoms count")
 
     (is (= 5866
            (ffirst

deps/graph-parser/test/logseq/graph_parser/mldoc_test.cljs

@@ -139,7 +139,7 @@ body"
             "Heading" 5648,
             "Hiccup" 9,
             "List" 22,
-            "Paragraph" 573,
+            "Paragraph" 571,
             "Properties" 87,
             "Property_Drawer" 423,
             "Quote" 24,

src/electron/electron/fs_watcher.cljs

@@ -8,7 +8,8 @@
             [electron.utils :as utils]
             [electron.logger :as logger]
             ["electron" :refer [app]]
-            [electron.window :as window]))
+            [electron.window :as window]
+            [logseq.common.graph :as common-graph]))
 
 ;; TODO: explore different solutions for different platforms
 ;; 1. https://github.com/Axosoft/nsfw
@@ -61,7 +62,7 @@
   [dir options]
   (let [watcher-opts (clj->js
                       {:ignored (fn [path]
-                                  (utils/ignored-path? dir path))
+                                  (common-graph/ignored-path? dir path))
                        :ignoreInitial true
                        :ignorePermissionErrors true
                        :interval polling-interval

src/electron/electron/handler.cljs

@@ -28,6 +28,7 @@
             [electron.state :as state]
             [electron.utils :as utils]
             [electron.window :as win]
+            [logseq.common.graph :as common-graph]
             [promesa.core :as p]))
 
 (defmulti handle (fn [_window args] (keyword (first args))))
@@ -38,28 +39,8 @@
 (defmethod handle :mkdir-recur [_window [_ dir]]
   (fs/mkdirSync dir #js {:recursive true}))
 
-(defn- readdir
-  "Read directory recursively, return all filenames"
-  [root-dir]
-  (->> (tree-seq
-        (fn [[is-dir _fpath]]
-          is-dir)
-        (fn [[_is-dir dir]]
-          (let [files (fs/readdirSync dir #js {:withFileTypes true})]
-            (->> files
-                 (remove #(.isSymbolicLink ^js %))
-                 (remove #(string/starts-with? (.-name ^js %) "."))
-                 (map #(do
-                         [(.isDirectory %)
-                          (.join node-path dir (.-name %))])))))
-        [true root-dir])
-       (filter (complement first))
-       (map second)
-       (map utils/fix-win-path!)
-       (vec)))
-
 (defmethod handle :readdir [_window [_ dir]]
-  (readdir dir))
+  (common-graph/readdir dir))
 
 (defmethod handle :listdir [_window [_ dir flat?]]
   (when (and dir (fs-extra/pathExistsSync dir))
@@ -147,21 +128,10 @@
 (defmethod handle :stat [_window [_ path]]
   (fs/statSync path))
 
-(defonce allowed-formats
-  #{:org :markdown :md :edn :json :js :css :excalidraw :tldr})
-
-(defn get-ext
-  [p]
-  (-> (.extname node-path p)
-      (subs 1)
-      keyword))
-
 (defn- get-files
   "Returns vec of file-objs"
   [path]
-  (->> (readdir path)
-       (remove (partial utils/ignored-path? path))
-       (filter #(contains? allowed-formats (get-ext %)))
+  (->> (common-graph/get-files path)
        (map (fn [path]
               (let [stat (fs/statSync path)]
                 (when-not (.isDirectory stat)
@@ -238,10 +208,10 @@
     dir))
 
 (defn- get-graphs
-  "Returns all graph names in the cache directory (strating with `logseq_local_`)"
+  "Returns all graph names in the cache directory (starting with `logseq_local_`)"
   []
   (let [dir (get-graphs-dir)]
-    (->> (readdir dir)
+    (->> (common-graph/readdir dir)
          (remove #{dir})
          (map #(node-path/basename % ".transit"))
          (map graph-name->path))))

src/electron/electron/utils.cljs

@@ -198,23 +198,6 @@
     (cfgs/set-item! :settings/agent {:type type :test test})
     (cfgs/set-item! :settings/agent {:type type :protocol type :host host :port port :test test})))
 
-
-(defn ignored-path?
-  "Ignore given path from file-watcher notification"
-  [dir path]
-  (when (string? path)
-    (or
-     (some #(string/starts-with? path (str dir "/" %))
-           ["." ".recycle" "node_modules" "logseq/bak" "version-files"])
-     (some #(string/includes? path (str "/" % "/"))
-           ["." ".recycle" "node_modules" "logseq/bak" "version-files"])
-     (some #(string/ends-with? path %)
-           [".DS_Store" "logseq/graphs-txid.edn"])
-     ;; hidden directory or file
-     (let [relpath (node-path/relative dir path)]
-       (or (re-find #"/\.[^.]+" relpath)
-           (re-find #"^\.[^.]+" relpath))))))
-
 (defn should-read-content?
   "Skip reading content of file while using file-watcher"
   [path]

src/test/frontend/handler/repo_conversion_test.cljs

@@ -97,8 +97,8 @@
   ;; Counts assertions help check for no major regressions. These counts should
   ;; only increase over time as the docs graph rarely has deletions
   (testing "Counts"
-    (is (= 211 (count files)) "Correct file count")
-    (is (= 42304 (count (d/datoms db :eavt))) "Correct datoms count")
+    (is (= 212 (count files)) "Correct file count")
+    (is (= 42315 (count (d/datoms db :eavt))) "Correct datoms count")
 
     (is (= 3600
            (ffirst