electron: file watcher wip

package.json

@@ -54,6 +54,7 @@
         "cljs:build-electron": "clojure -A:cljs compile app electron"
     },
     "dependencies": {
+        "chokidar": "^3.5.1",
         "codemirror": "^5.58.1",
         "diff": "5.0.0",
         "diff-match-patch": "^1.0.5",

resources/js/preload.js

@@ -5,6 +5,11 @@ contextBridge.exposeInMainWorld('apis', {
     return await ipcRenderer.invoke('main', arg)
   },
 
+  on: (channel, callback) => {
+    const newCallback = (_, data) => callback(data);
+    ipcRenderer.on(channel, newCallback);
+  },
+
   checkForUpdates: async (...args) => {
     await ipcRenderer.invoke('check-for-updates', ...args)
   },

src/electron/electron/handler.cljs

@@ -3,8 +3,10 @@
             [cljs-bean.core :as bean]
             ["fs" :as fs]
             ["path" :as path]
+            ["chokidar" :as watcher]
             [promesa.core :as p]
-            [goog.object :as gobj]))
+            [goog.object :as gobj]
+            [clojure.string :as string]))
 
 (defmulti handle (fn [_window args] (keyword (first args))))
 
@@ -64,6 +66,54 @@
 (defmethod handle :getFiles [window [_ path]]
   (get-files path))
 
+(defn- get-file-ext
+  [file]
+  (last (string/split file #"\.")))
+
+(defonce file-watcher-chan "file-watcher")
+(defn send-file-watcher! [win type payload]
+  (prn "file watch: " {:type type
+                       :payload payload})
+  (.. win -webContents
+      (send file-watcher-chan
+            (bean/->js {:type type :payload payload}))))
+
+(defn watch-dir!
+  [win dir]
+  (let [watcher (.watch watcher dir
+                        (clj->js
+                         {:ignored #"^\."
+                          :persistent true
+                          :awaitWriteFinish true}))]
+    (.on watcher "add"
+         (fn [path]
+           (send-file-watcher! win "add"
+                               {:dir dir
+                                :path path
+                                :content (read-file path)
+                                :stat (fs/statSync path)})))
+    (.on watcher "change"
+         (fn [path]
+           (send-file-watcher! win "change"
+                               {:dir dir
+                                :path path
+                                :content (read-file path)
+                                :stat (fs/statSync path)})))
+    (.on watcher "unlink"
+         (fn [path]
+           (send-file-watcher! win "unlink"
+                               {:dir dir
+                                :path path})))
+    (.on watcher "error"
+         (fn [path]
+           (println "Watch error happend: "
+                    {:path path})))
+    true))
+
+(defmethod handle :addDirWatcher [window [_ dir]]
+  (when dir
+    (watch-dir! window dir)))
+
 (defmethod handle :default [args]
   (println "Error: no ipc handler for: " (bean/->js args)))
 

src/main/frontend/config.cljs

@@ -299,6 +299,10 @@
   [s]
   (string/replace s local-db-prefix ""))
 
+(defn get-local-repo
+  [dir]
+  (str local-db-prefix dir))
+
 (defn get-repo-dir
   [repo-url]
   (if (util/electron?)

src/main/frontend/fs.cljs

@@ -105,6 +105,10 @@
           [(:path dir) paths])
         result))))
 
+(defn watch-dir!
+  [dir]
+  (protocol/watch-dir! node-record dir))
+
 (defn mkdir-if-not-exists
   [dir]
   (when dir

src/main/frontend/fs/bfs.cljs

@@ -35,4 +35,6 @@
   (open-dir [this ok-handler]
     nil)
   (get-files [this path-or-handle ok-handler]
+    nil)
+  (watch-dir! [this dir]
     nil))

src/main/frontend/fs/nfs.cljs

@@ -198,4 +198,8 @@
     (utils/openDirectory #js {:recursive true}
                          ok-handler))
   (get-files [this path-or-handle ok-handler]
-    (utils/getFiles path-or-handle true ok-handler)))
+    (utils/getFiles path-or-handle true ok-handler))
+
+  ;; TODO:
+  (watch-dir! [this dir]
+    nil))

src/main/frontend/fs/node.cljs

@@ -44,4 +44,6 @@
   (open-dir [this ok-handler]
     (ipc/ipc "openDir" {}))
   (get-files [this path-or-handle ok-handler]
-    (ipc/ipc "getFiles" path-or-handle)))
+    (ipc/ipc "getFiles" path-or-handle))
+  (watch-dir! [this dir]
+    (ipc/ipc "addDirWatcher" dir)))

src/main/frontend/fs/protocol.cljs

@@ -10,4 +10,5 @@
   (rename! [this repo old-path new-path])
   (stat [this dir path])
   (open-dir [this ok-handler])
-  (get-files [this path-or-handle ok-handler]))
+  (get-files [this path-or-handle ok-handler])
+  (watch-dir! [this dir]))

src/main/frontend/fs/watcher_handler.cljs

@@ -0,0 +1,38 @@
+(ns frontend.fs.watcher-handler
+  (:require [clojure.core.async :as async]
+            [lambdaisland.glogi :as log]
+            [frontend.handler.file :as file-handler]
+            [frontend.handler.page :as page-handler]
+            [frontend.config :as config]
+            [cljs-bean.core :as bean]))
+
+(defn handle-changed!
+  [type {:keys [dir path content stat] :as payload}]
+  (when dir
+    (let [repo (config/get-local-repo dir)]
+      (prn "handle file notifier: "
+           {:repo repo
+            :type type
+            :payload payload})
+     ;; (cond
+     ;;   (contains? #{"add" "change"} type)
+     ;;   ;; TODO: check content and mtime
+     ;;   (file-handler/alter-file repo path content {})
+
+     ;;   (= "unlink" type)
+     ;;   ;; TODO: Remove page and blocks too
+     ;;   ;; what if it's a mistaken, should we put it to .trash to have a way to restore it back?
+     ;;   (file-handler/remove-file! repo path)
+
+     ;;   :else
+     ;;   (log/error :fs/watcher-no-handler {:type type
+     ;;                                      :payload payload}))
+     )))
+
+(defn run-dirs-watcher!
+  []
+  ;; TODO: move "file-watcher" to electron.ipc.channels
+  (js/window.apis.on "file-watcher"
+                     (fn [data]
+                       (let [{:keys [type payload]} (bean/->clj data)]
+                         (handle-changed! type payload)))))

src/main/frontend/handler.cljs

@@ -19,6 +19,7 @@
             [frontend.handler.ui :as ui-handler]
             [frontend.handler.export :as export-handler]
             [frontend.handler.web.nfs :as nfs]
+            [frontend.fs.watcher-handler :as fs-watcher-handler]
             [frontend.ui :as ui]
             [goog.object :as gobj]
             [frontend.idb :as idb]
@@ -105,7 +106,8 @@
                                (fn []
                                  (js/console.error "Failed to request GitHub app tokens."))))
 
-                            (watch-for-date!)))
+                            (watch-for-date!)
+                            (file-handler/watch-for-local-dirs!)))
                          (p/catch (fn [error]
                                     (log/error :db/restore-failed error))))))]
     ;; clear this interval
@@ -156,4 +158,6 @@
     (reset! db/*sync-search-indice-f search/sync-search-indice!)
     (db/run-batch-txs!)
     (file-handler/run-writes-chan!)
-    (editor-handler/periodically-save!)))
+    (editor-handler/periodically-save!)
+    (when (util/electron?)
+      (fs-watcher-handler/run-dirs-watcher!))))

src/main/frontend/handler/file.cljs

@@ -306,3 +306,14 @@
         (<p! (apply alter-files-handler! args)))
       (recur))
     chan))
+
+(defn watch-for-local-dirs!
+  []
+  (when (util/electron?)
+    (let [repos (->> (state/get-repos)
+                     (filter (fn [repo]
+                               (config/local-db? (:url repo)))))
+         directories (map (fn [repo] (config/get-repo-dir (:url repo))) repos)]
+      (doseq [dir directories]
+        (prn "watch for dir changes: " dir)
+        (fs/watch-dir! dir)))))

src/main/frontend/handler/web/nfs.cljs

@@ -205,6 +205,54 @@
      :modified modified
      :deleted  deleted}))
 
+(defn- handle-diffs!
+  [repo nfs? old-files new-files handle-path path-handles re-index?]
+  (let [get-file-f (fn [path files] (some #(when (= (:file/path %) path) %) files))
+        {:keys [added modified deleted] :as diffs} (compute-diffs old-files new-files)
+        ;; Use the same labels as isomorphic-git
+        rename-f (fn [typ col] (mapv (fn [file] {:type typ :path file}) col))
+        _ (when (and nfs? (seq deleted))
+            (let [deleted (doall
+                           (-> (map (fn [path] (if (= "/" (first path))
+                                                 path
+                                                 (str "/" path))) deleted)
+                               (distinct)))]
+              (p/all (map (fn [path]
+                            (let [handle-path (str handle-path path)]
+                              (idb/remove-item! handle-path)
+                              (nfs/remove-nfs-file-handle! handle-path))) deleted))))
+        added-or-modified (set (concat added modified))
+        _ (when (and nfs? (seq added-or-modified))
+            (p/all (map (fn [path]
+                          (when-let [handle (get @path-handles path)]
+                            (idb/set-item! (str handle-path path) handle))) added-or-modified)))]
+    (-> (p/all (map (fn [path]
+                      (when-let [file (get-file-f path new-files)]
+                        (p/let [content (if nfs?
+                                          (.text (:file/file file))
+                                          (:file/content file))]
+                          (assoc file :file/content content)))) added-or-modified))
+        (p/then (fn [result]
+                  (let [files (map #(dissoc % :file/file :file/handle) result)
+                        non-modified? (fn [file]
+                                        (let [content (:file/content file)
+                                              old-content (:file/content (get-file-f (:file/path file) old-files))]
+                                          (= content old-content)))
+                        non-modified-files (->> (filter non-modified? files)
+                                                (map :file/path))
+                        [modified-files modified] (if re-index?
+                                                    [files (set modified)]
+                                                    [(remove non-modified? files) (set/difference (set modified) (set non-modified-files))])
+                        diffs (concat
+                               (rename-f "remove" deleted)
+                               (rename-f "add" added)
+                               (rename-f "modify" modified))]
+                    (when (or (and (seq diffs) (seq modified-files))
+                              (seq diffs))
+                      (repo-handler/load-repo-to-db! repo
+                                                     {:diffs     diffs
+                                                      :nfs-files modified-files}))))))))
+
 (defn- reload-dir!
   ([repo]
    (reload-dir! repo false))
@@ -236,53 +284,8 @@
                                                               (contains? file-paths
                                                                          (string/replace-first path (str dir-name "/") ""))))
                                                     (into {})))))
-                        (set-files! @path-handles))
-                    get-file-f (fn [path files] (some #(when (= (:file/path %) path) %) files))
-                    {:keys [added modified deleted] :as diffs} (compute-diffs old-files new-files)
-                    ;; Use the same labels as isomorphic-git
-                    rename-f (fn [typ col] (mapv (fn [file] {:type typ :path file}) col))
-                    _ (when (and nfs? (seq deleted))
-                        (let [deleted (doall
-                                       (-> (map (fn [path] (if (= "/" (first path))
-                                                             path
-                                                             (str "/" path))) deleted)
-                                           (distinct)))]
-                          (p/all (map (fn [path]
-                                        (let [handle-path (str handle-path path)]
-                                          (idb/remove-item! handle-path)
-                                          (nfs/remove-nfs-file-handle! handle-path))) deleted))))
-                    added-or-modified (set (concat added modified))
-                    _ (when (and nfs? (seq added-or-modified))
-                        (p/all (map (fn [path]
-                                      (when-let [handle (get @path-handles path)]
-                                        (idb/set-item! (str handle-path path) handle))) added-or-modified)))]
-              (-> (p/all (map (fn [path]
-                                (when-let [file (get-file-f path new-files)]
-                                  (p/let [content (if nfs?
-                                                    (.text (:file/file file))
-                                                    (:file/content file))]
-                                    (assoc file :file/content content)))) added-or-modified))
-                  (p/then (fn [result]
-                            (let [files (map #(dissoc % :file/file :file/handle) result)
-                                  non-modified? (fn [file]
-                                                  (let [content (:file/content file)
-                                                        old-content (:file/content (get-file-f (:file/path file) old-files))]
-                                                    (= content old-content)))
-                                  non-modified-files (->> (filter non-modified? files)
-                                                          (map :file/path))
-                                  [modified-files modified] (if re-index?
-                                                              [files (set modified)]
-                                                              [(remove non-modified? files) (set/difference (set modified) (set non-modified-files))])
-                                  diffs (concat
-                                         (rename-f "remove" deleted)
-                                         (rename-f "add" added)
-                                         (rename-f "modify" modified))]
-                              (when (or (and (seq diffs) (seq modified-files))
-                                        (seq diffs) ; delete
-)
-                                (repo-handler/load-repo-to-db! repo
-                                                               {:diffs     diffs
-                                                                :nfs-files modified-files})))))))))
+                        (set-files! @path-handles))]
+              (handle-diffs! repo nfs? old-files new-files handle-path path-handles re-index?))))
         (p/catch (fn [error]
                    (log/error :nfs/load-files-error error)))
         (p/finally (fn [_]

yarn.lock

@@ -1097,6 +1097,21 @@ chokidar@^3.3.0, chokidar@^3.3.1:
   optionalDependencies:
     fsevents "~2.1.2"
 
+chokidar@^3.5.1:
+  version "3.5.1"
+  resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.5.1.tgz#ee9ce7bbebd2b79f49f304799d5468e31e14e68a"
+  integrity sha512-9+s+Od+W0VJJzawDma/gvBNQqkTiqYTWLuZoyAsivsI4AaWTCzHG06/TMjsf1cYe9Cb97UCEhjz7HvnPk2p/tw==
+  dependencies:
+    anymatch "~3.1.1"
+    braces "~3.0.2"
+    glob-parent "~5.1.0"
+    is-binary-path "~2.1.0"
+    is-glob "~4.0.1"
+    normalize-path "~3.0.0"
+    readdirp "~3.5.0"
+  optionalDependencies:
+    fsevents "~2.3.1"
+
 cipher-base@^1.0.0, cipher-base@^1.0.1, cipher-base@^1.0.3:
   version "1.0.4"
   resolved "https://registry.yarnpkg.com/cipher-base/-/cipher-base-1.0.4.tgz#8760e4ecc272f4c363532f926d874aae2c1397de"
@@ -2374,6 +2389,11 @@ fsevents@~2.1.2:
   resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.1.3.tgz#fb738703ae8d2f9fe900c33836ddebee8b97f23e"
   integrity sha512-Auw9a4AxqWpa9GUfj370BMPzzyncfBABW8Mab7BGWBYDj4Isgq+cDKtx0i6u9jcX9pQDnswsaaOTgTmA5pEjuQ==
 
+fsevents@~2.3.1:
+  version "2.3.1"
+  resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.1.tgz#b209ab14c61012636c8863507edf7fb68cc54e9f"
+  integrity sha512-YR47Eg4hChJGAB1O3yEAOkGO+rlzutoICGqGo9EZ4lKWokzZRSyIW1QmTzqjtw8MJdj9srP869CuWw/hyzSiBw==
+
 function-bind@^1.1.1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d"