enhance(sync): bb file-sync:integration-tests, add random tests

scripts/src/logseq/tasks/file_sync.clj

@@ -3,14 +3,16 @@
 
 * Login to electron app and toggle file-sync on
 * Set up file-sync-auth.json file per #'read-config
-* Run `bb file-sync:integration-tests GRAPH_DIRECTORY`
+* Run `bb file-sync:integration-tests`
 * Wait for test results. Each action takes 10-20s and prints results as it goes"
   (:require [clojure.string :as str]
             [cheshire.core :as json]
+            [clojure.pprint :as pp]
             [babashka.fs :as fs]
             [babashka.curl :as curl]
             [clojure.data :as data]
-            [clojure.test :as t :refer [deftest is]])
+            [clojure.test :as t :refer [deftest is]]
+            [logseq.tasks.file-sync-actions :as file-sync-actions])
   (:import (java.net URLDecoder)))
 
 ;; Root directory for graph that is being tested
@@ -93,7 +95,9 @@
     (spit (fs/file dir file) new-content)))
 
 (defn run-action [action-map]
-  (println "\n===\nRUN" (pr-str action-map) "\n===")
+  (println "==\nRUN")
+  (pp/pprint ((juxt :action #(get-in % [:args :file])) action-map))
+  (println "===")
   (run-action* action-map))
 
 (defn- ensure-dir-is-synced!
@@ -133,39 +137,35 @@
          false)))
 
 (defn- wait&files-are-in-sync?
-  [dir graph-id subdir]
+  [dir graph-id subdir & [msg]]
   ;; Approximate polling time before file changes are picked up by client
-  (println "Wait 10s for logseq to pick up changes...")
+  (if msg
+    (println msg)
+    (println "Wait 10s for logseq to pick up changes..."))
   (Thread/sleep 10000)
   (try-fn-n-times #(files-are-in-sync? dir graph-id subdir) 10))
 
-(deftest file-changes
+(defn- clear-dir-pages
+  [subdir]
+  (fs/delete-tree (str @root-dir "/" subdir))
+  (fs/create-dir (str @root-dir "/" subdir)))
+
+(deftest rand-file-changes
   (let [subdir "pages"
-        ;; Directory must be in sync in order for assertions to pass
-        _ (ensure-dir-is-synced! @root-dir @root-graph-id subdir)
-        ;; These actions are data driven which allows us to spec to generate them
-        ;; when the API is able to handle more randomness
+        actions (:generated-action (file-sync-actions/generate-rand-actions 30))
         actions (mapv
                  #(assoc-in % [:args :dir] @root-dir)
-                 [{:action :create-file
-                   :args {:file (str subdir "/test.create-page.md")
-                          :blocks ["hello world"]}}
-                  {:action :move-file
-                   :args {:file (str subdir "/test.create-page.md")
-                          :new-file (str subdir "/test.create-page-new.md")}}
-                  {:action :update-file
-                   :args {:file (str subdir "/test.create-page-new.md")
-                          :blocks ["update line1" "update line2"]}}
-                  {:action :delete-file
-                   :args {:file (str subdir "/test.create-page-new.md")}}])]
-
-    (doseq [action-map actions]
-      (run-action action-map)
+                 actions)
+        partitioned-actions (partition-all 3 actions)]
+    (clear-dir-pages subdir)
+    (wait&files-are-in-sync? @root-dir @root-graph-id subdir
+                             (format "clear dir [%s], and ensure it's in sync" subdir))
+    (doseq [actions partitioned-actions]
+      (doseq [action actions]
+        (run-action action)
+        (Thread/sleep 500))
       (is (wait&files-are-in-sync? @root-dir @root-graph-id subdir)
-          (str "Test " (select-keys action-map [:action]))))))
-
-
-;; (deftest file)
+          (str "Test " (mapv (juxt :action #(get-in % [:args :file])) actions))))))
 
 (defn setup-vars
   []
@@ -177,7 +177,17 @@
     (reset! root-graph-id graph-id)))
 
 (defn integration-tests
-  "Run file-sync integration tests on graph directory"
+  "Run file-sync integration tests on graph directory
+  requirements:
+
+  * file-sync-auth.json, and it looks like:
+  ```
+  {\"id_token\":\"<id-token>\",
+    \"dir\": \"/Users/me/Documents/untitled folder 31\"}
+  ```
+
+  * you alse need to open logseq-app(or yarn electron-watch),
+    and open <dir> and start file-sync"
   [& _args]
   (setup-vars)
   (t/run-tests 'logseq.tasks.file-sync))

scripts/src/logseq/tasks/file_sync_actions.clj

@@ -0,0 +1,164 @@
+(ns logseq.tasks.file-sync-actions
+  (:require [clojure.test.check.generators :as gen]))
+
+
+(defmulti gen-action* (fn [& args] (first args)))
+
+(defmethod gen-action* :create-file
+  [_ page-index & _args]
+  (gen/let [blocks (gen/vector gen/string-alphanumeric 1 10)]
+    {:action :create-file
+     :args {:file (format "pages/test.page-%d.md" page-index)
+            :blocks blocks}}))
+
+(defmethod gen-action* :move-file
+  [_ origin-page-index & [moved?]]
+  (let [page-name (if moved?
+                      (format "pages/test.page-move-%d.md" origin-page-index)
+                      (format "pages/test.page-%d.md" origin-page-index))]
+    (gen/return
+     {:action :move-file
+      :args {:file page-name
+             :new-file (format "pages/test.page-move-%d.md" origin-page-index)}})))
+
+(defmethod gen-action* :update-file
+  [_ page-index & [moved?]]
+  (gen/let [append-blocks (gen/vector gen/string-alphanumeric 1 10)]
+    (let [page-name (if moved?
+                      (format "pages/test.page-move-%d.md" page-index)
+                      (format "pages/test.page-%d.md" page-index))]
+      {:action :update-file
+       :args {:file page-name
+              :blocks append-blocks}})))
+
+(defmethod gen-action* :delete-file
+  [_ page-index & [moved?]]
+  (let [page-name (if moved?
+                    (format "pages/test.page-move-%d.md" page-index)
+                    (format "pages/test.page-%d.md" page-index))]
+    (gen/return
+     {:action :delete-file
+      :args {:file page-name}})))
+
+
+(defmacro gen-actions-plan
+  "state monad
+  state: {:page-index [{:index 1 :moved? false}, ...]
+          :generated-action [...]}
+
+  (gen-actions-plan
+     [id+moved? get-rand-available-index-op
+      _ (when-op id+moved? (apply action-op action id+moved?))]
+     nil)"
+  [binds val-expr]
+  (let [binds (partition 2 binds)
+        psym (gensym "state_")
+        forms (reduce (fn [acc [id expr]]
+                        (concat acc `[[~id ~psym] (~expr ~psym)]))
+                      []
+                      binds)]
+    `(fn [~psym]
+       (let [~@forms]
+         [~val-expr ~psym]))))
+
+(defn- all-indexes
+  [state]
+  (let [r (map :index (:page-index state))]
+    (if (empty? r) '(0) r)))
+
+(defn- add-index
+  [state index moved?]
+  (update state :page-index conj {:index index :moved? moved?}))
+
+(defn- assign-page-index-op
+  [state]
+  (let [max-index (apply max (all-indexes state))
+          next-index (inc max-index)]
+      [next-index (add-index state next-index false)]))
+
+(defn- get-rand-available-index-op
+  [state]
+  (let [indexes (:page-index state)]
+    (if (empty? indexes)
+      [nil state]
+      (let [rand-index (rand-nth (vec indexes))]
+        [((juxt :index :moved?) rand-index) state]))))
+
+(defn- action-op
+  [action id & args]
+  (fn [state]
+    (let [generated-action (gen/generate (apply gen-action* action id args))
+          [moved?] args
+          state* (update state :generated-action conj generated-action)
+          state* (case action
+                   :move-file
+                   (update state* :page-index
+                           #(-> %
+                                (disj {:index id :moved? (boolean moved?)})
+                                (conj {:index id :moved? true})))
+                   :delete-file
+                   (update state* :page-index
+                           #(disj % {:index id :moved? (boolean moved?)}))
+                   state*)]
+      [nil state*])))
+
+(defmacro when-op
+  [x f]
+  `(fn [state#]
+     (if ~x
+       (~f state#)
+       [nil state#])))
+
+(defn- print-op
+  [x]
+  (fn [state]
+    (println x)
+    [nil state]))
+
+
+(defn rand-action-op
+  []
+  (let [action (gen/generate
+                (gen/frequency [[5 (gen/return :update-file)]
+                                [2 (gen/return :create-file)]
+                                [2 (gen/return :move-file)]
+                                [1 (gen/return :delete-file)]]))]
+    (case action
+      :create-file
+      (gen-actions-plan
+       [id assign-page-index-op
+        _ (action-op action id)]
+       nil)
+      :update-file
+      (gen-actions-plan
+       [id+moved? get-rand-available-index-op
+        _ (when-op id+moved? (apply action-op action id+moved?))]
+       nil)
+      :move-file
+      (gen-actions-plan
+       [id+moved? get-rand-available-index-op
+        _ (when-op id+moved? (apply action-op action id+moved?))]
+       nil)
+      :delete-file
+      (gen-actions-plan
+       [id+moved? get-rand-available-index-op
+        _ (when-op id+moved? (apply action-op action id+moved? ))]
+       nil))))
+
+(def empty-actions-plan {:page-index #{}
+                         :generated-action []})
+
+
+(defmacro generate-rand-actions
+  [max-n & {:keys [pre-create-files-n] :or {pre-create-files-n 2}}]
+  (let [pre-create-files-binds
+        (for [id (map (fn [_] (gensym)) (range pre-create-files-n))]
+          `[~id assign-page-index-op
+            ~'_ (action-op :create-file ~id)])
+        binds (apply concat
+                     (concat pre-create-files-binds (repeat max-n `[~'_ (rand-action-op)])))]
+    `(second
+      ((gen-actions-plan
+        ~binds
+        nil)
+       empty-actions-plan))))

src/main/frontend/fs/sync.cljs

@@ -1737,9 +1737,10 @@
                (.start sm)
 
 
-               (if (zero? txid)
-                 (offer! remote->local-full-sync-chan true)
-                 (offer! remote->local-sync-chan true))
+               ;; (if (zero? txid)
+               ;;   (offer! remote->local-full-sync-chan true)
+               ;;   (offer! remote->local-sync-chan true))
+               (offer! remote->local-full-sync-chan true)
                (offer! full-sync-chan true)
 
                ;; watch :network/online?