Merge branch 'master' into feat/whiteboards-onboarding

android/app/src/main/java/com/logseq/app/FsWatcher.java

@@ -1,7 +1,5 @@
 package com.logseq.app;
 
-import android.annotation.SuppressLint;
-import android.os.Build;
 import android.system.ErrnoException;
 import android.system.Os;
 import android.system.StructStat;
@@ -12,8 +10,9 @@ import android.net.Uri;
 
 import java.io.*;
 
-import java.util.ArrayList;
-import java.util.List;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Stack;
 import java.util.regex.Pattern;
 
 import java.io.File;
@@ -26,9 +25,9 @@ import com.getcapacitor.PluginCall;
 
 @CapacitorPlugin(name = "FsWatcher")
 public class FsWatcher extends Plugin {
-
-    List<SingleFileObserver> observers;
     private String mPath;
+    private PollingFsWatcher mWatcher;
+    private Thread mThread;
 
     @Override
     public void load() {
@@ -37,14 +36,11 @@ public class FsWatcher extends Plugin {
 
     @PluginMethod()
     public void watch(PluginCall call) {
-        if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) {
-            call.reject("Android version not supported");
-            return;
-        }
         String pathParam = call.getString("path");
         // check file:// or no scheme uris
         Uri u = Uri.parse(pathParam);
         Log.i("FsWatcher", "watching " + u);
+        // TODO: handle context:// uri
         if (u.getScheme() == null || u.getScheme().equals("file")) {
             File pathObj;
             try {
@@ -56,32 +52,15 @@ public class FsWatcher extends Plugin {
 
             mPath = pathObj.getAbsolutePath();
 
-            int mask = FileObserver.CLOSE_WRITE |
-                    FileObserver.MOVE_SELF | FileObserver.MOVED_FROM | FileObserver.MOVED_TO |
-                    FileObserver.DELETE | FileObserver.DELETE_SELF | FileObserver.CREATE;
-
-            if (observers != null) {
+            if (mWatcher != null) {
                 call.reject("already watching");
                 return;
             }
-            observers = new ArrayList<>();
-            observers.add(new SingleFileObserver(pathObj, mask));
-
-            // NOTE: only watch first level of directory
-            File[] files = pathObj.listFiles();
-            if (files != null) {
-                for (File file : files) {
-                    String filename = file.getName();
-                    if (file.isDirectory() && !filename.startsWith(".") && !filename.equals("bak") && !filename.equals("version-files") && !filename.equals("node_modules")) {
-                        observers.add(new SingleFileObserver(file, mask));
-                    }
-                }
-            }
 
-            this.initialNotify(pathObj);
+            mWatcher = new PollingFsWatcher(mPath);
+            mThread = new Thread(mWatcher);
+            mThread.start();
 
-            for (int i = 0; i < observers.size(); i++)
-                observers.get(i).startWatching();
             call.resolve();
         } else {
             call.reject(u.getScheme() + " scheme not supported");
@@ -92,77 +71,69 @@ public class FsWatcher extends Plugin {
     public void unwatch(PluginCall call) {
         Log.i("FsWatcher", "unwatch all...");
 
-        if (observers != null) {
-            for (int i = 0; i < observers.size(); ++i)
-                observers.get(i).stopWatching();
-            observers.clear();
-            observers = null;
+        if (mWatcher != null) {
+            mThread.interrupt();
+            mWatcher = null;
         }
 
         call.resolve();
     }
 
-    public void initialNotify(File pathObj) {
-        this.initialNotify(pathObj, 2);
-    }
-
-    public void initialNotify(File pathObj, int maxDepth) {
-        if (maxDepth == 0) {
-            return;
-        }
-        File[] files = pathObj.listFiles();
-        if (files != null) {
-            for (File file : files) {
-                String filename = file.getName();
-                if (file.isDirectory() && !filename.startsWith(".") && !filename.equals("bak") && !filename.equals("version-files") && !filename.equals("node_modules")) {
-                    this.initialNotify(file, maxDepth - 1);
-                } else if (file.isFile()
-                        && Pattern.matches("(?i)[^.].*?\\.(md|org|css|edn|js|markdown)$",
-                        file.getName())) {
-                    this.onObserverEvent(FileObserver.CREATE, file.getAbsolutePath());
-                }
-            }
-        }
-    }
-
     // add, change, unlink events
-    public void onObserverEvent(int event, String path) {
+    public void onObserverEvent(int event, String path, SimpleFileMetadata metadata) {
         JSObject obj = new JSObject();
         String content = null;
         File f = new File(path);
+
+        boolean shouldRead = false;
+        if (Pattern.matches("(?i)[^.].*?\\.(md|org|css|edn|js|markdown|excalidraw)$", f.getName())) {
+            shouldRead = true;
+        }
+
         obj.put("path", Uri.fromFile(f));
         obj.put("dir", Uri.fromFile(new File(mPath)));
+        JSObject stat;
 
         switch (event) {
-            case FileObserver.CLOSE_WRITE:
+            case FileObserver.MODIFY:
                 obj.put("event", "change");
-                try {
-                    obj.put("stat", getFileStat(path));
-                    content = getFileContents(f);
-                } catch (IOException | ErrnoException e) {
-                    e.printStackTrace();
+                stat = new JSObject();
+                stat.put("mtime", metadata.mtime);
+                stat.put("ctime", metadata.ctime);
+                stat.put("size", metadata.size);
+                obj.put("stat", stat);
+                if (shouldRead) {
+                    try {
+                        content = getFileContents(f);
+                    } catch (IOException e) {
+                        Log.e("FsWatcher", "error reading modified file");
+                        e.printStackTrace();
+                    }
                 }
+
                 Log.i("FsWatcher", "prepare event " + obj);
                 obj.put("content", content);
                 break;
-            case FileObserver.MOVED_TO:
             case FileObserver.CREATE:
                 obj.put("event", "add");
-                try {
-                    obj.put("stat", getFileStat(path));
-                    content = getFileContents(f);
-                } catch (IOException | ErrnoException e) {
-                    e.printStackTrace();
+                stat = new JSObject();
+                stat.put("mtime", metadata.mtime);
+                stat.put("ctime", metadata.ctime);
+                stat.put("size", metadata.size);
+                obj.put("stat", stat);
+                if (shouldRead) {
+                    try {
+                        content = getFileContents(f);
+                    } catch (IOException e) {
+                        Log.e("FsWatcher", "error reading new file");
+                        e.printStackTrace();
+                    }
                 }
-                Log.i("FsWatcher", "prepare event " + obj);
                 obj.put("content", content);
                 break;
-            case FileObserver.MOVE_SELF:
-            case FileObserver.MOVED_FROM:
             case FileObserver.DELETE:
-            case FileObserver.DELETE_SELF:
                 if (f.exists()) {
-                    Log.i("FsWatcher", "abandon notification due to file exists");
+                    Log.i("FsWatcher", "abandon delete notification due to file exists");
                     return;
                 } else {
                     obj.put("event", "unlink");
@@ -193,59 +164,120 @@ public class FsWatcher extends Plugin {
         return outputStream.toString("utf-8");
     }
 
-    public static JSObject getFileStat(final String path) throws ErrnoException {
-        File file = new File(path);
-        StructStat stat = Os.stat(path);
-        JSObject obj = new JSObject();
-        obj.put("atime", stat.st_atime);
-        obj.put("mtime", stat.st_mtime);
-        obj.put("ctime", stat.st_ctime);
-        obj.put("size", file.length());
-        return obj;
+    public class SimpleFileMetadata {
+        public long mtime;
+        public long ctime;
+        public long size;
+        public long ino;
+
+        public SimpleFileMetadata(File file) throws ErrnoException {
+            StructStat stat = Os.stat(file.getPath());
+            mtime = stat.st_mtime;
+            ctime = stat.st_ctime;
+            size = stat.st_size;
+            ino = stat.st_ino;
+        }
+
+        public boolean equals(SimpleFileMetadata other) {
+            return mtime == other.mtime && ctime == other.ctime && size == other.size && ino == other.ino;
+        }
     }
 
-    private class SingleFileObserver extends FileObserver {
-        private final String mPath;
 
-        public SingleFileObserver(String path, int mask) {
-            super(path, mask);
-            mPath = path;
-        }
+    public class PollingFsWatcher implements Runnable {
+        private String mPath;
+        private Map<String, SimpleFileMetadata> metaDb;
+
+        public PollingFsWatcher(String path) {
+            metaDb = new HashMap();
 
-        @SuppressLint("NewApi")
-        public SingleFileObserver(File path, int mask) {
-            super(path, mask);
-            mPath = path.getAbsolutePath();
+            File dir = new File(path);
+            try {
+                mPath = dir.getCanonicalPath();
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
         }
 
         @Override
-        public void onEvent(int event, String path) {
-            if (path != null && !path.equals("graphs-txid.edn") && !path.equals("broken-config.edn")) {
-                Log.d("FsWatcher", "got path=" + mPath + "/" + path + " event=" + event);
-                // TODO: handle newly created directory
-                if (Pattern.matches("(?i)[^.].*?\\.(md|org|css|edn|js|markdown)$", path)) {
-                    String fullPath = mPath + "/" + path;
-                    if (event == FileObserver.MOVE_SELF || event == FileObserver.MOVED_FROM ||
-                        event == FileObserver.DELETE || event == FileObserver.DELETE_SELF) {
-                        Log.d("FsWatcher", "defer delete notification for " + path);
-                        Thread timer = new Thread() {
-                            @Override
-                            public void run() {
-                                try {
-                                    // delay 500ms then send, enough for most syncing net disks
-                                    Thread.sleep(500);
-                                    FsWatcher.this.onObserverEvent(event, fullPath);
-                                } catch (InterruptedException e) {
-                                    e.printStackTrace();
-                                }
+        public void run() {
+            while (!Thread.currentThread().isInterrupted()) {
+                try {
+                    this.tick();
+                    Thread.sleep(2000); // The same as iOS fswatcher, 2s interval
+                } catch (InterruptedException e) {
+                    // e.printStackTrace();
+                    Log.i("FsWatcher", "interrupted, unwatch");
+                    break;
+                }
+            }
+
+        }
+
+        private void tick() {
+            Map<String, SimpleFileMetadata> newMetaDb = new HashMap();
+
+            Stack<String> paths = new Stack();
+            paths.push(mPath);
+            while (!paths.isEmpty()) {
+                String dir = paths.pop();
+                File curr = new File(dir);
+
+                File[] files = curr.listFiles();
+                if (files != null) {
+                    for (File file : files) {
+                        String filename = file.getName();
+                        if (file.isDirectory()) {
+                            if (!filename.startsWith(".") && !filename.equals("bak") && !filename.equals("version-files") && !filename.equals("node_modules")) {
+                                paths.push(file.getAbsolutePath());
                             }
-                        };
-                        timer.start();
-                    } else {
-                        FsWatcher.this.onObserverEvent(event, fullPath);
+                        } else if (file.isFile() && !filename.equals("graphs-txid.edn") && !filename.equals("broken-config.edn")) {
+                            try {
+                                SimpleFileMetadata metadata = new SimpleFileMetadata(file);
+                                newMetaDb.put(file.getAbsolutePath(), metadata);
+                            } catch (ErrnoException e) {
+                            }
+                        }
                     }
                 }
             }
+            this.updateMetaDb(newMetaDb);
+        }
+
+        private void updateMetaDb(Map<String, SimpleFileMetadata> newMetaDb) {
+            for (Map.Entry<String, SimpleFileMetadata> entry : newMetaDb.entrySet()) {
+                String path = entry.getKey();
+                SimpleFileMetadata newMeta = entry.getValue();
+                SimpleFileMetadata oldMeta = metaDb.remove(path);
+                if (oldMeta == null) {
+                    // new file
+                    onObserverEvent(FileObserver.CREATE, path, newMeta);
+                    Log.d("FsWatcher", "create " + path);
+                } else if (!oldMeta.equals(newMeta)) {
+                    // file changed
+                    onObserverEvent(FileObserver.MODIFY, path, newMeta);
+                    Log.d("FsWatcher", "changed " + path);
+                }
+            }
+            for (String path : metaDb.keySet()) {
+                // file deleted
+                Thread timer = new Thread() {
+                    @Override
+                    public void run() {
+                        try {
+                            // delay 500ms then send, enough for most syncing net disks
+                            Thread.sleep(500);
+                            onObserverEvent(FileObserver.DELETE, path, null);
+                            Log.d("FsWatcher", "deleted " + path);
+                        } catch (InterruptedException e) {
+                            e.printStackTrace();
+                        }
+                    }
+                };
+                timer.start();
+            }
+
+            this.metaDb = newMetaDb;
         }
     }
 }

src/main/frontend/handler/editor.cljs

@@ -1907,6 +1907,11 @@
          (edit-block! last-block' :max (:block/uuid last-block')))))
    0))
 
+(defn- nested-blocks
+  [blocks]
+  (let [ids (set (map :db/id blocks))]
+    (some? (some #(ids (:db/id (:block/parent %))) blocks))))
+
 (defn paste-blocks
   "Given a vec of blocks, insert them into the target page.
    keep-uuid?: if true, keep the uuid provided in the block structure."
@@ -1926,12 +1931,25 @@
         block (db/entity (:db/id target-block))
         page (if (:block/name block) block
                  (when target-block (:block/page (db/entity (:db/id target-block)))))
-        target-block (or target-block editing-block)
+        empty-target? (string/blank? (:block/content target-block))
+        paste-nested-blocks? (nested-blocks blocks)
+        target-block-has-children? (db/has-children? (:block/uuid target-block))
+        replace-empty-target? (if (and paste-nested-blocks? empty-target?
+                                       target-block-has-children?)
+                                false
+                                true)
+        target-block' (if replace-empty-target? target-block
+                          (db/pull (:db/id (:block/left target-block))))
         sibling? (cond
+                   (and paste-nested-blocks? empty-target?)
+                   (if (= (:block/parent target-block') (:block/parent target-block))
+                     true
+                     false)
+
                    (some? sibling?)
                    sibling?
 
-                   (db/has-children? (:block/uuid target-block))
+                   target-block-has-children?
                    false
 
                    :else
@@ -1944,15 +1962,15 @@
 
     (outliner-tx/transact!
       {:outliner-op :insert-blocks}
-      (when target-block
-        (let [format (or (:block/format target-block) (state/get-preferred-format))
+      (when target-block'
+        (let [format (or (:block/format target-block') (state/get-preferred-format))
               blocks' (map (fn [block]
                              (paste-block-cleanup block page exclude-properties format content-update-fn))
                            blocks)
-              result (outliner-core/insert-blocks! blocks' target-block {:sibling? sibling?
-                                                                         :outliner-op :paste
-                                                                         :replace-empty-target? true
-                                                                         :keep-uuid? keep-uuid?})]
+              result (outliner-core/insert-blocks! blocks' target-block' {:sibling? sibling?
+                                                                          :outliner-op :paste
+                                                                          :replace-empty-target? replace-empty-target?
+                                                                          :keep-uuid? keep-uuid?})]
           (edit-last-block-after-inserted! result))))))
 
 (defn- block-tree->blocks

src/main/frontend/handler/web/nfs.cljs

@@ -131,7 +131,6 @@
          *repo (atom nil)
          dir (or dir nil)
          dir (some-> dir
-                     (string/replace "file:///" "file://")
                      (string/replace " " "%20"))]
      ;; TODO: add ext filter to avoid loading .git or other ignored file handlers
      (->

src/main/frontend/modules/outliner/core.cljs

@@ -267,9 +267,28 @@
           (recur (rest blocks) (first blocks))
           matched)))))
 
+(defn- get-id
+  [x]
+  (cond
+    (map? x)
+    (:db/id x)
+
+    (vector? x)
+    (second x)
+
+    :else
+    x))
+
 (defn- compute-block-parent
-  [block parent target-block prev-hop top-level? sibling? get-new-id]
+  [block parent target-block prev-hop top-level? sibling? get-new-id outliner-op replace-empty-target? idx]
   (cond
+    ;; replace existing block
+    (and (= outliner-op :paste)
+         replace-empty-target?
+         (string/blank? (:block/content target-block))
+         (zero? idx))
+    (get-id (:block/parent target-block))
+
     prev-hop
     (:db/id (:block/parent prev-hop))
 
@@ -455,7 +474,7 @@
                                                  (not= (:block/parent block) (:block/parent target-block)))
                            prev-hop (if outdented-block? (find-outdented-block-prev-hop block blocks) nil)
                            left-exists-in-blocks? (contains? ids (:db/id (:block/left block)))
-                           parent (compute-block-parent block parent target-block prev-hop top-level? sibling? get-new-id)
+                           parent (compute-block-parent block parent target-block prev-hop top-level? sibling? get-new-id outliner-op replace-empty-target? idx)
                            left (compute-block-left blocks block left target-block prev-hop idx replace-empty-target? left-exists-in-blocks? get-new-id)]
                        (cond->
                          (merge block {:block/uuid uuid

src/test/frontend/modules/outliner/core_test.cljs

@@ -330,6 +330,37 @@
 
       (is (= [19 20] (get-children 18))))))
 
+(deftest test-paste-into-empty-block
+  (testing "
+    Paste a block into the first block (its content is empty)
+    [[22 [[2 [[3 [[4]
+                [5]]]
+            [6 [[7 [[8]]]]]
+            [9 [[10]
+                [11]]]]]
+        [12 [[13]
+             [14]
+             [15]]]
+        [16 [[17]]]]]]
+ "
+    (transact-tree! tree)
+    (db/transact! test-db [{:block/uuid 22
+                            :block/content ""}])
+    (let [target-block (get-block 22)]
+      (outliner-tx/transact!
+        {:graph test-db}
+        (outliner-core/insert-blocks! [{:block/left [:block/uuid 1]
+                                        :block/content "test"
+                                        :block/parent [:block/uuid 1]
+                                        :block/page 1}]
+                                      target-block
+                                      {:sibling? false
+                                       :outliner-op :paste
+                                       :replace-empty-target? true}))
+      (is (= "test" (:block/content (get-block 22))))
+      (is (= [22] (get-children 1)))
+      (is (= [2 12 16] (get-children 22))))))
+
 (deftest test-batch-transact
   (testing "add 4, 5 after 2 and delete 3"
     (let [tree [[1 [[2] [3]]]]]
@@ -691,6 +722,6 @@
 
   (do
     (frontend.test.fixtures/reset-datascript test-db)
-    (cljs.test/test-vars [#'random-deletes]))
+    (cljs.test/test-vars [#'test-paste-first-empty-block]))
 
   )