Merge branch 'master' into disable-webview-resize

src/electron/electron/backup_file.cljs

@@ -0,0 +1,51 @@
+(ns electron.backup-file
+  (:require [clojure.string :as string]
+            ["path" :as path]
+            ["fs" :as fs]
+            ["fs-extra" :as fs-extra]))
+
+(def backup-dir "logseq/bak")
+(def version-file-dir "version-files/local")
+
+(defn- get-backup-dir*
+  [repo relative-path bak-dir]
+  (let [relative-path* (string/replace relative-path repo "")
+        bak-dir (path/join repo bak-dir)
+        path (path/join bak-dir relative-path*)
+        parsed-path (path/parse path)]
+    (path/join (.-dir parsed-path)
+               (.-name parsed-path))))
+
+(defn get-backup-dir
+  [repo relative-path]
+  (get-backup-dir* repo relative-path backup-dir))
+
+(defn get-version-file-dir
+  [repo relative-path]
+  (get-backup-dir* repo relative-path version-file-dir))
+
+(defn- truncate-old-versioned-files!
+  "reserve the latest 3 version files"
+  [dir]
+  (let [files (fs/readdirSync dir (clj->js {:withFileTypes true}))
+        files (mapv #(.-name %) files)
+        old-versioned-files (drop 3 (reverse (sort files)))]
+    (doseq [file old-versioned-files]
+      (fs-extra/removeSync (path/join dir file)))))
+
+(defn backup-file
+  "backup CONTENT under DIR :backup-dir or :version-file-dir
+  :backup-dir = `backup-dir`
+  :version-file-dir = `version-file-dir`"
+  [repo dir relative-path ext content]
+  {:pre [(contains? #{:backup-dir :version-file-dir} dir)]}
+  (let [dir* (case dir
+               :backup-dir (get-backup-dir repo relative-path)
+               :version-file-dir (get-version-file-dir repo relative-path))
+        new-path (path/join dir*
+                            (str (string/replace (.toISOString (js/Date.)) ":" "_")
+                                 ext))]
+    (fs-extra/ensureDirSync dir*)
+    (fs/writeFileSync new-path content)
+    (fs/statSync new-path)
+    (truncate-old-versioned-files! dir*)))

src/electron/electron/handler.cljs

@@ -19,7 +19,8 @@
             [electron.git :as git]
             [electron.plugin :as plugin]
             [electron.window :as win]
-            [electron.file-sync-rsapi :as rsapi]))
+            [electron.file-sync-rsapi :as rsapi]
+            [electron.backup-file :as backup-file]))
 
 (defmulti handle (fn [_window args] (keyword (first args))))
 
@@ -65,45 +66,18 @@
   (let [result (.diff_main Diff old new)]
     (some (fn [a] (= -1 (first a))) result)))
 
-(defn- truncate-old-versioned-files!
-  [dir]
-  (let [files (fs/readdirSync dir (clj->js {:withFileTypes true}))
-        files (map #(.-name %) files)
-        old-versioned-files (drop 3 (reverse (sort files)))]
-    (doseq [file old-versioned-files]
-      (fs-extra/removeSync (path/join dir file)))))
-
-(defn- get-backup-dir
-  [repo path]
-  (let [path (string/replace path repo "")
-        bak-dir (str repo "/logseq/bak")
-        path (str bak-dir path)
-        parsed-path (path/parse path)]
-    (path/join (.-dir parsed-path)
-               (.-name parsed-path))))
-
-(defn backup-file
-  [repo path content]
-  (let [path-dir (get-backup-dir repo path)
-        ext (path/extname path)
-        new-path (path/join path-dir
-                            (str (string/replace (.toISOString (js/Date.)) ":" "_")
-                                 ext))]
-    (fs-extra/ensureDirSync path-dir)
-    (fs/writeFileSync new-path content)
-    (fs/statSync new-path)
-    (truncate-old-versioned-files! path-dir)
-    new-path))
-
 (defmethod handle :backupDbFile [_window [_ repo path db-content new-content]]
   (when (and (string? db-content)
              (string? new-content)
              (string-some-deleted? db-content new-content))
-    (backup-file repo path db-content)))
+    (backup-file/backup-file repo :backup-dir path (path/extname path) db-content)))
+
+(defmethod handle :addVersionFile [_window [_ repo path content]]
+  (backup-file/backup-file repo :version-file-dir path (path/extname path) content))
 
 (defmethod handle :openFileBackupDir [_window [_ repo path]]
   (when (string? path)
-    (let [dir (get-backup-dir repo path)]
+    (let [dir (backup-file/get-backup-dir repo path)]
       (.openPath shell dir))))
 
 (defmethod handle :readFile [_window [_ path]]
@@ -129,7 +103,7 @@
       (fs/statSync path)
       (catch :default e
         (let [backup-path (try
-                            (backup-file repo path content)
+                            (backup-file/backup-file repo :backup-dir path (path/extname path) content)
                             (catch :default e
                               (println "Backup file failed")
                               (js/console.dir e)))]

src/main/frontend/components/block.cljs

@@ -266,7 +266,7 @@
         href (config/get-local-asset-absolute-path href)]
     (when (or granted? (util/electron?) (mobile-util/native-platform?))
       (p/then (editor-handler/make-asset-url href) #(reset! src %)))
-    
+
     (when @src
       (let [ext (keyword (util/get-file-ext @src))
             share-fn (fn [event]
@@ -2438,7 +2438,7 @@
        (fn []
          (block-container-inner state repo config block))
        nil
-       false)
+       {:reset-height? false})
       (block-container-inner state repo config block))))
 
 (defn divide-lists
@@ -2768,7 +2768,7 @@
    (ui/lazy-visible
     (fn [] (custom-query* config q))
     nil
-    true)))
+    {:reset-height? true})))
 
 (defn admonition
   [config type result]

src/main/frontend/components/journal.cljs

@@ -66,7 +66,8 @@
 
 (rum/defc journal-cp
   [journal]
-  (ui/lazy-visible (fn [] (journal-cp-inner journal)) nil true))
+  (ui/lazy-visible (fn [] (journal-cp-inner journal)) nil
+                   {:reset-height? true}))
 
 (rum/defc journals < rum/reactive
   [latest-journals]

src/main/frontend/components/reference.cljs

@@ -93,8 +93,8 @@
       (when (or (seq refed-blocks-ids)
                 (seq scheduled-or-deadlines)
                 (seq filter-state))
-        [:div.references.mt-6.flex-1.flex-row
-         [:div.content
+        [:div.references.flex-1.flex-row
+         [:div.content.pt-6
           (when (seq scheduled-or-deadlines)
             (ui/foldable
              [:h2.font-bold.opacity-50 "SCHEDULED AND DEADLINE"]
@@ -172,7 +172,7 @@
     (fn []
       (references* page-name))
     nil
-    false)))
+    {:reset-height? false})))
 
 (rum/defcs unlinked-references-aux
   < rum/reactive db-mixins/query

src/main/frontend/db/model.cljs

@@ -590,7 +590,11 @@
   (let [db-before (or db-before current-db)
         cached-ids (map :db/id @result)
         cached-ids-set (set (conj cached-ids page-id))
-        first-changed-id (if (= outliner-op :move-blocks)
+        first-changed-id (cond
+                           (= (:real-outliner-op tx-meta) :indent-outdent)
+                           (last (:move-blocks tx-meta))
+
+                           (= outliner-op :move-blocks)
                            (let [{:keys [move-blocks target from-page to-page]} tx-meta]
                              (cond
                                (= page-id target) ; move to the first block
@@ -611,6 +615,7 @@
                                        (when (seq others)
                                          (recur others)))
                                      nil)))))
+                           :else
                            (let [insert? (= :insert-blocks outliner-op)]
                              (some #(when (and (or (and insert? (not (contains? cached-ids-set %)))
                                                    true)
@@ -635,7 +640,6 @@
         (let [start-page? (:block/name (db-utils/entity start-id))]
           (when-not start-page?
             (let [previous-blocks (take-while (fn [b] (not= start-id (:db/id b))) @result)
-                  previous-count (count previous-blocks)
                   limit 25
                   more (get-paginated-blocks-no-cache current-db start-id {:limit limit
                                                                            :include-start? true

src/main/frontend/diff.cljs

@@ -10,7 +10,7 @@
 
 (defn diff
   [s1 s2]
-  (-> ((gobj/get jsdiff "diffLines") s1 s2)
+  (-> ((gobj/get jsdiff "diffLines") s1 s2 (clj->js {"newlineIsToken" true}))
       bean/->clj))
 
 (def inline-special-chars

src/main/frontend/fs/sync.cljs

@@ -18,6 +18,9 @@
             [frontend.util.persist-var :as persist-var]
             [frontend.handler.notification :as notification]
             [frontend.context.i18n :refer [t]]
+            [frontend.diff :as diff]
+            [frontend.db :as db]
+            [frontend.fs :as fs]
             [medley.core :refer [dedupe-by]]
             [rum.core :as rum]))
 
@@ -162,7 +165,7 @@
                                          (offer! remote-changes-chan data)))))))
 
 (defn ws-listen!
-  "return channal which output messages from server"
+  "return channel which output messages from server"
   [graph-uuid *ws]
   (let [remote-changes-chan (chan (async/sliding-buffer 1))]
     (ws-listen!* graph-uuid *ws remote-changes-chan)
@@ -721,23 +724,67 @@
 
 (def remoteapi (->RemoteAPI))
 
+(defn- add-new-version-file
+  [repo path content]
+  (go
+    (println "add-new-version-file: "
+             (<! (p->c (ipc/ipc "addVersionFile" (config/get-local-dir repo) path content))))))
+
+(defn- is-journals-or-pages?
+  [filetxn]
+  (let [rel-path (relative-path filetxn)]
+    (or (string/starts-with? rel-path "journals/")
+        (string/starts-with? rel-path "pages/"))))
+
+(defn- need-add-version-file?
+  "when we need to create a new version file:
+  1. when apply a 'update' filetxn, it already exists(same page name) locally and has delete diffs
+  2. when apply a 'delete' filetxn, its origin remote content and local content are different
+     - TODO: we need to store origin remote content md5 in server db
+  3. create version files only for files under 'journals/', 'pages/' dir"
+  [^FileTxn filetxn origin-db-content]
+  (go
+    (cond
+      (.renamed? filetxn)
+      false
+      (.-deleted? filetxn)
+      false
+      (.-updated? filetxn)
+      (let [path (relative-path filetxn)
+            repo (state/get-current-repo)
+            file-path (config/get-file-path repo path)
+            content (<! (p->c (fs/read-file "" file-path)))]
+        (or (nil? content)
+            (some :removed (diff/diff origin-db-content content)))))))
+
 (defn- apply-filetxns
   [graph-uuid base-path filetxns]
-  (cond
-    (.renamed? (first filetxns))
-    (let [filetxn (first filetxns)]
-      (assert (= 1 (count filetxns)))
-      (rename-local-file rsapi graph-uuid base-path
-                         (relative-path (.-from-path filetxn))
-                         (relative-path (.-to-path filetxn))))
-
-    (.-updated? (first filetxns))
-    (update-local-files rsapi graph-uuid base-path (map relative-path filetxns))
-
-    (.-deleted? (first filetxns))
-    (let [filetxn (first filetxns)]
-      (assert (= 1 (count filetxns)))
-      (go
+  (go
+    (cond
+      (.renamed? (first filetxns))
+      (let [^FileTxn filetxn (first filetxns)
+            from-path (.-from-path filetxn)
+            to-path (.-to-path filetxn)]
+        (assert (= 1 (count filetxns)))
+        (<! (rename-local-file rsapi graph-uuid base-path
+                               (relative-path from-path)
+                               (relative-path to-path))))
+
+      (.-updated? (first filetxns))
+      (let [repo (state/get-current-repo)
+            txn->db-content-vec (->> filetxns
+                                     (mapv
+                                      #(when (is-journals-or-pages? %)
+                                         [% (db/get-file repo (config/get-file-path repo (relative-path %)))]))
+                                     (remove nil?))]
+        (<! (update-local-files rsapi graph-uuid base-path (map relative-path filetxns)))
+        (doseq [[filetxn origin-db-content] txn->db-content-vec]
+          (when (need-add-version-file? filetxn origin-db-content)
+            (add-new-version-file repo (relative-path filetxn) origin-db-content))))
+
+      (.-deleted? (first filetxns))
+      (let [filetxn (first filetxns)]
+        (assert (= 1 (count filetxns)))
         (let [r (<! (delete-local-files rsapi graph-uuid base-path [(relative-path filetxn)]))]
           (if (and (instance? ExceptionInfo r)
                    (string/index-of (str (ex-cause r)) "No such file or directory"))
@@ -928,7 +975,7 @@
   if local-txid != remote-txid, return {:need-sync-remote true}"))
 
 (defrecord Remote->LocalSyncer [user-uuid graph-uuid base-path repo *txid *sync-state
-                              ^:mutable local->remote-syncer *stopped]
+                                ^:mutable local->remote-syncer *stopped]
   Object
   (set-local->remote-syncer! [_ s] (set! local->remote-syncer s))
   (sync-files-remote->local!

src/main/frontend/handler/editor.cljs

@@ -1710,7 +1710,8 @@
   (let [blocks (get-selected-ordered-blocks)]
     (when (seq blocks)
       (outliner-tx/transact!
-        {:outliner-op :move-blocks}
+        {:outliner-op :move-blocks
+         :real-outliner-op :indent-outdent}
         (outliner-core/indent-outdent-blocks! blocks (= direction :right))))))
 
 (defn- get-link [format link label]
@@ -2035,7 +2036,8 @@
   (when-not (parent-is-page? node)
     (let [parent-node (tree/-get-parent node)]
       (outliner-tx/transact!
-        {:outliner-op :move-blocks}
+        {:outliner-op :move-blocks
+         :real-outliner-op :indent-outdent}
         (save-current-block!)
         (outliner-core/move-blocks! [(:data node)] (:data parent-node) true)))))
 
@@ -2592,7 +2594,8 @@
     (when block
       (state/set-editor-last-pos! pos)
       (outliner-tx/transact!
-        {:outliner-op :move-blocks}
+        {:outliner-op :move-blocks
+         :real-outliner-op :indent-outdent}
         (save-current-block!)
         (outliner-core/indent-outdent-blocks! [block] indent?)))
     (state/set-editor-op! :nil)))

src/main/frontend/handler/file_sync.cljs

@@ -1,7 +1,9 @@
 (ns frontend.handler.file-sync
   (:require ["path" :as path]
             [cljs-time.coerce :as tc]
+            [cljs-time.format :as tf]
             [cljs.core.async :as async :refer [go <!]]
+            [cljs.core.async.interop :refer [p->c]]
             [clojure.string :as string]
             [frontend.config :as config]
             [frontend.db :as db]
@@ -9,7 +11,8 @@
             [frontend.handler.notification :as notification]
             [frontend.state :as state]
             [frontend.util :as util]
-            [frontend.handler.user :as user]))
+            [frontend.handler.user :as user]
+            [frontend.fs :as fs]))
 
 (def hiding-login&file-sync (not config/dev?))
 (def refresh-file-sync-component (atom false))
@@ -65,31 +68,78 @@
         (notification/show! (ex-cause r) :error)
         (notification/show! [:div
                              [:div "Downloaded version file at: "]
-                             [:div key]] :success false)))))
+                             [:div key]] :success false))
+      (when-not (instance? ExceptionInfo r)
+        key))))
+
+(defn- list-file-local-versions
+  [page]
+  (go
+    (when-let [path (-> page :block/file :file/path)]
+      (let [base-path           (config/get-repo-dir (state/get-current-repo))
+            rel-path            (string/replace-first path base-path "")
+            version-files-dir   (->> (path/join "version-files/local" rel-path)
+                                     path/parse
+                                     (#(js->clj % :keywordize-keys true))
+                                     ((juxt :dir :name))
+                                     (apply path/join base-path))
+            version-file-paths* (<! (p->c (fs/readdir version-files-dir)))]
+        (when-not (instance? ExceptionInfo version-file-paths*)
+          (let [version-file-paths
+                (filterv
+                 ;; filter dir
+                 (fn [dir-or-file]
+                   (-> (path/parse dir-or-file)
+                       (js->clj :keywordize-keys true)
+                       :ext
+                       seq))
+                 (js->clj (<! (p->c (fs/readdir version-files-dir)))))]
+            (mapv
+             (fn [path]
+               (let [create-time
+                     (-> (path/parse path)
+                         (js->clj :keywordize-keys true)
+                         :name
+                         (#(tf/parse (tf/formatter "yyyy-MM-dd'T'HH_mm_ss.SSSZZ") %)))]
+                 {:create-time create-time :path path :relative-path (string/replace-first path base-path "")}))
+             version-file-paths)))))))
 
 (defn list-file-versions [graph-uuid page]
   (let [file-id (:db/id (:block/file page))]
     (when-let [path (:file/path (db/entity file-id))]
       (let [base-path (config/get-repo-dir (state/get-current-repo))
-            path* (string/replace-first path base-path "")]
+            path*     (string/replace-first path base-path "")]
         (go
-          (let [version-list (:VersionList
-                              (<! (sync/get-remote-file-versions sync/remoteapi graph-uuid path*)))]
+          (let [version-list       (:VersionList
+                                    (<! (sync/get-remote-file-versions sync/remoteapi graph-uuid path*)))
+                local-version-list (<! (list-file-local-versions page))
+                all-version-list   (->> (concat version-list local-version-list)
+                                        (sort-by #(or (tc/from-string (:CreateTime %))
+                                                      (:create-time %))
+                                                 >))]
             (notification/show! [:div
                                  [:div.font-bold "File history - " path*]
                                  [:hr.my-2]
-                                 (for [version version-list]
-                                   (let [version-uuid (:VersionUUID version)]
+                                 (for [version all-version-list]
+                                   (let [version-uuid (or (:VersionUUID version) (:relative-path version))
+                                         local?       (some? (:relative-path version))]
                                      [:div.my-4 {:key version-uuid}
                                       [:div
                                        [:a.text-xs.inline
-                                        {:on-click #(download-version-file graph-uuid
-                                                                           (:FileUUID version)
-                                                                           (:VersionUUID version))}
+                                        {:on-click #(if local?
+                                                      (js/window.apis.openPath (:path version))
+                                                      (go
+                                                        (let [relative-path
+                                                              (<! (download-version-file graph-uuid
+                                                                                         (:FileUUID version)
+                                                                                         (:VersionUUID version)))]
+                                                          (js/window.apis.openPath (path/join base-path relative-path)))))}
                                         version-uuid]
-                                       [:div.opacity-70 (str "Size: " (:Size version))]]
+                                       (when-not local?
+                                         [:div.opacity-70 (str "Size: " (:Size version))])]
                                       [:div.opacity-50
-                                       (util/time-ago (tc/from-string (:CreateTime version)))]]))]
+                                       (util/time-ago (or (tc/from-string (:CreateTime version))
+                                                          (:create-time version)))]]))]
                                 :success false)))))))
 
 (defn get-current-graph-uuid [] (second @sync/graphs-txid))

src/main/frontend/ui.cljs

@@ -879,7 +879,7 @@
   {:init (fn [state]
            (assoc state
                   ::ref (atom nil)
-                  ::height (atom 26)))
+                  ::height (atom 24)))
    :did-mount (fn [state]
                 (when (last (:rum/args state))
                   (let [observer (js/ResizeObserver. (fn [entries]
@@ -894,7 +894,6 @@
   [state visible? content-fn _reset-height?]
   [:div.lazy-visibility {:ref #(reset! (::ref state) %)
                          :style {:min-height @(::height state)}}
-
    (if visible?
      (when (fn? content-fn) (content-fn))
      [:div.shadow.rounded-md.p-4.w-full.mx-auto {:style {:height 64}}
@@ -909,17 +908,23 @@
 
 (rum/defcs lazy-visible <
   (rum/local false ::visible?)
-  [state content-fn sensor-opts reset-height?]
-  (if (or (util/mobile?) (mobile-util/native-platform?))
-    (content-fn)
-    (let [*visible? (::visible? state)]
-      (visibility-sensor
-       (merge
-        {:on-change #(reset! *visible? %)
-         :partialVisibility true
-         :offset {:top -300
-                  :bottom -300}
-         :scrollCheck true
-         :scrollThrottle 1}
-        sensor-opts)
-       (lazy-visible-inner @*visible? content-fn reset-height?)))))
+  (rum/local true ::active?)
+  [state content-fn sensor-opts {:keys [reset-height? once?]}]
+  (let [*active? (::active? state)]
+    (if (or (util/mobile?) (mobile-util/is-native-platform?))
+      (content-fn)
+      (let [*visible? (::visible? state)]
+        (visibility-sensor
+         (merge
+          {:on-change (fn [v]
+                        (reset! *visible? v)
+                        (when (and once? v)
+                          (reset! *active? false)))
+           :partialVisibility true
+           :offset {:top -300
+                    :bottom -300}
+           :scrollCheck true
+           :scrollThrottle 500
+           :active @*active?}
+          sensor-opts)
+         (lazy-visible-inner @*visible? content-fn reset-height?))))))