|
|
@@ -2,6 +2,7 @@
|
|
|
"Save pages to files for file-based graphs"
|
|
|
(:require [clojure.core.async :as async]
|
|
|
[clojure.string :as string]
|
|
|
+ [clojure.set :as set]
|
|
|
[frontend.worker.file.core :as file]
|
|
|
[logseq.outliner.tree :as otree]
|
|
|
[lambdaisland.glogi :as log]
|
|
|
@@ -13,17 +14,20 @@
|
|
|
[malli.core :as m]
|
|
|
[frontend.worker.state :as state]
|
|
|
[goog.object :as gobj]
|
|
|
- [logseq.db.sqlite.util :as sqlite-util]))
|
|
|
+ [logseq.db.sqlite.util :as sqlite-util]
|
|
|
+ [logseq.common.util :as common-util]))
|
|
|
|
|
|
(def *writes file/*writes)
|
|
|
(def dissoc-request! file/dissoc-request!)
|
|
|
+(def conj-page-write! file/conj-page-write!)
|
|
|
|
|
|
(defonce file-writes-chan
|
|
|
(let [coercer (m/coercer [:catn
|
|
|
[:repo :string]
|
|
|
[:page-id :any]
|
|
|
[:outliner-op :any]
|
|
|
- [:epoch :int]])]
|
|
|
+ [:epoch :int]
|
|
|
+ [:request-id :int]])]
|
|
|
(async/chan 10000 (map coercer))))
|
|
|
|
|
|
(def batch-write-interval 1000)
|
|
|
@@ -55,7 +59,7 @@
|
|
|
(dissoc block :db/id :block/page)))
|
|
|
|
|
|
(defn do-write-file!
|
|
|
- [repo conn page-db-id outliner-op context]
|
|
|
+ [repo conn page-db-id outliner-op context request-id]
|
|
|
(let [page-block (d/pull @conn '[*] page-db-id)
|
|
|
page-db-id (:db/id page-block)
|
|
|
whiteboard? (contains? (set (:block/type page-block)) "whiteboard")
|
|
|
@@ -65,7 +69,7 @@
|
|
|
(when (or (>= blocks-count 1) blocks-just-deleted?)
|
|
|
(if (and (or (> blocks-count 500) whiteboard?)
|
|
|
(not (state/tx-idle? repo {:diff 3000})))
|
|
|
- (async/put! file-writes-chan [repo page-db-id outliner-op (tc/to-long (t/now))])
|
|
|
+ (async/put! file-writes-chan [repo page-db-id outliner-op (tc/to-long (t/now)) request-id])
|
|
|
(let [pull-keys (if whiteboard? whiteboard-blocks-pull-keys-with-persisted-ids '[*])
|
|
|
blocks (ldb/get-page-blocks @conn (:block/name page-block) {:pull-keys pull-keys})
|
|
|
blocks (if whiteboard? (map cleanup-whiteboard-block blocks) blocks)]
|
|
|
@@ -76,22 +80,28 @@
|
|
|
(let [tree-or-blocks (if whiteboard? blocks
|
|
|
(otree/blocks->vec-tree repo @conn blocks (:block/name page-block)))]
|
|
|
(if page-block
|
|
|
- (file/save-tree! repo conn page-block tree-or-blocks blocks-just-deleted? context)
|
|
|
+ (file/save-tree! repo conn page-block tree-or-blocks blocks-just-deleted? context request-id)
|
|
|
(js/console.error (str "can't find page id: " page-db-id))))))))))
|
|
|
|
|
|
(defn write-files!
|
|
|
[conn pages context]
|
|
|
(when (seq pages)
|
|
|
- (doseq [[repo page-id outliner-op] (set (map #(take 3 %) pages))] ; remove time to dedupe pages to write
|
|
|
- (try (do-write-file! repo conn page-id outliner-op context)
|
|
|
- (catch :default e
|
|
|
- (worker-util/post-message :notification
|
|
|
- (pr-str
|
|
|
- [[:div
|
|
|
- [:p "Write file failed, please copy the changes to other editors in case of losing data."]
|
|
|
- "Error: " (str (gobj/get e "stack"))]
|
|
|
- :error]))
|
|
|
- (log/error :file/write-file-error {:error e}))))))
|
|
|
+ (let [all-request-ids (set (map last pages))
|
|
|
+ distincted-pages (common-util/distinct-by #(take 3 %) pages)
|
|
|
+ repeated-ids (set/difference all-request-ids (set (map last distincted-pages)))]
|
|
|
+ (doseq [id repeated-ids]
|
|
|
+ (dissoc-request! id))
|
|
|
+
|
|
|
+ (doseq [[repo page-id outliner-op _time request-id] distincted-pages]
|
|
|
+ (try (do-write-file! repo conn page-id outliner-op context request-id)
|
|
|
+ (catch :default e
|
|
|
+ (worker-util/post-message :notification
|
|
|
+ (pr-str
|
|
|
+ [[:div
|
|
|
+ [:p "Write file failed, please copy the changes to other editors in case of losing data."]
|
|
|
+ "Error: " (str (gobj/get e "stack"))]
|
|
|
+ :error]))
|
|
|
+ (log/error :file/write-file-error {:error e})))))))
|
|
|
|
|
|
(defn sync-to-file
|
|
|
[repo page-id tx-meta]
|
|
|
@@ -99,7 +109,8 @@
|
|
|
page-id
|
|
|
(not (:created-from-journal-template? tx-meta))
|
|
|
(not (:delete-files? tx-meta)))
|
|
|
- (async/put! file-writes-chan [repo page-id (:outliner-op tx-meta) (tc/to-long (t/now))])))
|
|
|
+ (let [request-id (conj-page-write! page-id)]
|
|
|
+ (async/put! file-writes-chan [repo page-id (:outliner-op tx-meta) (tc/to-long (t/now)) request-id]))))
|
|
|
|
|
|
(defn <ratelimit-file-writes!
|
|
|
[]
|