repo.cljs 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516
  1. (ns frontend.handler.repo
  2. (:refer-clojure :exclude [clone])
  3. (:require [clojure.string :as string]
  4. [frontend.config :as config]
  5. [frontend.context.i18n :refer [t]]
  6. [frontend.date :as date]
  7. [frontend.db :as db]
  8. [frontend.fs :as fs]
  9. [frontend.fs.nfs :as nfs]
  10. [frontend.handler.common :as common-handler]
  11. [frontend.handler.file :as file-handler]
  12. [frontend.handler.route :as route-handler]
  13. [frontend.handler.ui :as ui-handler]
  14. [frontend.handler.metadata :as metadata-handler]
  15. [frontend.idb :as idb]
  16. [frontend.search :as search]
  17. [frontend.spec :as spec]
  18. [frontend.state :as state]
  19. [frontend.util :as util]
  20. [frontend.util.fs :as util-fs]
  21. [lambdaisland.glogi :as log]
  22. [promesa.core :as p]
  23. [shadow.resource :as rc]
  24. [frontend.db.persist :as db-persist]
  25. [logseq.graph-parser.util :as gp-util]
  26. [logseq.graph-parser :as graph-parser]
  27. [electron.ipc :as ipc]
  28. [cljs-bean.core :as bean]
  29. [clojure.core.async :as async]
  30. [frontend.encrypt :as encrypt]
  31. [frontend.mobile.util :as mobile-util]))
  32. ;; Project settings should be checked in two situations:
  33. ;; 1. User changes the config.edn directly in logseq.com (fn: alter-file)
  34. ;; 2. Git pulls the new change (fn: load-files)
  35. (defn create-config-file-if-not-exists
  36. [repo-url]
  37. (spec/validate :repos/url repo-url)
  38. (let [repo-dir (config/get-repo-dir repo-url)
  39. app-dir config/app-name
  40. dir (str repo-dir "/" app-dir)]
  41. (p/let [_ (fs/mkdir-if-not-exists dir)]
  42. (let [default-content config/config-default-content
  43. path (str app-dir "/" config/config-file)]
  44. (p/let [file-exists? (fs/create-if-not-exists repo-url repo-dir (str app-dir "/" config/config-file) default-content)]
  45. (when-not file-exists?
  46. (file-handler/reset-file! repo-url path default-content)
  47. (common-handler/reset-config! repo-url default-content)))))))
  48. (defn create-contents-file
  49. [repo-url]
  50. (spec/validate :repos/url repo-url)
  51. (p/let [repo-dir (config/get-repo-dir repo-url)
  52. pages-dir (state/get-pages-directory)
  53. [org-path md-path] (map #(str "/" pages-dir "/contents." %) ["org" "md"])
  54. contents-file-exist? (some #(fs/file-exists? repo-dir %) [org-path md-path])]
  55. (when-not contents-file-exist?
  56. (let [format (state/get-preferred-format)
  57. path (str pages-dir "/contents."
  58. (config/get-file-extension format))
  59. file-path (str "/" path)
  60. default-content (case (name format)
  61. "org" (rc/inline "contents.org")
  62. "markdown" (rc/inline "contents.md")
  63. "")]
  64. (p/let [_ (fs/mkdir-if-not-exists (util/safe-path-join repo-dir pages-dir))
  65. file-exists? (fs/create-if-not-exists repo-url repo-dir file-path default-content)]
  66. (when-not file-exists?
  67. (file-handler/reset-file! repo-url path default-content)))))))
  68. (defn create-custom-theme
  69. [repo-url]
  70. (spec/validate :repos/url repo-url)
  71. (let [repo-dir (config/get-repo-dir repo-url)
  72. path (str config/app-name "/" config/custom-css-file)
  73. file-path (str "/" path)
  74. default-content ""]
  75. (p/let [_ (fs/mkdir-if-not-exists (util/safe-path-join repo-dir config/app-name))
  76. file-exists? (fs/create-if-not-exists repo-url repo-dir file-path default-content)]
  77. (when-not file-exists?
  78. (file-handler/reset-file! repo-url path default-content)))))
  79. (defn create-dummy-notes-page
  80. [repo-url content]
  81. (spec/validate :repos/url repo-url)
  82. (let [repo-dir (config/get-repo-dir repo-url)
  83. path (str (config/get-pages-directory) "/how_to_make_dummy_notes.md")
  84. file-path (str "/" path)]
  85. (p/let [_ (fs/mkdir-if-not-exists (util/safe-path-join repo-dir (config/get-pages-directory)))
  86. _file-exists? (fs/create-if-not-exists repo-url repo-dir file-path content)]
  87. (file-handler/reset-file! repo-url path content))))
  88. (defn- create-today-journal-if-not-exists
  89. [repo-url {:keys [content]}]
  90. (spec/validate :repos/url repo-url)
  91. (when (state/enable-journals? repo-url)
  92. (let [repo-dir (config/get-repo-dir repo-url)
  93. format (state/get-preferred-format repo-url)
  94. title (date/today)
  95. file-name (date/journal-title->default title)
  96. default-content (util/default-content-with-title format)
  97. template (state/get-default-journal-template)
  98. template (when (and template
  99. (not (string/blank? template)))
  100. template)
  101. content (cond
  102. content
  103. content
  104. template
  105. (str default-content template)
  106. :else
  107. default-content)
  108. path (util/safe-path-join (config/get-journals-directory) (str file-name "."
  109. (config/get-file-extension format)))
  110. file-path (str "/" path)
  111. page-exists? (db/entity repo-url [:block/name (util/page-name-sanity-lc title)])
  112. empty-blocks? (db/page-empty? repo-url (util/page-name-sanity-lc title))]
  113. (when (or empty-blocks? (not page-exists?))
  114. (p/let [_ (nfs/check-directory-permission! repo-url)
  115. _ (fs/mkdir-if-not-exists (util/safe-path-join repo-dir (config/get-journals-directory)))
  116. file-exists? (fs/file-exists? repo-dir file-path)]
  117. (when-not file-exists?
  118. (p/let [_ (file-handler/reset-file! repo-url path content)]
  119. (p/let [_ (fs/create-if-not-exists repo-url repo-dir file-path content)]
  120. (when-not (state/editing?)
  121. (ui-handler/re-render-root!)))))
  122. (when-not (state/editing?)
  123. (ui-handler/re-render-root!)))))))
  124. (defn create-default-files!
  125. ([repo-url]
  126. (create-default-files! repo-url false))
  127. ([repo-url encrypted?]
  128. (spec/validate :repos/url repo-url)
  129. (let [repo-dir (config/get-repo-dir repo-url)]
  130. (p/let [_ (fs/mkdir-if-not-exists (util/safe-path-join repo-dir config/app-name))
  131. _ (fs/mkdir-if-not-exists (util/safe-path-join repo-dir (str config/app-name "/" config/recycle-dir)))
  132. _ (fs/mkdir-if-not-exists (util/safe-path-join repo-dir (config/get-journals-directory)))
  133. _ (file-handler/create-metadata-file repo-url encrypted?)
  134. _ (create-config-file-if-not-exists repo-url)
  135. _ (create-contents-file repo-url)
  136. _ (create-custom-theme repo-url)]
  137. (state/pub-event! [:page/create-today-journal repo-url])))))
  138. (defn- load-pages-metadata!
  139. "force?: if set true, skip the metadata timestamp range check"
  140. ([repo file-paths files]
  141. (load-pages-metadata! repo file-paths files false))
  142. ([repo file-paths files force?]
  143. (try
  144. (let [file (config/get-pages-metadata-path)]
  145. (when (contains? (set file-paths) file)
  146. (when-let [content (some #(when (= (:file/path %) file) (:file/content %)) files)]
  147. (let [metadata (common-handler/safe-read-string content "Parsing pages metadata file failed: ")
  148. pages (db/get-all-pages repo)
  149. pages (zipmap (map :block/name pages) pages)
  150. metadata (->>
  151. (filter (fn [{:block/keys [name created-at updated-at]}]
  152. (when-let [page (get pages name)]
  153. (and
  154. (>= updated-at created-at) ;; metadata validation
  155. (or force? ;; when force is true, shortcut timestamp range check
  156. (and (or (nil? (:block/created-at page))
  157. (>= created-at (:block/created-at page)))
  158. (or (nil? (:block/updated-at page))
  159. (>= updated-at (:block/created-at page)))))
  160. (or ;; persistent metadata is the gold standard
  161. (not= created-at (:block/created-at page))
  162. (not= updated-at (:block/created-at page)))))) metadata)
  163. (remove nil?))]
  164. (when (seq metadata)
  165. (db/transact! repo metadata {:new-graph? true}))))))
  166. (catch js/Error e
  167. (log/error :exception e)))))
  168. (defn update-pages-metadata!
  169. "update pages meta content -> db. Only accept non-encrypted content!"
  170. [repo content force?]
  171. (let [path (config/get-pages-metadata-path)
  172. files [{:file/path path
  173. :file/content content}]
  174. file-paths [path]]
  175. (load-pages-metadata! repo file-paths files force?)))
  176. (defn- parse-and-load-file!
  177. [repo-url file {:keys [new-graph? verbose]}]
  178. (try
  179. (file-handler/alter-file repo-url
  180. (:file/path file)
  181. (:file/content file)
  182. (merge {:new-graph? new-graph?
  183. :re-render-root? false
  184. :from-disk? true}
  185. (when (some? verbose) {:verbose verbose})))
  186. (catch :default e
  187. (state/set-parsing-state! (fn [m]
  188. (update m :failed-parsing-files conj [(:file/path file) e])))))
  189. (state/set-parsing-state! (fn [m]
  190. (update m :finished inc))))
  191. (defn- after-parse
  192. [repo-url files file-paths db-encrypted? re-render? re-render-opts opts graph-added-chan]
  193. (load-pages-metadata! repo-url file-paths files true)
  194. (when (or (:new-graph? opts) (not (:refresh? opts)))
  195. (if (and (not db-encrypted?) (state/enable-encryption? repo-url))
  196. (state/pub-event! [:modal/encryption-setup-dialog repo-url
  197. #(create-default-files! repo-url %)])
  198. (create-default-files! repo-url db-encrypted?)))
  199. (when re-render?
  200. (ui-handler/re-render-root! re-render-opts))
  201. (state/pub-event! [:graph/added repo-url opts])
  202. (state/reset-parsing-state!)
  203. (state/set-loading-files! repo-url false)
  204. (async/offer! graph-added-chan true))
  205. (defn- parse-files-and-create-default-files-inner!
  206. [repo-url files delete-files delete-blocks file-paths db-encrypted? re-render? re-render-opts opts]
  207. (let [supported-files (graph-parser/filter-files files)
  208. delete-data (->> (concat delete-files delete-blocks)
  209. (remove nil?))
  210. chan (async/to-chan! supported-files)
  211. graph-added-chan (async/promise-chan)]
  212. (when (seq delete-data) (db/transact! repo-url delete-data))
  213. (state/set-current-repo! repo-url)
  214. (state/set-parsing-state! {:total (count supported-files)})
  215. ;; Synchronous for tests for not breaking anything
  216. (if util/node-test?
  217. (do
  218. (doseq [file supported-files]
  219. (state/set-parsing-state! (fn [m]
  220. (assoc m :current-parsing-file (:file/path file))))
  221. (parse-and-load-file! repo-url file (select-keys opts [:new-graph? :verbose])))
  222. (after-parse repo-url files file-paths db-encrypted? re-render? re-render-opts opts graph-added-chan))
  223. (async/go-loop []
  224. (if-let [file (async/<! chan)]
  225. (do
  226. (state/set-parsing-state! (fn [m]
  227. (assoc m :current-parsing-file (:file/path file))))
  228. (async/<! (async/timeout 10))
  229. (parse-and-load-file! repo-url file (select-keys opts [:new-graph? :verbose]))
  230. (recur))
  231. (after-parse repo-url files file-paths db-encrypted? re-render? re-render-opts opts graph-added-chan))))
  232. graph-added-chan))
  233. (defn- parse-files-and-create-default-files!
  234. [repo-url files delete-files delete-blocks file-paths db-encrypted? re-render? re-render-opts opts]
  235. (if db-encrypted?
  236. (p/let [files (p/all
  237. (map (fn [file]
  238. (p/let [content (encrypt/decrypt (:file/content file))]
  239. (assoc file :file/content content)))
  240. files))]
  241. (parse-files-and-create-default-files-inner! repo-url files delete-files delete-blocks file-paths db-encrypted? re-render? re-render-opts opts))
  242. (parse-files-and-create-default-files-inner! repo-url files delete-files delete-blocks file-paths db-encrypted? re-render? re-render-opts opts)))
  243. (defn parse-files-and-load-to-db!
  244. [repo-url files {:keys [delete-files delete-blocks re-render? re-render-opts _refresh?] :as opts
  245. :or {re-render? true}}]
  246. (let [file-paths (map :file/path files)
  247. metadata-file (config/get-metadata-path)
  248. metadata-content (some #(when (= (:file/path %) metadata-file)
  249. (:file/content %)) files)
  250. metadata (when metadata-content
  251. (common-handler/read-metadata! metadata-content))
  252. db-encrypted? (:db/encrypted? metadata)
  253. db-encrypted-secret (if db-encrypted? (:db/encrypted-secret metadata) nil)]
  254. (if db-encrypted?
  255. (let [close-fn #(parse-files-and-create-default-files! repo-url files delete-files delete-blocks file-paths db-encrypted? re-render? re-render-opts opts)]
  256. (state/set-state! :encryption/graph-parsing? true)
  257. (state/pub-event! [:modal/encryption-input-secret-dialog repo-url
  258. db-encrypted-secret
  259. close-fn]))
  260. (parse-files-and-create-default-files! repo-url files delete-files delete-blocks file-paths db-encrypted? re-render? re-render-opts opts))))
  261. (defn load-repo-to-db!
  262. [repo-url {:keys [diffs nfs-files refresh? new-graph? empty-graph?]}]
  263. (spec/validate :repos/url repo-url)
  264. (route-handler/redirect-to-home!)
  265. (state/set-parsing-state! {:graph-loading? true})
  266. (let [config (or (when-let [content (some-> (first (filter #(= (config/get-config-path repo-url) (:file/path %)) nfs-files))
  267. :file/content)]
  268. (common-handler/read-config content))
  269. (state/get-config repo-url))
  270. ;; NOTE: Use config while parsing. Make sure it's the corrent journal title format
  271. _ (state/set-config! repo-url config)
  272. relate-path-fn (fn [m k]
  273. (some-> (get m k)
  274. (string/replace (js/decodeURI (config/get-local-dir repo-url)) "")))
  275. nfs-files (common-handler/remove-hidden-files nfs-files config #(relate-path-fn % :file/path))
  276. diffs (common-handler/remove-hidden-files diffs config #(relate-path-fn % :path))
  277. load-contents (fn [files option]
  278. (file-handler/load-files-contents!
  279. repo-url
  280. files
  281. (fn [files-contents]
  282. (parse-files-and-load-to-db! repo-url files-contents (assoc option :refresh? refresh?)))))]
  283. (cond
  284. (and (not (seq diffs)) nfs-files)
  285. (parse-files-and-load-to-db! repo-url nfs-files {:new-graph? new-graph?
  286. :empty-graph? empty-graph?})
  287. :else
  288. (when (seq diffs)
  289. (let [filter-diffs (fn [type] (->> (filter (fn [f] (= type (:type f))) diffs)
  290. (map :path)))
  291. remove-files (filter-diffs "remove")
  292. modify-files (filter-diffs "modify")
  293. add-files (filter-diffs "add")
  294. delete-files (when (seq remove-files)
  295. (db/delete-files remove-files))
  296. delete-blocks (db/delete-blocks repo-url remove-files true)
  297. delete-blocks (->>
  298. (concat
  299. delete-blocks
  300. (db/delete-blocks repo-url modify-files false))
  301. (remove nil?))
  302. delete-pages (if (seq remove-files)
  303. (db/delete-pages-by-files remove-files)
  304. [])
  305. add-or-modify-files (some->>
  306. (concat modify-files add-files)
  307. (gp-util/remove-nils))
  308. options {:delete-files (concat delete-files delete-pages)
  309. :delete-blocks delete-blocks
  310. :re-render? true}]
  311. (if (seq nfs-files)
  312. (parse-files-and-load-to-db! repo-url nfs-files
  313. (assoc options
  314. :refresh? refresh?
  315. :re-render-opts {:clear-all-query-state? true}))
  316. (load-contents add-or-modify-files options)))))))
  317. (defn remove-repo!
  318. [{:keys [url] :as repo}]
  319. (let [delete-db-f (fn []
  320. (let [graph-exists? (db/get-db url)]
  321. (db/remove-conn! url)
  322. (db-persist/delete-graph! url)
  323. (search/remove-db! url)
  324. (state/delete-repo! repo)
  325. (when graph-exists? (ipc/ipc "graphUnlinked" repo))
  326. (when (= (state/get-current-repo) url)
  327. (state/set-current-repo! (:url (first (state/get-repos)))))))]
  328. (when (or (config/local-db? url) (= url "local"))
  329. (p/let [_ (idb/clear-local-db! url)] ; clear file handles
  330. (delete-db-f)))))
  331. (defn start-repo-db-if-not-exists!
  332. [repo]
  333. (state/set-current-repo! repo)
  334. (db/start-db-conn! repo))
  335. (defn- setup-local-repo-if-not-exists-impl!
  336. []
  337. ;; loop query if js/window.pfs is ready, interval 100ms
  338. (if js/window.pfs
  339. (let [repo config/local-repo]
  340. (p/do! (fs/mkdir-if-not-exists (str "/" repo))
  341. (state/set-current-repo! repo)
  342. (db/start-db-conn! repo)
  343. (when-not config/publishing?
  344. (let [dummy-notes (t :tutorial/dummy-notes)]
  345. (create-dummy-notes-page repo dummy-notes)))
  346. (when-not config/publishing?
  347. (let [tutorial (t :tutorial/text)
  348. tutorial (string/replace-first tutorial "$today" (date/today))]
  349. (create-today-journal-if-not-exists repo {:content tutorial})))
  350. (create-config-file-if-not-exists repo)
  351. (create-contents-file repo)
  352. (create-custom-theme repo)
  353. (state/set-db-restoring! false)
  354. (ui-handler/re-render-root!)))
  355. (p/then (p/delay 100) ;; TODO Junyi remove the string
  356. setup-local-repo-if-not-exists-impl!)))
  357. (defn setup-local-repo-if-not-exists!
  358. []
  359. ;; ensure `(state/set-db-restoring! false)` at exit
  360. (-> (setup-local-repo-if-not-exists-impl!)
  361. (p/timeout 3000)
  362. (p/catch (fn []
  363. (state/set-db-restoring! false)
  364. (prn "setup-local-repo failed! timeout 3000ms")))))
  365. (defn restore-and-setup-repo!
  366. "Restore the db of a graph from the persisted data, and setup. Create a new
  367. conn, or replace the conn in state with a new one."
  368. [repo]
  369. (p/let [_ (state/set-db-restoring! true)
  370. _ (db/restore-graph! repo)]
  371. (file-handler/restore-config! repo)
  372. ;; Don't have to unlisten the old listerner, as it will be destroyed with the conn
  373. (db/listen-and-persist! repo)
  374. (ui-handler/add-style-if-exists!)
  375. (state/set-db-restoring! false)))
  376. (defn rebuild-index!
  377. [url]
  378. (when-not (state/unlinked-dir? (config/get-repo-dir url))
  379. (when url
  380. (search/reset-indice! url)
  381. (db/remove-conn! url)
  382. (db/clear-query-state!)
  383. (-> (p/do! (db-persist/delete-graph! url))
  384. (p/catch (fn [error]
  385. (prn "Delete repo failed, error: " error)))))))
  386. (defn re-index!
  387. [nfs-rebuild-index! ok-handler]
  388. (when-let [repo (state/get-current-repo)]
  389. (let [dir (config/get-repo-dir repo)]
  390. (when-not (state/unlinked-dir? dir)
  391. (route-handler/redirect-to-home!)
  392. (let [local? (config/local-db? repo)]
  393. (if local?
  394. (p/let [_ (metadata-handler/set-pages-metadata! repo)]
  395. (nfs-rebuild-index! repo ok-handler))
  396. (rebuild-index! repo))
  397. (js/setTimeout
  398. (route-handler/redirect-to-home!)
  399. 500))))))
  400. (defn persist-db!
  401. ([]
  402. (persist-db! {}))
  403. ([handlers]
  404. (persist-db! (state/get-current-repo) handlers))
  405. ([repo {:keys [before on-success on-error]}]
  406. (->
  407. (p/do!
  408. (when before
  409. (before))
  410. (metadata-handler/set-pages-metadata! repo)
  411. (db/persist! repo)
  412. (when on-success
  413. (on-success)))
  414. (p/catch (fn [error]
  415. (js/console.error error)
  416. (when on-error
  417. (on-error)))))))
  418. (defn broadcast-persist-db!
  419. "Only works for electron
  420. Call backend to handle persisting a specific db on other window
  421. Skip persisting if no other windows is open (controlled by electron)
  422. step 1. [In HERE] a window ---broadcastPersistGraph----> electron
  423. step 2. electron ---------persistGraph-------> window holds the graph
  424. step 3. window w/ graph --broadcastPersistGraphDone-> electron
  425. step 4. [In HERE] a window <---broadcastPersistGraph---- electron"
  426. [graph]
  427. (p/let [_ (ipc/ipc "broadcastPersistGraph" graph)] ;; invoke for chaining promise
  428. nil))
  429. (defn get-repos
  430. []
  431. (p/let [nfs-dbs (db-persist/get-all-graphs)
  432. nfs-dbs (map (fn [db]
  433. {:url db
  434. :root (config/get-local-dir db)
  435. :nfs? true}) nfs-dbs)
  436. nfs-dbs (and (seq nfs-dbs)
  437. (cond (util/electron?)
  438. (ipc/ipc :inflateGraphsInfo nfs-dbs)
  439. (mobile-util/native-platform?)
  440. (util-fs/inflate-graphs-info nfs-dbs)
  441. :else
  442. nil))
  443. nfs-dbs (seq (bean/->clj nfs-dbs))]
  444. (cond
  445. (seq nfs-dbs)
  446. nfs-dbs
  447. :else
  448. [{:url config/local-repo
  449. :example? true}])))
  450. (defn combine-local-&-remote-graphs
  451. [local-repos remote-repos]
  452. (when-let [repos' (seq (concat (map #(if-let [sync-meta (seq (:sync-meta %))]
  453. (assoc % :GraphUUID (second sync-meta)) %)
  454. local-repos)
  455. (some->> remote-repos
  456. (map #(assoc % :remote? true)))))]
  457. (let [repos' (group-by :GraphUUID repos')
  458. repos'' (mapcat (fn [[k vs]]
  459. (if-not (nil? k)
  460. [(merge (first vs) (second vs))] vs))
  461. repos')]
  462. (sort-by (fn [repo]
  463. (let [graph-name (or (:GraphName repo)
  464. (last (string/split (:root repo) #"/")))]
  465. [(:remote? repo) (string/lower-case graph-name)])) repos''))))
  466. (defn get-detail-graph-info
  467. [url]
  468. (when-let [graphs (seq (and url (combine-local-&-remote-graphs
  469. (state/get-repos)
  470. (state/get-remote-repos))))]
  471. (first (filter #(when-let [url' (:url %)]
  472. (= url url')) graphs))))
  473. (defn refresh-repos!
  474. []
  475. (p/let [repos (get-repos)]
  476. (state/set-repos! repos)
  477. repos))
  478. (defn graph-ready!
  479. "Call electron that the graph is loaded."
  480. [graph]
  481. (ipc/ipc "graphReady" graph))