fix: remove redundancy paths on db

This commit also removes the fragile files including page-metadata.edn
and metadata.edn.

src/main/frontend/components/repo.cljs

@@ -4,10 +4,8 @@
             [frontend.config :as config]
             [frontend.context.i18n :refer [t]]
             [frontend.db :as db]
-            [frontend.handler.page :as page-handler]
             [frontend.handler.repo :as repo-handler]
             [frontend.handler.web.nfs :as nfs-handler]
-            [frontend.modules.shortcut.core :as shortcut]
             [frontend.state :as state]
             [frontend.ui :as ui]
             [frontend.util :as util]

src/main/frontend/config.cljs

@@ -300,9 +300,6 @@
 (def custom-css-file "custom.css")
 (def export-css-file "export.css")
 (def custom-js-file "custom.js")
-(def metadata-file "metadata.edn")
-(def pages-metadata-file "pages-metadata.edn")
-
 (def config-default-content (rc/inline "config.edn"))
 
 (defonce idb-db-prefix "logseq-db/")
@@ -421,20 +418,6 @@
    (when repo
      (get-file-path repo (str app-name "/" config-file)))))
 
-(defn get-metadata-path
-  ([]
-   (get-metadata-path (state/get-current-repo)))
-  ([repo]
-   (when repo
-     (get-file-path repo (str app-name "/" metadata-file)))))
-
-(defn get-pages-metadata-path
-  ([]
-   (get-pages-metadata-path (state/get-current-repo)))
-  ([repo]
-   (when repo
-     (get-file-path repo (str app-name "/" pages-metadata-file)))))
-
 (defn get-custom-css-path
   ([]
    (get-custom-css-path (state/get-current-repo)))

src/main/frontend/fs/capacitor_fs.cljs

@@ -1,6 +1,6 @@
 (ns frontend.fs.capacitor-fs
   "Implementation of fs protocol for mobile"
-  (:require ["@capacitor/filesystem" :refer [Encoding Filesystem Directory]]
+  (:require ["@capacitor/filesystem" :refer [Encoding Filesystem]]
             [cljs-bean.core :as bean]
             [clojure.string :as string]
             [goog.string :as gstring]
@@ -227,20 +227,25 @@
                              (js/encodeURI %)
 
                              :else
-                             (js/encodeURI (js/decodeURI %))))]
+                             (js/encodeURI (js/decodeURI %))))
+        path' (cond
+                (and path (string/starts-with? path "file:/"))
+                (safe-encode-url path)
 
-    (cond (string/blank? path)
-          (safe-encode-url dir)
+                (string/blank? path)
+                (safe-encode-url dir)
 
-          (string/blank? dir)
-          (safe-encode-url path)
+                (string/blank? dir)
+                (safe-encode-url path)
 
-          (string/starts-with? path dir)
-          (safe-encode-url path)
+                (string/starts-with? path dir)
+                (safe-encode-url path)
 
-          :else
-          (let [path' (safe-encode-url path)]
-            (str dir "/" path')))))
+                :else
+                (let [path' (safe-encode-url path)]
+                  (str dir "/" path')))
+        path' (string/replace path' "///private/" "///")]
+    path'))
 
 (defn- local-container-path?
   "Check whether `path' is logseq's container `localDocumentsPath' on iOS"

src/main/frontend/fs/watcher_handler.cljs

@@ -7,15 +7,16 @@
             [frontend.handler.editor :as editor]
             [frontend.handler.file :as file-handler]
             [frontend.handler.page :as page-handler]
-            [frontend.handler.repo :as repo-handler]
             [frontend.handler.ui :as ui-handler]
             [logseq.graph-parser.util :as gp-util]
             [logseq.graph-parser.config :as gp-config]
             [logseq.graph-parser.util.block-ref :as block-ref]
+            [frontend.mobile.util :as mobile-util]
             [lambdaisland.glogi :as log]
             [promesa.core :as p]
             [frontend.state :as state]
-            [frontend.fs :as fs]))
+            [frontend.fs :as fs]
+            [frontend.fs.capacitor-fs :as capacitor-fs]))
 
 ;; all IPC paths must be normalized! (via gp-util/path-normalize)
 
@@ -49,10 +50,12 @@
   [type {:keys [dir path content stat global-dir] :as payload}]
   (when dir
     (let [path (gp-util/path-normalize path)
+          path (if (mobile-util/native-platform?)
+                 (capacitor-fs/normalize-file-protocol-path nil path)
+                 path)
           ;; Global directory events don't know their originating repo so we rely
           ;; on the client to correctly identify it
           repo (if global-dir (state/get-current-repo) (config/get-local-repo dir))
-          pages-metadata-path (config/get-pages-metadata-path)
           {:keys [mtime]} stat
           db-content (or (db/get-file repo path) "")]
       (when (or content (contains? #{"unlink" "unlinkDir" "addDir"} type))
@@ -67,8 +70,7 @@
           nil
 
           (and (= "add" type)
-               (not= (string/trim content) (string/trim db-content))
-               (not= path pages-metadata-path))
+               (not= (string/trim content) (string/trim db-content)))
           (let [backup? (not (string/blank? db-content))]
             (handle-add-and-change! repo path content db-content mtime backup?))
 
@@ -78,7 +80,6 @@
 
           (and (= "change" type)
                (not= (string/trim content) (string/trim db-content))
-               (not= path pages-metadata-path)
                (not (gp-config/local-asset? (string/replace-first path dir ""))))
           (when-not (and
                      (string/includes? path (str "/" (config/get-journals-directory) "/"))
@@ -103,19 +104,6 @@
             (println "reloading custom.css")
             (ui-handler/add-style-if-exists!))
 
-          ;; When metadata is added to watcher, update timestamps in db accordingly
-          ;; This event is not triggered on re-index
-          ;; Persistent metadata is gold standard when db is offline, so it's forced
-          (and (contains? #{"add"} type)
-               (= path pages-metadata-path))
-          (p/do! (repo-handler/update-pages-metadata! repo content true))
-
-          ;; Change is triggered by external changes, so update to the db
-          ;; Don't forced update when db is online, but resolving conflicts
-          (and (contains? #{"change"} type)
-               (= path pages-metadata-path))
-          (p/do! (repo-handler/update-pages-metadata! repo content false))
-
           (contains? #{"add" "change" "unlink"} type)
           nil
 

src/main/frontend/handler.cljs

@@ -29,7 +29,6 @@
             [frontend.handler.repo-config :as repo-config-handler]
             [frontend.handler.global-config :as global-config-handler]
             [frontend.handler.plugin-config :as plugin-config-handler]
-            [frontend.handler.metadata :as metadata-handler]
             [frontend.idb :as idb]
             [frontend.mobile.util :as mobile-util]
             [frontend.modules.instrumentation.core :as instrument]
@@ -238,7 +237,6 @@
   (persist-var/load-vars)
   (user-handler/restore-tokens-from-localstorage)
   (user-handler/refresh-tokens-loop)
-  (metadata-handler/run-set-page-metadata-job!)
   (js/setTimeout instrument! (* 60 1000)))
 
 (defn stop! []

src/main/frontend/handler/common/file.cljs

@@ -8,7 +8,8 @@
             [frontend.mobile.util :as mobile-util]
             [logseq.graph-parser :as graph-parser]
             [logseq.graph-parser.util :as gp-util]
-            [logseq.graph-parser.config :as gp-config]))
+            [logseq.graph-parser.config :as gp-config]
+            [frontend.fs.capacitor-fs :as capacitor-fs]))
 
 (defn- page-exists-in-another-file
   "Conflict of files towards same page"
@@ -40,6 +41,7 @@
   ([repo-url file content {:keys [verbose] :as options}]
    (let [electron-local-repo? (and (util/electron?)
                                    (config/local-db? repo-url))
+         repo-dir (config/get-repo-dir repo-url)
          file (cond
                 (and electron-local-repo?
                      util/win32?
@@ -49,13 +51,10 @@
                 (and electron-local-repo? (or
                                            util/win32?
                                            (not= "/" (first file))))
-                (str (config/get-repo-dir repo-url) "/" file)
+                (str repo-dir "/" file)
 
-                (and (mobile-util/native-android?) (not= "/" (first file)))
-                file
-
-                (and (mobile-util/native-ios?) (not= "/" (first file)))
-                file
+                (mobile-util/native-platform?)
+                (capacitor-fs/normalize-file-protocol-path repo-dir file)
 
                 :else
                 file)

src/main/frontend/handler/events.cljs

@@ -767,9 +767,9 @@
                       :native-icloud? (not (string/blank? (state/get-icloud-container-root-url)))
                       :logged?        (user-handler/logged-in?)} opts)]
     (if (mobile-util/native-ios?)
-      (do (state/set-modal!
-           #(graph-picker/graph-picker-cp opts')
-           {:label "graph-setup"}))
+      (state/set-modal!
+       #(graph-picker/graph-picker-cp opts')
+       {:label "graph-setup"})
       (page-handler/ls-dir-files! st/refresh! opts'))))
 
 (defmethod handle :file/alter [[_ repo path content]]

src/main/frontend/handler/file.cljs

@@ -219,25 +219,3 @@
       ;; after an app refresh can cause stale page data to load
       (fs/unwatch-dir! dir)
       (fs/watch-dir! dir))))
-
-(defn create-metadata-file
-  [repo-url]
-  (let [repo-dir (config/get-repo-dir repo-url)
-        path (str config/app-name "/" config/metadata-file)
-        file-path (str "/" path)
-        default-content "{}"]
-    (p/let [_ (fs/mkdir-if-not-exists (util/safe-path-join repo-dir config/app-name))
-            file-exists? (fs/create-if-not-exists repo-url repo-dir file-path default-content)]
-      (when-not file-exists?
-        (file-common-handler/reset-file! repo-url path default-content)))))
-
-(defn create-pages-metadata-file
-  [repo-url]
-  (let [repo-dir (config/get-repo-dir repo-url)
-        path (str config/app-name "/" config/pages-metadata-file)
-        file-path (str "/" path)
-        default-content "{}"]
-    (p/let [_ (fs/mkdir-if-not-exists (util/safe-path-join repo-dir config/app-name))
-            file-exists? (fs/create-if-not-exists repo-url repo-dir file-path default-content)]
-      (when-not file-exists?
-        (file-common-handler/reset-file! repo-url path default-content)))))

src/main/frontend/handler/metadata.cljs

@@ -1,86 +0,0 @@
-(ns frontend.handler.metadata
-  "System-component-like ns that manages writing to pages-metadata.edn"
-  (:require [cljs.reader :as reader]
-            [cljs.pprint]
-            [datascript.db :as ddb]
-            [frontend.config :as config]
-            [frontend.db :as db]
-            [frontend.fs :as fs]
-            [frontend.handler.common :as common-handler]
-            [frontend.handler.file :as file-handler]
-            [frontend.state :as state]
-            [promesa.core :as p]))
-
-(def default-metadata-str "{}")
-
-(defn set-metadata!
-  [k v]
-  (when-let [repo (state/get-current-repo)]
-    (let [path (config/get-metadata-path)
-          file-content (db/get-file path)]
-      (p/let [_ (file-handler/create-metadata-file repo)]
-        (let [metadata-str (or file-content default-metadata-str)
-              metadata (try
-                         (reader/read-string metadata-str)
-                         (catch :default e
-                           (println "Parsing metadata.edn failed: ")
-                           (js/console.dir e)
-                           {}))
-              new-metadata (cond
-                             (= k :block/properties)
-                             (update metadata :block/properties v) ; v should be a function
-                             :else
-                             (let [ks (if (vector? k) k [k])]
-                               (assoc-in metadata ks v)))
-              new-content (pr-str new-metadata)]
-          (file-handler/set-file-content! repo path new-content))))))
-
-(defn set-pages-metadata!
-  [repo]
-  (when-not (state/unlinked-dir? (config/get-repo-dir repo))
-    (let [path (config/get-pages-metadata-path repo)
-          all-pages (->> (db/get-all-pages repo)
-                         (common-handler/fix-pages-timestamps)
-                         (map #(select-keys % [:block/name :block/created-at :block/updated-at]))
-                         (sort-by :block/name)
-                         (vec))]
-      (p/let [_ (-> (file-handler/create-pages-metadata-file repo)
-                    (p/catch (fn [] nil)))]
-        (let [new-content (with-out-str (cljs.pprint/pprint all-pages))]
-          (fs/write-file! repo
-                          (config/get-repo-dir repo)
-                          path
-                          new-content
-                          {}))))))
-
-(defn- handler-properties!
-  [all-properties properties-tx]
-  (reduce
-   (fn [acc datom]
-     (let [v (:v datom)
-           id (or (get v :id)
-                  (get v :title))]
-       (if id
-         (let [added? (ddb/datom-added datom)
-               remove-all-properties? (and (not added?)
-                                           ;; only id
-                                           (= 1 (count v)))]
-           (if remove-all-properties?
-             (dissoc acc id)
-             (assoc acc id v)))
-         acc)))
-   all-properties
-   properties-tx))
-
-(defn update-properties!
-  [properties-tx]
-  (set-metadata! :block/properties #(handler-properties! % properties-tx)))
-
-(defn run-set-page-metadata-job!
-  []
-  (js/setInterval
-   (fn []
-     (when-let [repo (state/get-current-repo)]
-       (when (state/input-idle? repo :diff 3000)
-         (set-pages-metadata! repo))))
-   (* 1000 60 10)))

src/main/frontend/handler/repo.cljs

@@ -14,7 +14,6 @@
             [frontend.handler.common.file :as file-common-handler]
             [frontend.handler.route :as route-handler]
             [frontend.handler.ui :as ui-handler]
-            [frontend.handler.metadata :as metadata-handler]
             [frontend.handler.global-config :as global-config-handler]
             [frontend.idb :as idb]
             [frontend.search :as search]
@@ -22,7 +21,6 @@
             [frontend.state :as state]
             [frontend.util :as util]
             [frontend.util.fs :as util-fs]
-            [lambdaisland.glogi :as log]
             [promesa.core :as p]
             [shadow.resource :as rc]
             [frontend.db.persist :as db-persist]
@@ -128,52 +126,11 @@
     (p/let [_ (fs/mkdir-if-not-exists (util/safe-path-join repo-dir config/app-name))
             _ (fs/mkdir-if-not-exists (util/safe-path-join repo-dir (str config/app-name "/" config/recycle-dir)))
             _ (fs/mkdir-if-not-exists (util/safe-path-join repo-dir (config/get-journals-directory)))
-            _ (file-handler/create-metadata-file repo-url)
             _ (repo-config-handler/create-config-file-if-not-exists repo-url)
             _ (create-contents-file repo-url)
             _ (create-custom-theme repo-url)]
       (state/pub-event! [:page/create-today-journal repo-url]))))
 
-(defn- load-pages-metadata!
-  "force?: if set true, skip the metadata timestamp range check"
-  ([repo file-paths files]
-   (load-pages-metadata! repo file-paths files false))
-  ([repo file-paths files force?]
-   (try
-     (let [file (config/get-pages-metadata-path)]
-       (when (contains? (set file-paths) file)
-         (when-let [content (some #(when (= (:file/path %) file) (:file/content %)) files)]
-           (let [metadata (common-handler/safe-read-string content "Parsing pages metadata file failed: ")
-                 pages (db/get-all-pages repo)
-                 pages (zipmap (map :block/name pages) pages)
-                 metadata (->>
-                           (filter (fn [{:block/keys [name created-at updated-at]}]
-                                     (when-let [page (get pages name)]
-                                       (and
-                                        (>= updated-at created-at) ;; metadata validation
-                                        (or force? ;; when force is true, shortcut timestamp range check
-                                            (and (or (nil? (:block/created-at page))
-                                                     (>= created-at (:block/created-at page)))
-                                                 (or (nil? (:block/updated-at page))
-                                                     (>= updated-at (:block/created-at page)))))
-                                        (or ;; persistent metadata is the gold standard
-                                         (not= created-at (:block/created-at page))
-                                         (not= updated-at (:block/created-at page)))))) metadata)
-                           (remove nil?))]
-             (when (seq metadata)
-               (db/transact! repo metadata {:new-graph? true}))))))
-     (catch :default e
-       (log/error :exception e)))))
-
-(defn update-pages-metadata!
-  "update pages meta content -> db. Only accept non-encrypted content!"
-  [repo content force?]
-  (let [path (config/get-pages-metadata-path)
-        files [{:file/path path
-                :file/content content}]
-        file-paths [path]]
-    (load-pages-metadata! repo file-paths files force?)))
-
 (defonce *file-tx (atom nil))
 
 (defn- parse-and-load-file!
@@ -202,8 +159,7 @@
       nil)))
 
 (defn- after-parse
-  [repo-url files file-paths re-render? re-render-opts opts graph-added-chan]
-  (load-pages-metadata! repo-url file-paths files true)
+  [repo-url re-render? re-render-opts opts graph-added-chan]
   (when (or (:new-graph? opts) (not (:refresh? opts)))
     (create-default-files! repo-url))
   (when re-render?
@@ -217,7 +173,7 @@
   (async/offer! graph-added-chan true))
 
 (defn- parse-files-and-create-default-files-inner!
-  [repo-url files delete-files delete-blocks file-paths re-render? re-render-opts opts]
+  [repo-url files delete-files delete-blocks re-render? re-render-opts opts]
   (let [supported-files (graph-parser/filter-files files)
         delete-data (->> (concat delete-files delete-blocks)
                          (remove nil?))
@@ -239,7 +195,7 @@
           (parse-and-load-file! repo-url file (assoc
                                                (select-keys opts [:new-graph? :verbose])
                                                :skip-db-transact? false)))
-        (after-parse repo-url files file-paths re-render? re-render-opts opts graph-added-chan))
+        (after-parse repo-url re-render? re-render-opts opts graph-added-chan))
       (async/go-loop [tx []]
         (if-let [item (async/<! chan)]
           (let [[idx file] item
@@ -267,18 +223,17 @@
               (recur tx')))
           (do
             (when (seq tx) (db/transact! repo-url tx {:from-disk? true}))
-            (after-parse repo-url files file-paths re-render? re-render-opts opts graph-added-chan)))))
+            (after-parse repo-url re-render? re-render-opts opts graph-added-chan)))))
     graph-added-chan))
 
 (defn- parse-files-and-create-default-files!
-  [repo-url files delete-files delete-blocks file-paths re-render? re-render-opts opts]
-  (parse-files-and-create-default-files-inner! repo-url files delete-files delete-blocks file-paths re-render? re-render-opts opts))
+  [repo-url files delete-files delete-blocks re-render? re-render-opts opts]
+  (parse-files-and-create-default-files-inner! repo-url files delete-files delete-blocks re-render? re-render-opts opts))
 
 (defn parse-files-and-load-to-db!
   [repo-url files {:keys [delete-files delete-blocks re-render? re-render-opts _refresh?] :as opts
                    :or {re-render? true}}]
-  (let [file-paths (map :file/path files)]
-    (parse-files-and-create-default-files! repo-url files delete-files delete-blocks file-paths re-render? re-render-opts opts)))
+  (parse-files-and-create-default-files! repo-url files delete-files delete-blocks re-render? re-render-opts opts))
 
 (defn load-repo-to-db!
   [repo-url {:keys [diffs nfs-files refresh? new-graph? empty-graph?]}]
@@ -427,8 +382,7 @@
        (route-handler/redirect-to-home!)
        (let [local? (config/local-db? repo)]
          (if local?
-           (p/let [_ (metadata-handler/set-pages-metadata! repo)]
-             (nfs-rebuild-index! repo ok-handler))
+           (nfs-rebuild-index! repo ok-handler)
            (rebuild-index! repo))
          (js/setTimeout
           (route-handler/redirect-to-home!)
@@ -444,7 +398,6 @@
     (p/do!
      (when before
        (before))
-     (metadata-handler/set-pages-metadata! repo)
      (db/persist! repo)
      (when on-success
        (on-success)))

src/main/frontend/mobile/graph_picker.cljs

@@ -34,7 +34,7 @@
    :on-click #(when (fn? on-toggle)
                 (on-toggle (not on?)))))
 
-(rum/defc graph-picker-cp
+(rum/defc ^:large-vars/cleanup-todo graph-picker-cp
   [{:keys [onboarding-and-home? logged? native-icloud?] :as opts}]
   (let [can-logseq-sync? (and logged? (state/enable-sync?))
         [step set-step!] (rum/use-state :init)