Merge branch 'master' into enhance/mobile-ux-2

deps/graph-parser/.carve/ignore

@@ -32,3 +32,5 @@ logseq.graph-parser.property/->block-content
 logseq.graph-parser.property/property-value-from-content
 ;; API
 logseq.graph-parser.whiteboard/page-block->tldr-page
+;; API
+logseq.graph-parser/get-blocks-to-delete

deps/graph-parser/src/logseq/graph_parser.cljs

@@ -6,11 +6,77 @@
             [logseq.graph-parser.util :as gp-util]
             [logseq.graph-parser.date-time-util :as date-time-util]
             [logseq.graph-parser.config :as gp-config]
+            [logseq.db.schema :as db-schema]
             [clojure.string :as string]
             [clojure.set :as set]))
 
+(defn- retract-blocks-tx
+  [blocks retain-uuids]
+  (mapcat (fn [{uuid :block/uuid eid :db/id}]
+            (if (and uuid (contains? retain-uuids uuid))
+              (map (fn [attr] [:db.fn/retractAttribute eid attr]) db-schema/retract-attributes)
+              [[:db.fn/retractEntity eid]]))
+          blocks))
+
+(defn- get-file-page
+  "Copy of db/get-file-page. Too basic to couple to main app"
+  [db file-path]
+  (ffirst
+   (d/q
+    '[:find ?page-name
+      :in $ ?path
+      :where
+      [?file :file/path ?path]
+      [?page :block/file ?file]
+      [?page :block/original-name ?page-name]]
+    db
+    file-path)))
+
+(defn- get-page-blocks-no-cache
+  "Copy of db/get-page-blocks-no-cache. Too basic to couple to main app"
+  [db page {:keys [pull-keys]
+            :or {pull-keys '[*]}}]
+  (let [sanitized-page (gp-util/page-name-sanity-lc page)
+        page-id (:db/id (d/entity db [:block/name sanitized-page]))]
+    (when page-id
+      (let [datoms (d/datoms db :avet :block/page page-id)
+            block-eids (mapv :e datoms)]
+        (d/pull-many db pull-keys block-eids)))))
+
+(defn get-blocks-to-delete
+  "Returns the transactional operations to retract blocks belonging to the
+  given page name and file path. This function is required when a file is being
+  parsed from disk; before saving the parsed, blocks from the previous version
+  of that file need to be retracted.
+
+  The 'Page' parsed from the new file version is passed separately from the
+  file-path, as the page name can be set via properties in the file, and thus
+  can change between versions. If it has changed, existing blocks for both the
+  old and new page name will be retracted.
+
+  Blocks are by default fully cleared via retractEntity. However, a collection
+  of block UUIDs to retain can be passed, and any blocks with matching uuids
+  will instead have their attributes cleared individually via
+  'retractAttribute'. This will preserve block references to the retained
+  UUIDs."
+  [db file-page file-path retain-uuid-blocks]
+  (let [existing-file-page (get-file-page db file-path)
+        pages-to-clear (distinct (filter some? [existing-file-page (:block/name file-page)]))
+        blocks (mapcat (fn [page]
+                         (get-page-blocks-no-cache db page {:pull-keys [:db/id :block/uuid]}))
+                       pages-to-clear)
+        retain-uuids (set (keep :block/uuid retain-uuid-blocks))]
+    (retract-blocks-tx (distinct blocks) retain-uuids)))
+
 (defn parse-file
-  "Parse file and save parsed data to the given db. Main parse fn used by logseq app"
+  "Parse file and save parsed data to the given db. Main parse fn used by logseq app.
+Options available:
+
+* :new? - Boolean which indicates if this file already exists. Default is true.
+* :delete-blocks-fn - Optional fn which is called with the new page, file and existing block uuids
+  which may be referenced elsewhere.
+* :skip-db-transact? - Boolean which skips transacting in order to batch transactions. Default is false
+* :extract-options - Options map to pass to extract/extract"
   [conn file content {:keys [new? delete-blocks-fn extract-options skip-db-transact?]
                       :or {new? true
                            delete-blocks-fn (constantly [])
@@ -31,20 +97,20 @@
                       blocks []
                       ast []}}
               (cond (contains? gp-config/mldoc-support-formats format)
-                    (extract/extract file content extract-options')
+                (extract/extract file content extract-options')
 
-                    (gp-config/whiteboard? file)
-                    (extract/extract-whiteboard-edn file content extract-options')
+                (gp-config/whiteboard? file)
+                (extract/extract-whiteboard-edn file content extract-options')
 
-                    :else nil)
-              delete-blocks (delete-blocks-fn (first pages) file)
+                :else nil)
               block-ids (map (fn [block] {:block/uuid (:block/uuid block)}) blocks)
+              delete-blocks (delete-blocks-fn @conn (first pages) file block-ids)
               block-refs-ids (->> (mapcat :block/refs blocks)
                                   (filter (fn [ref] (and (vector? ref)
                                                          (= :block/uuid (first ref)))))
                                   (map (fn [ref] {:block/uuid (second ref)}))
                                   (seq))
-                   ;; To prevent "unique constraint" on datascript
+              ;; To prevent "unique constraint" on datascript
               block-ids (set/union (set block-ids) (set block-refs-ids))
               pages (extract/with-ref-pages pages blocks)
               pages-index (map #(select-keys % [:block/name]) pages)]

deps/graph-parser/src/logseq/graph_parser/cli.cljs

@@ -49,7 +49,8 @@ TODO: Fail fast when process exits 1"
     (mapv
      (fn [{:file/keys [path content]}]
        (let [{:keys [ast]}
-             (graph-parser/parse-file conn path content {:extract-options extract-options})]
+             (graph-parser/parse-file conn path content (merge {:extract-options extract-options}
+                                                               (:parse-file-options options)))]
          {:file path :ast ast}))
      files)))
 
@@ -59,12 +60,14 @@ TODO: Fail fast when process exits 1"
   as it can't assume that the metadata in logseq/ is up to date. Directory is
   assumed to be using git. This fn takes the following options:
 * :verbose - When enabled prints more information during parsing. Defaults to true
-* :files - Specific files to parse instead of parsing the whole directory"
+* :files - Specific files to parse instead of parsing the whole directory
+* :conn - Database connection to use instead of creating new one
+* :parse-file-options - Options map to pass to graph-parser/parse-file"
   ([dir]
    (parse-graph dir {}))
   ([dir options]
    (let [files (or (:files options) (build-graph-files dir))
-         conn (ldb/start-conn)
+         conn (or (:conn options) (ldb/start-conn))
          config (read-config dir)
         _ (when-not (:files options) (println "Parsing" (count files) "files..."))
          asts (parse-files conn files (merge options {:config config}))]

deps/graph-parser/test/logseq/graph_parser_test.cljs

@@ -74,7 +74,7 @@
                                                         (throw (js/Error "Testing unexpected failure")))]
         (try
           (graph-parser/parse-file conn "foo.md" "- id:: 628953c1-8d75-49fe-a648-f4c612109098"
-                                   {:delete-blocks-fn (fn [page _file]
+                                   {:delete-blocks-fn (fn [_db page _file _uuids]
                                                         (reset! deleted-page page))})
           (catch :default _)))
       (is (= nil @deleted-page)

e2e-tests/page-search.spec.ts

@@ -37,11 +37,12 @@ import { IsMac, createRandomPage, newBlock, newInnerBlock, randomString, lastBlo
   await page.waitForSelector('[placeholder="Search or create page"]')
   await page.fill('[placeholder="Search or create page"]', 'Einführung in die Allgemeine Sprachwissenschaft' + rand)
 
-  await page.waitForTimeout(500)
+  await page.waitForTimeout(2000) // wait longer for search contents to render
   const results = await page.$$('#ui__ac-inner>div')
-  expect(results.length).toEqual(3) // 2 blocks + 1 page
+  expect(results.length).toBeGreaterThan(3) // 2 blocks + 1 page + 2 page content
   await page.keyboard.press("Escape")
   await page.keyboard.press("Escape")
+  await page.waitForTimeout(1000) // wait for modal disappear
 })
 
 async function alias_test(page: Page, page_name: string, search_kws: string[]) {

resources/css/common.css

@@ -520,7 +520,6 @@ i.ti {
 h1.title {
   margin-bottom: 1.5rem;
   color: var(--ls-title-text-color, #222);
-  font-family: -apple-system, system-ui, var(--ls-font-family), sans-serif;
   font-size: var(--ls-page-title-size, 36px);
   font-weight: 500;
 }

src/electron/electron/handler.cljs

@@ -285,28 +285,47 @@
   (async/put! state/persistent-dbs-chan true)
   true)
 
+;; Search related IPCs
 (defmethod handle :search-blocks [_window [_ repo q opts]]
   (search/search-blocks repo q opts))
 
-(defmethod handle :rebuild-blocks-indice [_window [_ repo data]]
+(defmethod handle :search-pages [_window [_ repo q opts]]
+  (search/search-pages repo q opts))
+
+(defmethod handle :rebuild-indice [_window [_ repo block-data page-data]]
   (search/truncate-blocks-table! repo)
   ;; unneeded serialization
-  (search/upsert-blocks! repo (bean/->js data))
+  (search/upsert-blocks! repo (bean/->js block-data))
+  (search/truncate-pages-table! repo)
+  (search/upsert-pages! repo (bean/->js page-data))
   [])
 
 (defmethod handle :transact-blocks [_window [_ repo data]]
   (let [{:keys [blocks-to-remove-set blocks-to-add]} data]
+    ;; Order matters! Same id will delete then upsert sometimes.
     (when (seq blocks-to-remove-set)
       (search/delete-blocks! repo blocks-to-remove-set))
     (when (seq blocks-to-add)
       ;; unneeded serialization
       (search/upsert-blocks! repo (bean/->js blocks-to-add)))))
 
-(defmethod handle :truncate-blocks [_window [_ repo]]
-  (search/truncate-blocks-table! repo))
+(defmethod handle :transact-pages [_window [_ repo data]]
+  (let [{:keys [pages-to-remove-set pages-to-add]} data]
+    ;; Order matters! Same id will delete then upsert sometimes.
+    (when (seq pages-to-remove-set)
+      (search/delete-pages! repo pages-to-remove-set))
+    (when (seq pages-to-add)
+      ;; unneeded serialization
+      (search/upsert-pages! repo (bean/->js pages-to-add)))))
+
+(defmethod handle :truncate-indice [_window [_ repo]]
+  (search/truncate-blocks-table! repo)
+  (search/truncate-pages-table! repo))
 
 (defmethod handle :remove-db [_window [_ repo]]
   (search/delete-db! repo))
+;; ^^^^
+;; Search related IPCs End
 
 (defn clear-cache!
   [window]

src/electron/electron/search.cljs

@@ -1,4 +1,5 @@
 (ns electron.search
+  "Provides both page level and block level index"
   (:require ["path" :as path]
             ["fs-extra" :as fs]
             ["better-sqlite3" :as sqlite3]
@@ -31,25 +32,52 @@
   (when db
     (.prepare db sql)))
 
-(defn add-triggers!
+(defn add-blocks-fts-triggers!
+  "Table bindings of blocks tables and the blocks FTS virtual tables"
   [db]
-  (let [triggers ["CREATE TRIGGER IF NOT EXISTS blocks_ad AFTER DELETE ON blocks
-    BEGIN
-        DELETE from blocks_fts where rowid = old.id;
-    END;"
+  (let [triggers [;; add
+                  "CREATE TRIGGER IF NOT EXISTS blocks_ad AFTER DELETE ON blocks
+                  BEGIN
+                      DELETE from blocks_fts where rowid = old.id;
+                  END;"
+                  ;; insert
                   "CREATE TRIGGER IF NOT EXISTS blocks_ai AFTER INSERT ON blocks
-    BEGIN
-        INSERT INTO blocks_fts (rowid, uuid, content, page)
-        VALUES (new.id, new.uuid, new.content, new.page);
-    END;
-"
+                  BEGIN
+                      INSERT INTO blocks_fts (rowid, uuid, content, page)
+                      VALUES (new.id, new.uuid, new.content, new.page);
+                  END;"
+                  ;; update
                   "CREATE TRIGGER IF NOT EXISTS blocks_au AFTER UPDATE ON blocks
-    BEGIN
-        DELETE from blocks_fts where rowid = old.id;
-        INSERT INTO blocks_fts (rowid, uuid, content, page)
-        VALUES (new.id, new.uuid, new.content, new.page);
-    END;"
-                  ]]
+                  BEGIN
+                      DELETE from blocks_fts where rowid = old.id;
+                      INSERT INTO blocks_fts (rowid, uuid, content, page)
+                      VALUES (new.id, new.uuid, new.content, new.page);
+                  END;"]]
+    (doseq [trigger triggers]
+      (let [stmt (prepare db trigger)]
+        (.run ^object stmt)))))
+
+(defn add-pages-fts-triggers!
+  "Table bindings of pages tables and the pages FTS virtual tables"
+  [db]
+  (let [triggers [;; add
+                  "CREATE TRIGGER IF NOT EXISTS pages_ad AFTER DELETE ON pages
+                  BEGIN
+                      DELETE from pages_fts where rowid = old.id;
+                  END;"
+                  ;; insert
+                  "CREATE TRIGGER IF NOT EXISTS pages_ai AFTER INSERT ON pages
+                  BEGIN
+                      INSERT INTO pages_fts (rowid, uuid, content)
+                      VALUES (new.id, new.uuid, new.content);
+                  END;"
+                  ;; update
+                  "CREATE TRIGGER IF NOT EXISTS pages_au AFTER UPDATE ON pages
+                  BEGIN
+                      DELETE from pages_fts where rowid = old.id;
+                      INSERT INTO pages_fts (rowid, uuid, content)
+                      VALUES (new.id, new.uuid, new.content);
+                  END;"]]
     (doseq [trigger triggers]
       (let [stmt (prepare db trigger)]
         (.run ^object stmt)))))
@@ -68,6 +96,19 @@
   (let [stmt (prepare db "CREATE VIRTUAL TABLE IF NOT EXISTS blocks_fts USING fts5(uuid, content, page)")]
     (.run ^object stmt)))
 
+(defn create-pages-table!
+  [db]
+  (let [stmt (prepare db "CREATE TABLE IF NOT EXISTS pages (
+                        id INTEGER PRIMARY KEY,
+                        uuid TEXT NOT NULL,
+                        content TEXT NOT NULL)")]
+    (.run ^object stmt)))
+
+(defn create-pages-fts-table!
+  [db]
+  (let [stmt (prepare db "CREATE VIRTUAL TABLE IF NOT EXISTS pages_fts USING fts5(uuid, content)")]
+    (.run ^object stmt)))
+
 (defn get-search-dir
   []
   (let [path (.getPath ^object app "userData")]
@@ -96,7 +137,10 @@
       (try (let [db (sqlite3 db-full-path nil)]
              (create-blocks-table! db)
              (create-blocks-fts-table! db)
-             (add-triggers! db)
+             (create-pages-table! db)
+             (create-pages-fts-table! db)
+             (add-blocks-fts-triggers! db)
+             (add-pages-fts-triggers! db)
              (swap! databases assoc db-sanitized-name db))
            (catch :default e
              (logger/error (str e ": " db-name))
@@ -111,6 +155,36 @@
       (doseq [db-name dbs]
         (open-db! db-name)))))
 
+(defn- clj-list->sql
+  "Turn clojure list into SQL list
+   '(1 2 3 4)
+   ->
+   \"('1','2','3','4')\""
+  [ids]
+  (str "(" (->> (map (fn [id] (str "'" id "'")) ids)
+                (string/join ", ")) ")"))
+
+(defn upsert-pages!
+  [repo pages]
+  (if-let [db (get-db repo)]
+    ;; TODO: what if a CONFLICT on uuid
+    (let [insert (prepare db "INSERT INTO pages (id, uuid, content) VALUES (@id, @uuid, @content) ON CONFLICT (id) DO UPDATE SET content = @content")
+          insert-many (.transaction ^object db
+                                    (fn [pages]
+                                      (doseq [page pages]
+                                        (.run ^object insert page))))]
+      (insert-many pages))
+    (do
+      (open-db! repo)
+      (upsert-pages! repo pages))))
+
+(defn delete-pages!
+  [repo ids]
+  (when-let [db (get-db repo)]
+    (let [sql (str "DELETE from pages WHERE id IN " (clj-list->sql ids))
+          stmt (prepare db sql)]
+      (.run ^object stmt))))
+
 (defn upsert-blocks!
   [repo blocks]
   (if-let [db (get-db repo)]
@@ -128,9 +202,7 @@
 (defn delete-blocks!
   [repo ids]
   (when-let [db (get-db repo)]
-    (let [ids (->> (map (fn [id] (str "'" id "'")) ids)
-                   (string/join ", "))
-          sql (str "DELETE from blocks WHERE id IN (" ids ")")
+    (let [sql (str "DELETE from blocks WHERE id IN " (clj-list->sql ids))
           stmt (prepare db sql)]
       (.run ^object stmt))))
 
@@ -150,19 +222,35 @@
        (.all ^object stmt  input limit))
      :keywordize-keys true)))
 
+(defn- get-match-inputs
+  [q]
+  (let [match-input (-> q
+                        (string/replace " and " " AND ")
+                        (string/replace " & " " AND ")
+                        (string/replace " or " " OR ")
+                        (string/replace " | " " OR ")
+                        (string/replace " not " " NOT "))]
+    (if (not= q match-input)
+      [(string/replace match-input "," "")]
+      [q
+       (str "\"" match-input "\"")])))
+
+(defn distinct-by
+  [f col]
+  (reduce
+   (fn [acc x]
+     (if (some #(= (f x) (f %)) acc)
+       acc
+       (vec (conj acc x))))
+   []
+   col))
+
 (defn search-blocks
+  ":page - the page to specificly search on"
   [repo q {:keys [limit page]}]
   (when-let [database (get-db repo)]
     (when-not (string/blank? q)
-      (let [match-input (-> q
-                            (string/replace " and " " AND ")
-                            (string/replace " & " " AND ")
-                            (string/replace " or " " OR ")
-                            (string/replace " | " " OR ")
-                            (string/replace " not " " NOT "))
-            match-input (if (not= q match-input)
-                          (string/replace match-input "," "")
-                          (str "\"" match-input "\""))
+      (let [match-inputs (get-match-inputs q)
             non-match-input (str "%" (string/replace q #"\s+" "%") "%")
             limit  (or limit 20)
             select "select rowid, uuid, content, page from blocks_fts where "
@@ -172,12 +260,62 @@
                            " content match ? order by rank limit ?")
             non-match-sql (str select
                                pg-sql
-                               " content like ? limit ?")]
+                               " content like ? limit ?")
+            matched-result (->>
+                            (map
+                              (fn [match-input]
+                                (search-blocks-aux database match-sql match-input page limit))
+                              match-inputs)
+                            (apply concat))]
+        (->>
+         (concat matched-result
+                 (search-blocks-aux database non-match-sql non-match-input page limit))
+         (distinct-by :id)
+         (take limit)
+         (vec))))))
+
+(defn- search-pages-res-unpack
+  [arr]
+  (let [[rowid uuid content snippet] arr]
+    {:id      rowid
+     :uuid    uuid
+     :content content
+     :snippet snippet}))
+
+(defn- search-pages-aux
+  [database sql input limit]
+  (let [stmt (prepare database sql)]
+    (map search-pages-res-unpack (-> (.raw ^object stmt)
+                                     (.all input limit)
+                                     (js->clj)))))
+
+(defn search-pages
+  [repo q {:keys [limit]}]
+  (when-let [database (get-db repo)]
+    (when-not (string/blank? q)
+      (let [match-inputs (get-match-inputs q)
+            non-match-input (str "%" (string/replace q #"\s+" "%") "%")
+            limit  (or limit 20)
+            ;; https://www.sqlite.org/fts5.html#the_highlight_function
+            ;; the 2nd column in pages_fts (content)
+            ;; pfts_2lqh is a key for retrieval
+            ;; highlight and snippet only works for some matching with high rank
+            snippet-aux "snippet(pages_fts, 1, '$pfts_2lqh>$', '$<pfts_2lqh$', '...', 32)"
+            select (str "select rowid, uuid, content, " snippet-aux " from pages_fts where ")
+            match-sql (str select
+                           " content match ? order by rank limit ?")
+            non-match-sql (str select
+                               " content like ? limit ?")
+            matched-result (->>
+                            (map
+                              (fn [match-input]
+                                (search-pages-aux database match-sql match-input limit))
+                              match-inputs)
+                            (apply concat))]
         (->>
-         (concat
-          (search-blocks-aux database match-sql match-input page limit)
-          (search-blocks-aux database non-match-sql non-match-input page limit))
-         (distinct)
+         (concat matched-result
+          (search-pages-aux database non-match-sql non-match-input limit))
+         (distinct-by :id)
          (take limit)
          (vec))))))
 
@@ -191,6 +329,16 @@
                         "delete from blocks_fts;")]
       (.run ^object stmt))))
 
+(defn truncate-pages-table!
+  [repo]
+  (when-let [database (get-db repo)]
+    (let [stmt (prepare database
+                        "delete from pages;")
+          _ (.run ^object stmt)
+          stmt (prepare database
+                        "delete from pages_fts;")]
+      (.run ^object stmt))))
+
 (defn delete-db!
   [repo]
   (when-let [database (get-db repo)]
@@ -205,9 +353,3 @@
   (when-let [database (get-db repo)]
     (let [stmt (prepare database sql)]
       (.all ^object stmt))))
-
-(comment
-  (def repo (first (keys @databases)))
-  (query repo
-         "select * from blocks_fts")
-  (delete-db! repo))

src/main/frontend/components/block.cljs

@@ -385,8 +385,12 @@
             share-fn (fn [event]
                        (util/stop event)
                        (when (mobile-util/native-platform?)
-                         (.share Share #js {:url path
-                                            :title "Open file with your favorite app"})))]
+                         ;; File URL must be legal, so filename muse be URI-encoded
+                         (let [[rel-dir basename] (util/get-dir-and-basename href)
+                               basename (js/encodeURIComponent basename)
+                               asset-url (str repo-dir rel-dir "/" basename)]
+                           (.share Share (clj->js {:url asset-url
+                                                   :title "Open file with your favorite app"})))))]
 
         (cond
           (contains? config/audio-formats ext)
@@ -401,7 +405,7 @@
           [:a.asset-ref.is-plaintext {:href (rfe/href :file {:path path})
                                       :on-click (fn [_event]
                                                   (p/let [result (fs/read-file repo-dir path)]
-                                                    (db/set-file-content! repo path result )))}
+                                                    (db/set-file-content! repo path result)))}
            title]
 
           (= ext :pdf)
@@ -2491,12 +2495,14 @@
 (rum/defc breadcrumb-separator [] [:span.mx-2.opacity-50 "➤"])
 
 (defn breadcrumb
+  "block-id - uuid of the target block of breadcrumb. page uuid is also acceptable"
   [config repo block-id {:keys [show-page? indent? end-separator? level-limit _navigating-block]
                          :or {show-page? true
                               level-limit 3}
                          :as opts}]
   (let [parents (db/get-block-parents repo block-id (inc level-limit))
-        page (db/get-block-page repo block-id)
+        page (or (db/get-block-page repo block-id) ;; only return for block uuid
+                 (model/query-block-by-uuid block-id)) ;; return page entity when received page uuid
         page-name (:block/name page)
         page-original-name (:block/original-name page)
         show? (or (seq parents) show-page? page-name)

src/main/frontend/components/conversion.cljs

@@ -13,6 +13,7 @@
             [frontend.context.i18n :refer [t]]
             [rum.core :as rum]
             [frontend.handler.file-sync :as file-sync-handler]
+            [frontend.fs.sync :as sync]
             [frontend.handler.notification :as notification]))
 
 (defn- ask-for-re-index
@@ -28,15 +29,26 @@
 
 (defn- <close-modal-on-done
   "Ask users to re-index when the modal is exited"
-  [sync?]
-  (async/go (state/close-settings!)
-            (async/<! (async/timeout 100)) ;; modal race condition requires investigation
-            (if sync?
-              (notification/show!
-               [:div "Please re-index this graph after all the changes are synced."]
-               :warning
-               false)
-              (ask-for-re-index))))
+  [sync? rename-items]
+  (async/go
+    (state/close-modal!)
+    (async/<! (async/timeout 100)) ;; modal race condition requires investigation
+    (let [renamed-paths (keep (fn [{:keys [file file-name target]}]
+                                (when (not= file-name target)
+                                  (sync/relative-path (:file/path file)))) rename-items)
+          graph-txid (second @sync/graphs-txid)]
+      (when (and (seq renamed-paths) sync? graph-txid)
+        (async/<!
+         (sync/<delete-remote-files-control
+          sync/remoteapi
+          graph-txid
+          renamed-paths))))
+    (if sync?
+      (notification/show!
+       [:div "Please re-index this graph after all the changes are synced."]
+       :warning
+       false)
+      (ask-for-re-index))))
 
 (rum/defc legacy-warning
   [repo *target-format *dir-format *solid-format]
@@ -123,10 +135,11 @@
                                         (merge ret {:page page :file file}))))
                                (remove nil?))
             sync? (file-sync-handler/current-graph-sync-on?)
-            <rename-all   #(async/go (doseq [{:keys [file target status]} rename-items]
-                                       (when (not= status :unreachable)
-                                         (async/<! (p->c (page-handler/rename-file! file target (constantly nil) true)))))
-                                     (<close-modal-on-done sync?))]
+            <rename-all   #(async/go
+                             (doseq [{:keys [file target status]} rename-items]
+                               (when (not= status :unreachable)
+                                 (async/<! (p->c (page-handler/rename-file! file target (constantly nil) true)))))
+                             (<close-modal-on-done sync? rename-items))]
 
         (if (not-empty rename-items)
           [:div ;; Normal UX stage 2: close stage 1 UI, show the action description as admolition
@@ -154,12 +167,7 @@
                      rename-fn      #(page-handler/rename-file! file target rm-item-fn)
                      rename-but     [:a {:on-click rename-fn
                                          :title (t :file-rn/apply-rename)}
-                                     [:span (t :file-rn/rename src-file-name tgt-file-name)]]
-                     rename-but-sm  (ui/button
-                                     (t :file-rn/rename-sm)
-                                     :on-click rename-fn
-                                     :class "text-sm p-1 mr-1"
-                                     :style {:word-break "normal"})]
+                                     [:span (t :file-rn/rename src-file-name tgt-file-name)]]]
                  [:tr {:key (:block/name page)}
                   [:td [:div [:p "📄 " old-title]]
                    (case status
@@ -168,6 +176,5 @@
                       [:p (t :file-rn/otherwise-breaking) " \"" changed-title \"]]
                      :unreachable
                      [:div [:p "🔴 " (t :file-rn/unreachable-title changed-title)]]
-                     [:div [:p "🟢 " (t :file-rn/optional-rename) rename-but]])]
-                  [:td rename-but-sm]]))]]]
+                     [:div [:p "🟢 " (t :file-rn/optional-rename) rename-but]])]]))]]]
           [:div "🎉 " (t :file-rn/no-action)]))]]))

src/main/frontend/components/search.cljs

@@ -22,7 +22,8 @@
             [frontend.context.i18n :refer [t]]
             [frontend.date :as date]
             [reitit.frontend.easy :as rfe]
-            [frontend.modules.shortcut.core :as shortcut]))
+            [frontend.modules.shortcut.core :as shortcut]
+            [frontend.util.text :as text-util]))
 
 (defn highlight-exact-query
   [content q]
@@ -62,12 +63,43 @@
                              (conj result [:span content])))]
             [:p {:class "m-0"} elements]))))))
 
+(defn highlight-page-content-query
+  "Return hiccup of highlighted page content FTS result"
+  [content q]
+  (when-not (or (string/blank? content) (string/blank? q))
+    [:div (loop [content content ;; why recur? because there might be multiple matches
+                 result  []]
+            (let [[b-cut hl-cut e-cut] (text-util/cut-by content "$pfts_2lqh>$" "$<pfts_2lqh$")
+                  hiccups-add [(when-not (string/blank? b-cut)
+                                 [:span b-cut])
+                               (when-not (string/blank? hl-cut)
+                                 [:mark.p-0.rounded-none hl-cut])]
+                  hiccups-add (remove nil? hiccups-add)
+                  new-result (concat result hiccups-add)]
+              (if-not (string/blank? e-cut)
+                (recur e-cut new-result)
+                new-result)))]))
+
 (rum/defc search-result-item
   [icon content]
   [:.search-result
    (ui/type-icon icon)
    [:.self-center content]])
 
+(rum/defc page-content-search-result-item
+  [repo uuid format snippet q search-mode]
+  [:div
+   (when (not= search-mode :page)
+     [:div {:class "mb-1" :key "parents"}
+      (block/breadcrumb {:id "block-search-block-parent"
+                         :block? true
+                         :search? true}
+                        repo
+                        (clojure.core/uuid uuid)
+                        {:indent? false})])
+   [:div {:class "font-medium" :key "content"}
+    (highlight-page-content-query (search-handler/sanity-search-content format snippet) q)]])
+
 (rum/defc block-search-result-item
   [repo uuid format content q search-mode]
   (let [content (search-handler/sanity-search-content format content)]
@@ -157,6 +189,21 @@
         (println "[Error] Block page missing: "
                  {:block-id block-uuid
                   :block (db/pull [:block/uuid block-uuid])})))
+
+    :page-content
+    (let [page-uuid (uuid (:block/uuid data))
+          page (model/get-block-by-uuid page-uuid)
+          page-name (:block/name page)]
+      (if page
+        (cond
+          (model/whiteboard-page? page-name)
+          (route/redirect-to-whiteboard! page-name)
+          :else
+          (route/redirect-to-page! page-name))
+        ;; search indice outdated
+        (println "[Error] page missing: "
+                 {:page-uuid page-uuid
+                  :page page})))
     nil)
   (state/close-modal!))
 
@@ -172,6 +219,19 @@
          repo
          (:db/id page)
          :page)))
+    
+    :page-content
+    (let [page-uuid (uuid (:block/uuid data))
+          page (model/get-block-by-uuid page-uuid)]
+      (if page
+        (state/sidebar-add-block!
+         repo
+         (:db/id page)
+         :page)
+        ;; search indice outdated
+        (println "[Error] page missing: "
+                 {:page-uuid page-uuid
+                  :page page})))
 
     :block
     (let [block-uuid (uuid (:block/uuid data))
@@ -254,10 +314,24 @@
                                 (do (log/error "search result with non-existing uuid: " data)
                                     (str "Cache is outdated. Please click the 'Re-index' button in the graph's dropdown menu."))))])
 
+       :page-content
+       (let [{:block/keys [snippet uuid]} data  ;; content here is normalized
+             repo (state/sub :git/current-repo)
+             page (model/query-block-by-uuid uuid)  ;; it's actually a page
+             format (db/get-page-format page)]
+         [:span {:data-block-ref uuid}
+          (search-result-item {:name "page"
+                               :title (t :search-item/page)
+                               :extension? true}
+                              (if page
+                                (page-content-search-result-item repo uuid format snippet search-q search-mode)
+                                (do (log/error "search result with non-existing uuid: " data)
+                                    (str "Cache is outdated. Please click the 'Re-index' button in the graph's dropdown menu."))))])
+
        nil)]))
 
 (rum/defc search-auto-complete
-  [{:keys [engine pages files blocks has-more?] :as result} search-q all?]
+  [{:keys [engine pages files pages-content blocks has-more?] :as result} search-q all?]
   (let [pages (when-not all? (map (fn [page]
                                     (let [alias (model/get-redirect-page-name page)]
                                       (cond->
@@ -270,6 +344,7 @@
                                   (remove nil? pages)))
         files (when-not all? (map (fn [file] {:type :file :data file}) files))
         blocks (map (fn [block] {:type :block :data block}) blocks)
+        pages-content (map (fn [pages-content] {:type :page-content :data pages-content}) pages-content)
         search-mode (state/sub :search/mode)
         new-page (if (or
                       (some? engine)
@@ -284,13 +359,13 @@
                      [{:type :new-page}]))
         result (cond
                  config/publishing?
-                 (concat pages files blocks)
+                 (concat pages files blocks) ;; Browser doesn't have page content FTS
 
                  (= :whiteboard/link search-mode)
-                 (concat pages blocks)
+                 (concat pages blocks pages-content)
 
                  :else
-                 (concat new-page pages files blocks))
+                 (concat new-page pages files blocks pages-content))
         result (if (= search-mode :graph)
                  [{:type :graph-add-filter}]
                  result)

src/main/frontend/config.cljs

@@ -66,11 +66,6 @@
     "http://localhost:3000"
     (util/format "https://%s.com" app-name)))
 
-(def api
-  (if dev?
-    "http://localhost:3000/api/v1/"
-    (str website "/api/v1/")))
-
 (def asset-domain (util/format "https://asset.%s.com"
                                app-name))
 
@@ -132,7 +127,7 @@
    *** Warning!!! ***
    For UX logic only! Don't use for FS logic
    iPad / Android Pad doesn't trigger!
-   
+
    Same as config/mobile?"
   (when-not util/node-test?
     (util/safe-re-find #"Mobi" js/navigator.userAgent)))

src/main/frontend/db.cljs

@@ -38,13 +38,13 @@
 
  [frontend.db.model
   blocks-count blocks-count-cache clean-export! delete-blocks get-pre-block
-  delete-file-blocks! delete-page-blocks delete-files delete-pages-by-files
+  delete-files delete-pages-by-files
   filter-only-public-pages-and-blocks get-all-block-contents get-all-tagged-pages
   get-all-templates get-block-and-children get-block-by-uuid get-block-children sort-by-left
   get-block-parent get-block-parents parents-collapsed? get-block-referenced-blocks get-all-referenced-blocks-uuid
   get-block-children-ids get-block-immediate-children get-block-page
   get-custom-css get-date-scheduled-or-deadlines
-  get-file-blocks get-file-last-modified-at get-file get-file-page get-file-page-id file-exists?
+  get-file-last-modified-at get-file get-file-page get-file-page-id file-exists?
   get-files get-files-blocks get-files-full get-journals-length get-pages-with-file
   get-latest-journals get-page get-page-alias get-page-alias-names get-paginated-blocks
   get-page-blocks-count get-page-blocks-no-cache get-page-file get-page-format get-page-properties

src/main/frontend/db/model.cljs

@@ -212,17 +212,6 @@
              (conn/get-db repo-url) pred)
         db-utils/seq-flatten)))
 
-(defn get-file-blocks
-  [repo-url path]
-  (-> (d/q '[:find ?block
-             :in $ ?path
-             :where
-             [?file :file/path ?path]
-             [?p :block/file ?file]
-             [?block :block/page ?p]]
-           (conn/get-db repo-url) path)
-      db-utils/seq-flatten))
-
 (defn set-file-last-modified-at!
   [repo path last-modified-at]
   (when (and repo path last-modified-at)
@@ -274,6 +263,7 @@
   (db-utils/entity [:block/uuid (if (uuid? id) id (uuid id))]))
 
 (defn query-block-by-uuid
+  "Return block or page entity, depends on the uuid"
   [id]
   (db-utils/pull [:block/uuid (if (uuid? id) id (uuid id))]))
 
@@ -785,6 +775,8 @@
         react)))))
 
 (defn get-page-blocks-no-cache
+  "Return blocks of the designated page, without using cache.
+   page - name / title of the page"
   ([page]
    (get-page-blocks-no-cache (state/get-current-repo) page nil))
   ([repo-url page]
@@ -1528,6 +1520,7 @@
             assets (get-assets datoms)]
         [@(d/conn-from-datoms datoms db-schema/schema) assets]))))
 
+;; Deprecated?
 (defn delete-blocks
   [repo-url files _delete-page?]
   (when (seq files)
@@ -1538,21 +1531,6 @@
   [files]
   (mapv (fn [path] [:db.fn/retractEntity [:file/path path]]) files))
 
-(defn delete-file-blocks!
-  [repo-url path]
-  (let [blocks (get-file-blocks repo-url path)]
-    (mapv (fn [eid] [:db.fn/retractEntity eid]) blocks)))
-
-(defn delete-page-blocks
-  [repo-url page]
-  (when page
-    (when-let [db (conn/get-db repo-url)]
-      (let [page (db-utils/pull [:block/name (util/page-name-sanity-lc page)])]
-        (when page
-          (let [datoms (d/datoms db :avet :block/page (:db/id page))
-                block-eids (mapv :e datoms)]
-            (mapv (fn [eid] [:db.fn/retractEntity eid]) block-eids)))))))
-
 (defn delete-pages-by-files
   [files]
   (let [pages (->> (mapv get-file-page files)

src/main/frontend/dicts.cljc

@@ -132,7 +132,6 @@
         :file-rn/all-action "Apply all Actions!"
         :file-rn/select-format "(Developer Mode Option, Dangerous!) Select filename format"
         :file-rn/rename "rename file \"{1}\" to \"{2}\""
-        :file-rn/rename-sm "Rename"
         :file-rn/apply-rename "Apply the file rename operation"
         :file-rn/affected-pages "Affected Pages after the format change"
         :file-rn/suggest-rename "Action required: "
@@ -1274,7 +1273,6 @@
         :file-rn/otherwise-breaking "Ou le titre deviendra"
         :file-rn/re-index "La réindexation est fortement recommandée après que les fichiers aient été renommés, puis sur les autres postes après synchronisation."
         :file-rn/rename "renommer le fichier \"{1}\" en \"{2}\""
-        :file-rn/rename-sm "Renommer"
         :file-rn/select-confirm-proceed "Dev: format d'écriture"
         :file-rn/select-format "(Option du Mode Developpeur, Danger !) Sélectionnez le format de nom de fichier"
         :file-rn/suggest-rename "Action requise : "
@@ -1586,7 +1584,6 @@
            :file-rn/all-action "应用所有操作！"
            :file-rn/select-format "(开发者模式选项，危险！) 选择文件名格式"
            :file-rn/rename "重命名文件 \"{1}\" 到 \"{2}\""
-           :file-rn/rename-sm "重命名"
            :file-rn/apply-rename "应用文件重命名操作"
            :file-rn/affected-pages "格式改变后，影响的文件"
            :file-rn/suggest-rename "需要的操作： "
@@ -2272,7 +2269,6 @@
         :file-rn/all-action "¡Aplicar todas las acciones!"
         :file-rn/select-format "(Opción modo desarrollador, ¡peligroso!) Seccione el formato de nombre de archivo"
         :file-rn/rename "Renombrar \"{1}\" a \"{2}\""
-        :file-rn/rename-sm "Renombrar"
         :file-rn/apply-rename "Aplicar la operación de cambio de nombre de archivo"
         :file-rn/affected-pages "Páginas afectadas después del cambio de formato"
         :file-rn/suggest-rename "Acción necesaria: "
@@ -4622,7 +4618,6 @@
         :file-rn/all-action "Tüm Eylemleri Uygula!"
         :file-rn/select-format "(Geliştirici Modu Seçeneği, Tehlikeli!) Dosya adı biçimini seçin"
         :file-rn/rename "\"{1}\" dosyasını \"{2}\" olarak yeniden adlandır"
-        :file-rn/rename-sm "Yeniden adlandır"
         :file-rn/apply-rename "Dosya yeniden adlandırma işlemini uygula"
         :file-rn/affected-pages "Biçim değişikliğinden sonra Etkilenen Sayfalar"
         :file-rn/suggest-rename "Eylem gereklidir: "

src/main/frontend/extensions/pdf/toolbar.cljs

@@ -519,6 +519,8 @@
         [:span.nu.flex.items-center.opacity-70
          [:input {:ref            *page-ref
                   :type           "number"
+                  :min            1
+                  :max            total-page-num
                   :class          (util/classnames [{:is-long (> (util/safe-parse-int current-page-num) 999)}])
                   :default-value  current-page-num
                   :on-mouse-enter #(.select ^js (.-target %))
@@ -553,4 +555,4 @@
         viewer-theme
         {:t              t
          :hide-settings! #(set-settings-visible! false)
-         :select-theme!  #(set-viewer-theme! %)}))]))
+         :select-theme!  #(set-viewer-theme! %)}))]))

src/main/frontend/fs/sync.cljs

@@ -494,7 +494,8 @@
         reserved-paths (filter f paths)]
     (when (seq reserved-paths)
       (let [paths (if path-string? reserved-paths (map -relative-path reserved-paths))]
-        (state/pub-event! [:ui/notify-outdated-filename-format paths])
+        (when (seq paths)
+          (state/pub-event! [:ui/notify-outdated-filename-format paths]))
         (prn "Skipped uploading those file paths with reserved chars: " paths)))
     (vec (remove f paths))))
 
@@ -750,7 +751,6 @@
       (when (some-> r first :path (not= filepath))
         (-> r first :path)))))
 
-
 (defn <local-file-not-exist?
   [graph-uuid irsapi base-path filepath]
   (go
@@ -778,6 +778,22 @@
 
 (declare <rsapi-cancel-all-requests)
 
+(defn- build-local-file-metadatas
+  [this graph-uuid result]
+  (loop [[[path metadata] & others] (js->clj result)
+         result #{}]
+    (if-not (and path metadata)
+      ;; finish
+      result
+      (let [normalized-path (path-normalize path)
+            encryptedFname (if (not= path normalized-path)
+                             (first (<! (<encrypt-fnames this graph-uuid [normalized-path])))
+                             (get metadata "encryptedFname"))]
+        (recur others
+               (conj result
+                     (->FileMetadata (get metadata "size") (get metadata "md5") normalized-path
+                                     encryptedFname (get metadata "mtime") false nil)))))))
+
 (deftype RSAPI [^:mutable graph-uuid' ^:mutable private-key' ^:mutable public-key']
   IToken
   (<get-token [_this]
@@ -795,26 +811,17 @@
     (set! private-key' private-key)
     (set! public-key' public-key)
     (p->c (ipc/ipc "set-env" graph-uuid (if prod? "prod" "dev") private-key public-key)))
-  (<get-local-all-files-meta [_ graph-uuid base-path]
+  (<get-local-all-files-meta [this graph-uuid base-path]
     (go
       (let [r (<! (<retry-rsapi #(p->c (ipc/ipc "get-local-all-files-meta" graph-uuid base-path))))]
         (if (instance? ExceptionInfo r)
           r
-          (->> r
-               js->clj
-               (map (fn [[path metadata]]
-                      (->FileMetadata (get metadata "size") (get metadata "md5") (path-normalize path)
-                                      (get metadata "encryptedFname") (get metadata "mtime") false nil)))
-               set)))))
-  (<get-local-files-meta [_ graph-uuid base-path filepaths]
+          (build-local-file-metadatas this graph-uuid r)))))
+  (<get-local-files-meta [this graph-uuid base-path filepaths]
     (go
       (let [r (<! (<retry-rsapi #(p->c (ipc/ipc "get-local-files-meta" graph-uuid base-path filepaths))))]
         (assert (not (instance? ExceptionInfo r)) "get-local-files-meta shouldn't return exception")
-        (->> r
-             js->clj
-             (map (fn [[path metadata]]
-                    (->FileMetadata (get metadata "size") (get metadata "md5") (path-normalize path)
-                                    (get metadata "encryptedFname") (get metadata "mtime") false nil)))))))
+        (build-local-file-metadatas this graph-uuid r))))
   (<rename-local-file [_ graph-uuid base-path from to]
     (<retry-rsapi #(p->c (ipc/ipc "rename-local-file" graph-uuid base-path
                                   (path-normalize from)
@@ -887,36 +894,22 @@
                                                    :secretKey secret-key
                                                    :publicKey public-key}))))
 
-  (<get-local-all-files-meta [_ graph-uuid base-path]
+  (<get-local-all-files-meta [this graph-uuid base-path]
     (go
       (let [r (<! (p->c (.getLocalAllFilesMeta mobile-util/file-sync (clj->js {:graphUUID graph-uuid
                                                                                :basePath base-path}))))]
         (if (instance? ExceptionInfo r)
           r
-          (->> (.-result r)
-               js->clj
-               (map (fn [[path metadata]]
-                      (->FileMetadata (get metadata "size") (get metadata "md5")
-                                      ;; return decoded path, keep it consistent with RSAPI
-                                      (path-normalize path)
-                                      (get metadata "encryptedFname") (get metadata "mtime") false nil)))
-               set)))))
-
-  (<get-local-files-meta [_ graph-uuid base-path filepaths]
+          (build-local-file-metadatas this graph-uuid (.-result r))))))
+
+  (<get-local-files-meta [this graph-uuid base-path filepaths]
     (go
       (let [r (<! (p->c (.getLocalFilesMeta mobile-util/file-sync
                                             (clj->js {:graphUUID graph-uuid
                                                       :basePath base-path
                                                       :filePaths filepaths}))))]
         (assert (not (instance? ExceptionInfo r)) "get-local-files-meta shouldn't return exception")
-        (->> (.-result r)
-             js->clj
-             (map (fn [[path metadata]]
-                    (->FileMetadata (get metadata "size") (get metadata "md5")
-                                    ;; return decoded path, keep it consistent with RSAPI
-                                    (path-normalize path)
-                                    (get metadata "encryptedFname") (get metadata "mtime") false nil)))
-             set))))
+        (build-local-file-metadatas this graph-uuid (.-result r)))))
 
   (<rename-local-file [_ graph-uuid base-path from to]
     (p->c (.renameLocalFile mobile-util/file-sync
@@ -1175,10 +1168,7 @@
     (loop [[raw-path & other-paths] raw-paths]
       (when raw-path
         (let [normalized-path (path-normalize raw-path)]
-          (when (and (not= normalized-path raw-path)
-                     (get path->encrypted-path-map normalized-path))
-            ;; raw-path is un-normalized path and there are related normalized version one,
-            ;; then filter out this raw-path
+          (when (not= normalized-path raw-path)
             (println :filter-files-with-unnormalized-path raw-path)
             (conj! *encrypted-paths-to-drop (get path->encrypted-path-map raw-path))))
         (recur other-paths)))
@@ -1228,7 +1218,7 @@
                 (mapv
                  #(->FileMetadata (:size %)
                                   (:checksum %)
-                                  (path-normalize (get encrypted-path->path-map (:encrypted-path %)))
+                                  (get encrypted-path->path-map (:encrypted-path %))
                                   (:encrypted-path %)
                                   (:last-modified %)
                                   true nil)
@@ -1370,9 +1360,12 @@
   IRemoteControlAPI
   (<delete-remote-files-control [this graph-uuid filepaths]
     (user/<wrap-ensure-id&access-token
-     (let [current-txid (:TXId (<! (<get-remote-graph this nil graph-uuid)))
-           files (<! (<encrypt-fnames rsapi graph-uuid filepaths))]
-       (<! (.<request this "delete_files" {:GraphUUID graph-uuid :TXId current-txid :Files files}))))))
+     (let [partitioned-files (partition-all 20 (<! (<encrypt-fnames rsapi graph-uuid filepaths)))]
+       (loop [[files & others] partitioned-files]
+         (when files
+           (let [current-txid (:TXId (<! (<get-remote-graph this nil graph-uuid)))]
+             (<! (.<request this "delete_files" {:GraphUUID graph-uuid :TXId current-txid :Files files}))
+             (recur others))))))))
 
 (comment
   (declare remoteapi)
@@ -1753,7 +1746,6 @@
                                     (<! (<get-local-files-meta
                                          rsapi (:current-syncing-graph-uuid sync-state) dir [path])))
                     checksum (and (coll? files-meta) (some-> files-meta first :etag))]
-                (println :files-watch (->FileChangeEvent type dir path stat checksum))
                 (>! local-changes-chan (->FileChangeEvent type dir path stat checksum))))))))))
 
 (defn local-changes-revised-chan-builder
@@ -3125,50 +3117,51 @@
 
 (defn <sync-start
   []
-  (go
-    (when (false? @*sync-entered?)
-      (reset! *sync-entered? true)
-      (let [*sync-state                 (atom (sync-state))
-            current-user-uuid           (<! (user/<user-uuid))
-            ;; put @graph-uuid & get-current-repo together,
-            ;; prevent to get older repo dir and current graph-uuid.
-            _                           (<! (p->c (persist-var/-load graphs-txid)))
-            [user-uuid graph-uuid txid] @graphs-txid
-            txid                        (or txid 0)
-            repo                        (state/get-current-repo)]
-        (when-not (instance? ExceptionInfo current-user-uuid)
-          (when (and repo
-                     @network-online-cursor
-                     user-uuid graph-uuid txid
-                     (graph-sync-off? graph-uuid)
-                     (user/logged-in?)
-                     (not (config/demo-graph? repo)))
-            (try
-              (when-let [sm (sync-manager-singleton current-user-uuid graph-uuid
-                                                    (config/get-repo-dir repo) repo
-                                                    txid *sync-state)]
-                (when (check-graph-belong-to-current-user current-user-uuid user-uuid)
-                  (if-not (<! (<check-remote-graph-exists graph-uuid)) ; remote graph has been deleted
-                    (clear-graphs-txid! repo)
-                    (do
-                      (state/set-file-sync-state graph-uuid @*sync-state)
-                      (state/set-file-sync-manager graph-uuid sm)
-
-                      ;; update global state when *sync-state changes
-                      (add-watch *sync-state ::update-global-state
-                                 (fn [_ _ _ n]
-                                   (state/set-file-sync-state graph-uuid n)))
-
-                      (state/set-state! [:file-sync/graph-state :current-graph-uuid] graph-uuid)
-
-                      (.start sm)
-
-                      (offer! remote->local-full-sync-chan true)
-                      (offer! full-sync-chan true)))))
-              (catch :default e
-                (prn "Sync start error: ")
-                (log/error :exception e)))))
-        (reset! *sync-entered? false)))))
+  (when-not (false? (state/enable-sync?))
+    (go
+      (when (false? @*sync-entered?)
+        (reset! *sync-entered? true)
+        (let [*sync-state                 (atom (sync-state))
+              current-user-uuid           (<! (user/<user-uuid))
+              ;; put @graph-uuid & get-current-repo together,
+              ;; prevent to get older repo dir and current graph-uuid.
+              _                           (<! (p->c (persist-var/-load graphs-txid)))
+              [user-uuid graph-uuid txid] @graphs-txid
+              txid                        (or txid 0)
+              repo                        (state/get-current-repo)]
+          (when-not (instance? ExceptionInfo current-user-uuid)
+            (when (and repo
+                       @network-online-cursor
+                       user-uuid graph-uuid txid
+                       (graph-sync-off? graph-uuid)
+                       (user/logged-in?)
+                       (not (config/demo-graph? repo)))
+              (try
+                (when-let [sm (sync-manager-singleton current-user-uuid graph-uuid
+                                                      (config/get-repo-dir repo) repo
+                                                      txid *sync-state)]
+                  (when (check-graph-belong-to-current-user current-user-uuid user-uuid)
+                    (if-not (<! (<check-remote-graph-exists graph-uuid)) ; remote graph has been deleted
+                      (clear-graphs-txid! repo)
+                      (do
+                        (state/set-file-sync-state graph-uuid @*sync-state)
+                        (state/set-file-sync-manager graph-uuid sm)
+
+                        ;; update global state when *sync-state changes
+                        (add-watch *sync-state ::update-global-state
+                                   (fn [_ _ _ n]
+                                     (state/set-file-sync-state graph-uuid n)))
+
+                        (state/set-state! [:file-sync/graph-state :current-graph-uuid] graph-uuid)
+
+                        (.start sm)
+
+                        (offer! remote->local-full-sync-chan true)
+                        (offer! full-sync-chan true)))))
+                (catch :default e
+                  (prn "Sync start error: ")
+                  (log/error :exception e)))))
+          (reset! *sync-entered? false))))))
 
 (defn- restart-if-stopped!
   [is-active?]

src/main/frontend/handler/common/file.cljs

@@ -19,20 +19,20 @@
       (when (not= file current-file)
         current-file))))
 
-(defn- get-delete-blocks [repo-url first-page file]
-  (let [delete-blocks (->
-                       (concat
-                        (db/delete-file-blocks! repo-url file)
-                        (when first-page (db/delete-page-blocks repo-url (:block/name first-page))))
-                       (distinct))]
-    (when-let [current-file (page-exists-in-another-file repo-url first-page file)]
-      (when (not= file current-file)
-        (let [error (str "Page already exists with another file: " current-file ", current file: " file ". Please keep only one of them and re-index your graph.")]
-          (state/pub-event! [:notification/show
-                             {:content error
-                              :status :error
-                              :clear? false}]))))
-    delete-blocks))
+(defn- validate-existing-file
+  [repo-url file-page file-path]
+  (when-let [current-file (page-exists-in-another-file repo-url file-page file-path)]
+    (when (not= file-path current-file)
+      (let [error (str "Page already exists with another file: " current-file ", current file: " file-path ". Please keep only one of them and re-index your graph.")]
+        (state/pub-event! [:notification/show
+                           {:content error
+                            :status :error
+                            :clear? false}])))))
+
+(defn- validate-and-get-blocks-to-delete
+  [repo-url db file-page file-path retain-uuid-blocks]
+  (validate-existing-file repo-url file-page file-path)
+  (graph-parser/get-blocks-to-delete db file-page file-path retain-uuid-blocks))
 
 (defn reset-file!
   "Main fn for updating a db with the results of a parsed file"
@@ -62,7 +62,7 @@
          new? (nil? (db/entity [:file/path file]))
          options (merge (dissoc options :verbose)
                         {:new? new?
-                         :delete-blocks-fn (partial get-delete-blocks repo-url)
+                         :delete-blocks-fn (partial validate-and-get-blocks-to-delete repo-url)
                          :extract-options (merge
                                            {:user-config (state/get-config)
                                             :date-formatter (state/get-date-formatter)

src/main/frontend/handler/conversion.cljs

@@ -15,10 +15,10 @@
   (set-config! repo :file/name-format format))
 
 (defn- calc-current-name
-  "If the file body is parsed as the same page name, but the page name has a 
-   different file sanitization result under the current sanitization form, return 
+  "If the file body is parsed as the same page name, but the page name has a
+   different file sanitization result under the current sanitization form, return
    the new file name.
-   Return: 
+   Return:
      the file name for the page name under the current file naming rules, or `nil`
      if no change of path happens"
   [format file-body prop-title]
@@ -33,7 +33,7 @@
 
 (defn- calc-previous-name
   "We want to recover user's title back under new file name sanity rules.
-   Return: 
+   Return:
      the file name for that page name under the current file naming rules,
      and the new title if no action applied, or `nil` if no break change happens"
   [old-format new-format file-body]
@@ -72,7 +72,7 @@
   [old-format new-format file-body prop-title]
   ;; dont rename journal page. officially it's stored as `yyyy_mm_dd`
   ;; If it's a journal file imported with custom :journal/page-title-format,
-  ;;   and it includes reserved characters, format config change / file renaming is required. 
+  ;;   and it includes reserved characters, format config change / file renaming is required.
   ;;   It's about user's own data management decision and should be handled
   ;;   by user manually.
   ;; the 'expected' title of the user when updating from the previous format, or title will be broken in new format
@@ -88,7 +88,7 @@
       ret)))
 
 (defn calc-rename-target
-  "Return the renaming status and new file body to recover the original title of the file in previous version. 
+  "Return the renaming status and new file body to recover the original title of the file in previous version.
    The return title should be the same as the title in the index file in the previous version.
    return nil if no rename is needed.
    page: the page entity
@@ -96,6 +96,7 @@
    old-format, new-format: the filename formats
    Return:
      {:status        :informal | :breaking | :unreachable
+      :file-name original file name
       :target        the new file name
       :old-title     the old title
       :changed-title the new title} | nil"
@@ -113,6 +114,7 @@
            manual-prop-title?
            (fs-util/include-reserved-chars? file-body))
       {:status        :informal
+       :file-name     file-body
        :target        (fs-util/file-name-sanity file-body new-format)
        :old-title     prop-title
        :changed-title prop-title})))

src/main/frontend/handler/editor.cljs

@@ -17,13 +17,11 @@
             [frontend.handler.block :as block-handler]
             [frontend.handler.common :as common-handler]
             [frontend.handler.export :as export]
-            [frontend.handler.image :as image-handler]
             [frontend.handler.notification :as notification]
             [frontend.handler.repeated :as repeated]
             [frontend.handler.route :as route-handler]
             [frontend.handler.assets :as assets-handler]
             [frontend.idb :as idb]
-            [frontend.image :as image]
             [frontend.mobile.util :as mobile-util]
             [frontend.modules.outliner.core :as outliner-core]
             [frontend.modules.outliner.transaction :as outliner-tx]
@@ -1322,9 +1320,10 @@
 
 (defn get-asset-file-link
   [format url file-name image?]
-  (let [pdf? (and url (string/ends-with? (string/lower-case url) ".pdf"))]
+  (let [pdf? (and url (string/ends-with? (string/lower-case url) ".pdf"))
+        video? (and url (util/ext-of-video? url))]
     (case (keyword format)
-      :markdown (util/format (str (when (or image? pdf?) "!") "[%s](%s)") file-name url)
+      :markdown (util/format (str (when (or image? video? pdf?) "!") "[%s](%s)") file-name url)
       :org (if image?
              (util/format "[[%s]]" url)
              (util/format "[[%s][%s]]" url file-name))
@@ -1464,7 +1463,7 @@
   [id ^js files format uploading? drop-or-paste?]
   (let [repo (state/get-current-repo)
         block (state/get-edit-block)]
-    (if (config/local-db? repo)
+    (when (config/local-db? repo)
       (-> (save-assets! block repo (js->clj files))
           (p/then
            (fn [res]
@@ -1487,28 +1486,7 @@
             (fn []
               (reset! uploading? false)
               (reset! *asset-uploading? false)
-              (reset! *asset-uploading-process 0))))
-      (image/upload
-       files
-       (fn [file file-name file-type]
-         (image-handler/request-presigned-url
-          file file-name file-type
-          uploading?
-          (fn [signed-url]
-            (insert-command! id
-                             (get-asset-file-link format signed-url file-name true)
-                             format
-                             {:last-pattern (if drop-or-paste? "" (state/get-editor-command-trigger))
-                              :restore?     true})
-
-            (reset! *asset-uploading? false)
-            (reset! *asset-uploading-process 0))
-          (fn [e]
-            (let [process (* (/ (gobj/get e "loaded")
-                                (gobj/get e "total"))
-                             100)]
-              (reset! *asset-uploading? false)
-              (reset! *asset-uploading-process process)))))))))
+              (reset! *asset-uploading-process 0)))))))
 
 ;; Editor should track some useful information, like editor modes.
 ;; For example:

src/main/frontend/handler/events.cljs

@@ -88,7 +88,8 @@
           (state/set-state! :user/info result)
           (let [status (if (user-handler/alpha-or-beta-user?) :welcome :unavailable)]
             (when (and (= status :welcome) (user-handler/logged-in?))
-              (file-sync-handler/set-sync-enabled! true)
+              (when-not (false? (state/enable-sync?)) ; user turns it off
+                (file-sync-handler/set-sync-enabled! true))
               (async/<! (file-sync-handler/load-session-graphs))
               (p/let [repos (repo-handler/refresh-repos!)]
                 (when-let [repo (state/get-current-repo)]
@@ -362,10 +363,14 @@
         (state/pub-event! [:graph/dir-gone dir]))))
   ;; FIXME: an ugly implementation for redirecting to page on new window is restored
   (repo-handler/graph-ready! repo)
-  (when (and (util/electron?)
-             (not (config/demo-graph?))
-             (= :legacy (state/get-filename-format)))
-    (state/pub-event! [:ui/notify-outdated-filename-format []])))
+  (js/setTimeout
+   (fn []
+     (let [filename-format (state/get-filename-format repo)]
+       (when (and (util/electron?)
+                  (not (config/demo-graph?))
+                  (not= filename-format :triple-lowbar))
+         (state/pub-event! [:ui/notify-outdated-filename-format []]))))
+   3000))
 
 (defmethod handle :notification/show [[_ {:keys [content status clear?]}]]
   (notification/show! content status clear?))
@@ -460,7 +465,9 @@
       (when-let [left-sidebar-node (gdom/getElement "left-sidebar")]
         (set! (.. left-sidebar-node -style -bottom) "0px"))
       (when-let [right-sidebar-node (gdom/getElementByClass "sidebar-item-list")]
-        (set! (.. right-sidebar-node -style -paddingBottom) "150px")))))
+        (set! (.. right-sidebar-node -style -paddingBottom) "150px"))
+      (when-let [toolbar (.querySelector main-node "#mobile-editor-toolbar")]
+        (set! (.. toolbar -style -bottom) 0)))))
 
 (defn update-file-path [deprecated-repo current-repo deprecated-app-id current-app-id]
   (let [files (db-model/get-files-entity deprecated-repo)

src/main/frontend/handler/image.cljs

@@ -2,7 +2,6 @@
   (:require [clojure.string :as string]
             [frontend.config :as config]
             [frontend.fs :as fs]
-            [frontend.handler.notification :as notification]
             [frontend.image :as image]
             [frontend.state :as state]
             [frontend.util :as util]
@@ -51,46 +50,3 @@
                (js/console.dir error))))))
       (catch :default _e
         nil))))
-
-(defn request-presigned-url
-  [file filename mime-type uploading? url-handler on-processing]
-  (cond
-    (> (gobj/get file "size") (* 12 1024 1024))
-    (notification/show! [:p "Sorry, we don't support any file that's larger than 12MB."] :error)
-
-    :else
-    (do
-      (reset! uploading? true)
-      ;; start uploading?
-      (util/post (str config/api "presigned_url")
-                 {:filename filename
-                  :mime-type mime-type}
-                 (fn [{:keys [presigned-url s3-object-key] :as resp}]
-                   (if presigned-url
-                     (util/upload presigned-url
-                                  file
-                                  (fn [_result]
-                                    ;; request cdn signed url
-                                    (util/post (str config/api "signed_url")
-                                               {:s3-object-key s3-object-key}
-                                               (fn [{:keys [signed-url]}]
-                                                 (reset! uploading? false)
-                                                 (if signed-url
-                                                   (url-handler signed-url)
-                                                   (prn "Something error, can't get a valid signed url.")))
-                                               (fn [_error]
-                                                 (reset! uploading? false)
-                                                 (prn "Something error, can't get a valid signed url."))))
-                                  (fn [error]
-                                    (reset! uploading? false)
-                                    (prn "upload failed.")
-                                    (js/console.dir error))
-                                  (fn [e]
-                                    (on-processing e)))
-                     ;; TODO: notification, or re-try
-                     (do
-                       (reset! uploading? false)
-                       (prn "failed to get any presigned url, resp: " resp))))
-                 (fn [_error]
-                   ;; (prn "Get token failed, error: " error)
-                   (reset! uploading? false))))))

src/main/frontend/handler/search.cljs

@@ -44,12 +44,14 @@
                         (:db/id (db/entity repo [:block/name (util/page-name-sanity-lc page-db-id)]))
                         page-db-id)
            opts (if page-db-id (assoc opts :page (str page-db-id)) opts)]
-       (p/let [blocks (search/block-search repo q opts)]
+       (p/let [blocks (search/block-search repo q opts)
+               pages-content (search/page-content-search repo q opts)]
          (let [result (merge
                        {:blocks blocks
                         :has-more? (= limit (count blocks))}
                        (when-not page-db-id
-                         {:pages (search/page-search q)
+                         {:pages-content pages-content
+                          :pages (search/page-search q)
                           :files (search/file-search q)}))
                search-key (if more? :search/more-result :search/result)]
            (swap! state/state assoc search-key result)

src/main/frontend/image.cljs

@@ -1,8 +1,6 @@
 (ns frontend.image
   "Image related utility fns"
   (:require ["/frontend/exif" :as exif]
-            [clojure.string :as string]
-            [frontend.date :as date]
             [goog.object :as gobj]))
 
 (defn reverse?
@@ -70,19 +68,3 @@
   (.createObjectURL (or (.-URL js/window)
                         (.-webkitURL js/window))
                     file))
-
-;; (defn build-image
-;;   []
-;;   (let [img (js/Image.)]
-;;     ))
-
-(defn upload
-  [files file-handler & {:keys [files-limit]
-                         :or {files-limit 1}}]
-  (doseq [file (take files-limit (array-seq files))]
-    (let [file-type (gobj/get file "type")
-          ymd (->> (vals (date/year-month-day-padded))
-                   (string/join "_"))
-          file-name (str ymd "_" (gobj/get file "name"))]
-      (when (= 0 (.indexOf file-type "image/"))
-        (file-handler file file-name file-type)))))

src/main/frontend/modules/datascript_report/core.cljs

@@ -13,6 +13,8 @@
       nil)))
 
 (defn get-entity-from-db-after-or-before
+  "Get the entity from db after if possible; otherwise get entity from db before
+   Useful for fetching deleted elements"
   [db-before db-after db-id]
   (let [r (safe-pull db-after '[*] db-id)]
     (if (= keys-of-deleted-entity (count r))
@@ -21,6 +23,7 @@
       r)))
 
 (defn get-blocks-and-pages
+  "Calculate updated blocks and pages based on the db-before and db-after from tx-report"
   [{:keys [db-before db-after tx-data tx-meta]}]
   (let [updated-db-ids (-> (mapv first tx-data) (set))
         result (reduce
@@ -39,6 +42,7 @@
                 {:blocks #{}
                  :pages #{}}
                 updated-db-ids)
+        ;; updated pages logged in tx-meta (usually from move op)
         tx-meta-pages (->> [(:from-page tx-meta) (:target-page tx-meta)]
                            (remove nil?)
                            (map #(get-entity-from-db-after-or-before db-before db-after %))

src/main/frontend/search.cljs

@@ -14,7 +14,9 @@
             [frontend.util :as util]
             [frontend.util.property :as property]
             [goog.object :as gobj]
-            [promesa.core :as p]))
+            [promesa.core :as p]
+            [clojure.set :as set]
+            [frontend.modules.datascript-report.core :as db-report]))
 
 (defn get-engine
   [repo]
@@ -95,11 +97,24 @@
       (when-not (string/blank? q)
         (protocol/query engine q option)))))
 
+(defn page-content-search
+  [repo q option]
+  (when-let [engine (get-engine repo)]
+    (let [q (util/search-normalize q (state/enable-search-remove-accents?))
+          q (if (util/electron?) q (escape-str q))]
+      (when-not (string/blank? q)
+        (protocol/query-page engine q option)))))
+
 (defn- transact-blocks!
   [repo data]
   (when-let [engine (get-engine repo)]
     (protocol/transact-blocks! engine data)))
 
+(defn- transact-pages!
+  [repo data] 
+  (when-let [engine (get-engine repo)]
+    (protocol/transact-pages! engine data)))
+
 (defn exact-matched?
   "Check if two strings points toward same search result"
   [q match]
@@ -124,7 +139,7 @@
            q (clean-str q)]
        (when-not (string/blank? q)
          (let [indice (or (get-in @indices [repo :pages])
-                          (search-db/make-pages-indice!))
+                          (search-db/make-pages-title-indice!))
                result (->> (.search indice q (clj->js {:limit limit}))
                            (bean/->clj))]
            ;; TODO: add indexes for highlights
@@ -191,8 +206,48 @@
            (let [result (fuzzy-search result q :limit limit)]
              (vec result))))))))
 
-(defn sync-search-indice!
-  [repo tx-report]
+(defn- get-pages-from-datoms-impl
+  [pages]
+  (let [pages-result (db/pull-many '[:db/id :block/name :block/original-name] (set (map :e pages)))
+        pages-to-add-set (->> (filter :added pages)
+                              (map :e)
+                              (set))
+        pages-to-add (->> (filter (fn [page]
+                                    (contains? pages-to-add-set (:db/id page))) pages-result)
+                          (map (fn [p] (or (:block/original-name p)
+                                           (:block/name p))))
+                          (map search-db/original-page-name->index))
+        pages-to-remove-set (->> (remove :added pages)
+                                 (map :v))
+        pages-to-remove-id-set (->> (remove :added pages)
+                                    (map :e)
+                                    set)]
+    {:pages-to-add        pages-to-add
+     :pages-to-remove-set pages-to-remove-set
+     :pages-to-add-id-set pages-to-add-set
+     :pages-to-remove-id-set pages-to-remove-id-set}))
+
+(defn- get-blocks-from-datoms-impl
+  [blocks]
+  (when (seq blocks)
+    (let [blocks-result (->> (db/pull-many '[:db/id :block/uuid :block/format :block/content :block/page] (set (map :e blocks)))
+                             (map (fn [b] (assoc b :block/page (get-in b [:block/page :db/id])))))
+          blocks-to-add-set (->> (filter :added blocks)
+                                 (map :e)
+                                 (set))
+          blocks-to-add (->> (filter (fn [block]
+                                       (contains? blocks-to-add-set (:db/id block)))
+                                     blocks-result)
+                             (map search-db/block->index)
+                             (remove nil?))
+          blocks-to-remove-set (->> (remove :added blocks)
+                                    (map :e)
+                                    (set))]
+      {:blocks-to-remove-set blocks-to-remove-set
+       :blocks-to-add        blocks-to-add})))
+
+(defn- get-direct-blocks-and-pages 
+  [tx-report]
   (let [data (:tx-data tx-report)
         datoms (filter
                 (fn [datom]
@@ -200,50 +255,78 @@
                 data)]
     (when (seq datoms)
       (let [datoms (group-by :a datoms)
-            pages (:block/name datoms)
-            blocks (:block/content datoms)]
-        (when (seq pages)
-          (let [pages-result (db/pull-many '[:db/id :block/name :block/original-name] (set (map :e pages)))
-                pages-to-add-set (->> (filter :added pages)
-                                      (map :e)
-                                      (set))
-                pages-to-add (->> (filter (fn [page]
-                                            (contains? pages-to-add-set (:db/id page))) pages-result)
-                                  (map (fn [p] (or (:block/original-name p)
-                                                   (:block/name p))))
-                                  (map search-db/original-page-name->index))
-                pages-to-remove-set (->> (remove :added pages)
-                                         (map :v))]
-            (swap! search-db/indices update-in [repo :pages]
-                   (fn [indice]
-                     (when indice
-                       (doseq [page-name pages-to-remove-set]
-                         (.remove indice
-                                  (fn [page]
-                                    (= (util/safe-page-name-sanity-lc page-name)
-                                       (util/safe-page-name-sanity-lc (gobj/get page "original-name"))))))
-                       (when (seq pages-to-add)
-                         (doseq [page pages-to-add]
-                           (.add indice (bean/->js page)))))
-                     indice))))
+            blocks (:block/content datoms)
+            pages (:block/name datoms)]
+        (merge (get-blocks-from-datoms-impl blocks)
+               (get-pages-from-datoms-impl pages))))))
+
+(defn- get-indirect-pages
+  "Return the set of pages that will have content updated"
+  [tx-report]
+  (let [data   (:tx-data tx-report)
+        datoms (filter
+                (fn [datom]
+                  (and (:added datom)
+                       (contains? #{:file/content} (:a datom))))
+                data)]
+    (when (seq datoms)
+      (->> datoms
+           (mapv (fn [datom]
+                   (let [tar-db  (:db-after tx-report)]
+                     ;; Reverse query the corresponding page id of the modified `:file/content`)
+                     (when-let [page-id (->> (:e datom)
+                                             (db-report/safe-pull tar-db '[:block/_file])
+                                             (:block/_file)
+                                             (first)
+                                             (:db/id))]
+                       ;; Fetch page entity according to what page->index requested
+                       (db-report/safe-pull tar-db '[:db/id :block/uuid
+                                                     :block/original-name
+                                                     {:block/file [:file/content]}]
+                                            page-id)))))
+           (remove nil?)))))
+
+;; TODO merge with logic in `invoke-hooks` when feature and test is sufficient
+(defn sync-search-indice!
+  [repo tx-report]
+  (let [{:keys [pages-to-add pages-to-remove-set pages-to-remove-id-set
+                blocks-to-add blocks-to-remove-set]} (get-direct-blocks-and-pages tx-report) ;; directly modified block & pages
+        updated-pages (get-indirect-pages tx-report)]
+    ;; update page title indice
+    (when (or (seq pages-to-add) (seq pages-to-remove-set))
+      (swap! search-db/indices update-in [repo :pages]
+             (fn [indice]
+               (when indice
+                 (doseq [page-name pages-to-remove-set]
+                   (.remove indice
+                            (fn [page]
+                              (= (util/safe-page-name-sanity-lc page-name)
+                                 (util/safe-page-name-sanity-lc (gobj/get page "original-name"))))))
+                 (when (seq pages-to-add)
+                   (doseq [page pages-to-add]
+                     (.add indice (bean/->js page)))))
+               indice)))
+
+    ;; update block indice
+    (when (or (seq blocks-to-add) (seq blocks-to-remove-set))
+      (transact-blocks! repo
+                        {:blocks-to-remove-set blocks-to-remove-set
+                         :blocks-to-add        blocks-to-add}))
 
-        (when (seq blocks)
-          (let [blocks-result (->> (db/pull-many '[:db/id :block/uuid :block/format :block/content :block/page] (set (map :e blocks)))
-                                   (map (fn [b] (assoc b :block/page (get-in b [:block/page :db/id])))))
-                blocks-to-add-set (->> (filter :added blocks)
-                                       (map :e)
-                                       (set))
-                blocks-to-add (->> (filter (fn [block]
-                                             (contains? blocks-to-add-set (:db/id block)))
-                                           blocks-result)
-                                   (map search-db/block->index)
-                                   (remove nil?))
-                blocks-to-remove-set (->> (remove :added blocks)
-                                          (map :e)
-                                          (set))]
-            (transact-blocks! repo
-                              {:blocks-to-remove-set blocks-to-remove-set
-                               :blocks-to-add blocks-to-add})))))))
+    ;; update page indice
+    (when (or (seq pages-to-remove-id-set) (seq updated-pages)) ;; when move op happens, no :block/content provided
+      (let [indice-pages   (map search-db/page->index updated-pages)
+            invalid-set    (->> (map (fn [updated indiced] ;; get id of pages without valid page index
+                                       (if indiced nil (:db/id updated)))
+                                     updated-pages indice-pages)
+                                (remove nil?)
+                                set)
+            pages-to-add   (->> indice-pages
+                                (remove nil?)
+                                set)
+            pages-to-remove-set (set/union pages-to-remove-id-set invalid-set)]
+        (transact-pages! repo {:pages-to-remove-set pages-to-remove-set
+                               :pages-to-add        pages-to-add})))))
 
 (defn rebuild-indices!
   ([]
@@ -251,10 +334,10 @@
   ([repo]
    (when repo
      (when-let [engine (get-engine repo)]
-       (let [pages (search-db/make-pages-indice!)]
+       (let [page-titles (search-db/make-pages-title-indice!)]
          (p/let [blocks (protocol/rebuild-blocks-indice! engine)]
-           (let [result {:pages pages
-                         :blocks blocks}]
+           (let [result {:pages         page-titles ;; TODO: rename key to :page-titles
+                         :blocks        blocks}]
              (swap! indices assoc repo result)
              indices)))))))
 

src/main/frontend/search/agency.cljs

@@ -31,6 +31,13 @@
         (protocol/query e q opts))
       (protocol/query e1 q opts)))
 
+  (query-page [_this q opts]
+    (println "D:Search > Query-page contents:" repo q opts)
+    (let [[e1 e2] (get-registered-engines repo)]
+      (doseq [e e2]
+        (protocol/query-page e q opts))
+      (protocol/query-page e1 q opts)))
+
   (rebuild-blocks-indice! [_this]
     (println "D:Search > Initial blocks indice!:" repo)
     (let [[e1 e2] (get-registered-engines repo)]
@@ -43,6 +50,11 @@
     (doseq [e (get-flatten-registered-engines repo)]
       (protocol/transact-blocks! e data)))
 
+  (transact-pages! [_this data]
+    (println "D:Search > Transact pages!:" repo)
+    (doseq [e (get-flatten-registered-engines repo)]
+      (protocol/transact-pages! e data)))
+
   (truncate-blocks! [_this]
     (println "D:Search > Truncate blocks!" repo)
     (doseq [e (get-flatten-registered-engines repo)]

src/main/frontend/search/browser.cljs

@@ -35,6 +35,7 @@
   protocol/Engine
   (query [_this q option]
     (p/promise (search-blocks repo q option)))
+  (query-page [_this _q _opt] nil) ;; Page index is not available with fuse.js until sufficient performance benchmarking
   (rebuild-blocks-indice! [_this]
     (let [indice (search-db/make-blocks-indice! repo)]
       (p/promise indice)))
@@ -51,6 +52,7 @@
                  (doseq [block blocks-to-add]
                    (.add indice (bean/->js block)))))
              indice)))
+  (transact-pages! [_this _data] nil) ;; Page index is not available with fuse.js until sufficient performance benchmarking
   (truncate-blocks! [_this]
     (swap! indices assoc-in [repo :blocks] nil))
   (remove-db! [_this]

src/main/frontend/search/db.cljs

@@ -10,15 +10,33 @@
 
 (defonce indices (atom nil))
 
+(defn- sanitize
+  [content]
+  (util/search-normalize content (state/enable-search-remove-accents?)))
+
+(defn- max-len
+  []
+  (state/block-content-max-length (state/get-current-repo)))
+
 (defn block->index
   "Convert a block to the index for searching"
   [{:block/keys [uuid page content] :as block}]
-  (when-let [content (util/search-normalize content (state/enable-search-remove-accents?))]
-    (when-not (> (count content) (state/block-content-max-length (state/get-current-repo)))
-      {:id (:db/id block)
+  (when-not (> (count content) (max-len))
+    {:id (:db/id block)
+     :uuid (str uuid)
+     :page page
+     :content (sanitize content)}))
+
+(defn page->index
+  "Convert a page name to the index for searching (page content level)
+   Generate index based on the DB content AT THE POINT OF TIME"
+  [{:block/keys [uuid _original-name] :as page}]
+  (when-let [content (some-> (:block/file page)
+                             (:file/content))]
+    (when-not (> (count content) (* (max-len) 10))
+      {:id   (:db/id page)
        :uuid (str uuid)
-       :page page
-       :content content})))
+       :content (sanitize content)})))
 
 (defn build-blocks-indice
   ;; TODO: Remove repo effects fns further up the call stack. db fns need standardization on taking connection
@@ -29,6 +47,14 @@
        (remove nil?)
        (bean/->js)))
 
+(defn build-pages-indice 
+  [repo]
+  (->> (db/get-all-pages repo)
+       (map #(db/entity (:db/id %))) ;; get full file-content
+       (map page->index)
+       (remove nil?)
+       (bean/->js)))
+
 (defn make-blocks-indice!
   [repo]
   (let [blocks (build-blocks-indice repo)
@@ -46,9 +72,11 @@
   [p] {:name (util/search-normalize p (state/enable-search-remove-accents?))
        :original-name p})
 
-(defn make-pages-indice!
-  "Build a page indice from scratch.
-   Incremental page indice is implemented in frontend.search.sync-search-indice!"
+(defn make-pages-title-indice!
+  "Build a page title indice from scratch.
+   Incremental page title indice is implemented in frontend.search.sync-search-indice!
+   Rename from the page indice since 10.25.2022, since this is only used for page title search.
+   From now on, page indice is talking about page content search."
   []
   (when-let [repo (state/get-current-repo)]
     (let [pages (->> (db/get-pages (state/get-current-repo))

src/main/frontend/search/node.cljs

@@ -17,12 +17,22 @@
                 {:block/uuid uuid
                  :block/content content
                  :block/page page})) result)))
+  (query-page [_this q opts]
+    (p/let [result (ipc/ipc "search-pages" repo q opts)
+            result (bean/->clj result)]
+      (keep (fn [{:keys [content snippet uuid]}]
+              (when-not (> (count content) (* 10 (state/block-content-max-length repo)))
+                {:block/uuid uuid
+                 :block/snippet snippet})) result)))
   (rebuild-blocks-indice! [_this]
-    (let [indice (search-db/build-blocks-indice repo)]
-      (ipc/ipc "rebuild-blocks-indice" repo indice)))
+    (let [blocks-indice (search-db/build-blocks-indice repo)
+          pages-indice  (search-db/build-pages-indice repo)]
+      (ipc/ipc "rebuild-indice" repo blocks-indice pages-indice)))
   (transact-blocks! [_this data]
     (ipc/ipc "transact-blocks" repo (bean/->js data)))
   (truncate-blocks! [_this]
-    (ipc/ipc "truncate-blocks" repo))
+    (ipc/ipc "truncate-indice" repo))
+  (transact-pages! [_this data]
+    (ipc/ipc "transact-pages" repo (bean/->js data)))
   (remove-db! [_this]
     (ipc/ipc "remove-db" repo)))

src/main/frontend/search/plugin.cljs

@@ -23,6 +23,9 @@
   (query [_this q opts]
     (call-service! service "search:query" (merge {:q q} opts) true))
 
+  (query-page [_this q opts]
+    (call-service! service "search:queryPage" (merge {:q q} opts) true))
+
   (rebuild-blocks-indice! [_this]
    ;; Not pushing all data for performance temporarily
    ;;(let [blocks (search-db/build-blocks-indice repo)])
@@ -34,6 +37,12 @@
                      {:data {:added   blocks-to-add
                              :removed blocks-to-remove-set}})))
 
+  (transact-pages! [_this data]
+    (let [{:keys [pages-to-remove-set pages-to-add]} data]
+      (call-service! service "search:transactpages"
+                     {:data {:added   pages-to-add
+                             :removed pages-to-remove-set}})))
+
   (truncate-blocks! [_this]
     (call-service! service "search:truncateBlocks" {}))
 

src/main/frontend/search/protocol.cljs

@@ -1,8 +1,10 @@
 (ns ^:no-doc frontend.search.protocol)
 
 (defprotocol Engine
-  (query [this q option])
-  (rebuild-blocks-indice! [this])
+  (query [this q option]) 
+  (query-page [this q option])
+  (rebuild-blocks-indice! [this]) ;; TODO: rename to rebuild-indice!
   (transact-blocks! [this data])
-  (truncate-blocks! [this])
+  (truncate-blocks! [this]) ;; TODO: rename to truncate-indice!
+  (transact-pages! [this data])
   (remove-db! [this]))

src/main/frontend/state.cljs

@@ -51,7 +51,7 @@
      :journals-length                       3
 
      :search/q                              ""
-     :search/mode                           :global
+     :search/mode                           :global  ;; inner page or full graph? {:page :global}
      :search/result                         nil
      :search/graph-filters                  []
      :search/engines                        {}
@@ -69,7 +69,7 @@
 
      ;; ui
      :ui/viewport                           {}
-     
+
      ;; left sidebar
      :ui/navigation-item-collapsed?         {}
 
@@ -310,6 +310,9 @@
    :default-arweave-gateway "https://arweave.net"
 
    ;; For flushing the settings of old versions. Don't bump this value.
+   ;; There are only two kinds of graph, one is not upgraded (:legacy) and one is upgraded (:triple-lowbar)
+   ;; For not upgraded graphs, the config will have no key `:file/name-format`
+   ;; Then the default value is applied
    :file/name-format :legacy})
 
 ;; State that most user config is dependent on

src/main/frontend/util.cljc

@@ -201,6 +201,11 @@
              (string/ends-with? %))
         [".png" ".jpg" ".jpeg" ".bmp" ".gif" ".webp" ".svg"]))
 
+(defn ext-of-video? [s]
+  (some #(-> (string/lower-case s)
+             (string/ends-with? %))
+        [".mp4" ".mkv" ".mov" ".wmv" ".avi" ".webm" ".mpg" ".ts" ".ogg" ".flv"]))
+
 ;; ".lg:absolute.lg:inset-y-0.lg:right-0.lg:w-1/2"
 (defn hiccup->class
   [class]
@@ -223,29 +228,6 @@
                            (.then #(on-ok %)))
                        (on-failed resp)))))))))
 
-#?(:cljs
-   (defn upload
-     [url file on-ok on-failed on-progress]
-     (let [xhr (js/XMLHttpRequest.)]
-       (.open xhr "put" url)
-       (gobj/set xhr "onload" on-ok)
-       (gobj/set xhr "onerror" on-failed)
-       (when (and (gobj/get xhr "upload")
-                  on-progress)
-         (gobj/set (gobj/get xhr "upload")
-                   "onprogress"
-                   on-progress))
-       (.send xhr file))))
-
-#?(:cljs
-   (defn post
-     [url body on-ok on-failed]
-     (fetch url {:method "post"
-                 :headers {:Content-Type "application/json"}
-                 :body (js/JSON.stringify (clj->js body))}
-            on-ok
-            on-failed)))
-
 (defn zero-pad
   [n]
   (if (< n 10)

src/main/frontend/util/text.cljs

@@ -118,6 +118,26 @@
              []
              ks))))
 
+(defn cut-by
+  "Cut string by specifid wrapping symbols, only match the first occurrence.
+     value - string to cut
+     before - cutting symbol (before)
+     end - cutting symbol (end)"
+  [value before end]
+  (let [b-pos (string/index-of value before)
+        b-len (count before)]
+    (if b-pos
+      (let [b-cut (subs value 0 b-pos)
+            m-cut (subs value (+ b-pos b-len))
+            e-len (count end)
+            e-pos (string/index-of m-cut end)]
+        (if e-pos
+          (let [e-cut (subs m-cut (+ e-pos e-len))
+                m-cut (subs m-cut 0 e-pos)]
+            [b-cut m-cut e-cut])
+          [b-cut m-cut nil]))
+      [value nil nil])))
+
 (defn get-graph-name-from-path
   [path]
   (when (string? path)

src/test/frontend/db/model_test.cljs

@@ -121,7 +121,4 @@
          (#'model/get-unnecessary-namespaces-name '("one/two/tree" "one" "one/two" "non nested tag" "non nested link")))
       "Must be  one/two one"))
 
-
-
-
 #_(cljs.test/test-ns 'frontend.db.model-test)

src/test/frontend/extensions/zotero/extractor_test.cljs

@@ -43,7 +43,6 @@
         (is (= 8 authors)))
 
       (testing "tags"
-        (prn (-> properties :tags))
         ;; tags split by `,` are counted into different tags
         ;; https://github.com/logseq/logseq/commit/435c2110bcc2d30ed743ba31375450f1a705b00b
         (is (= 20 tags)))))

src/test/frontend/handler/repo_test.cljs

@@ -1,9 +1,11 @@
 (ns frontend.handler.repo-test
-  (:require [cljs.test :refer [deftest use-fixtures]]
+  (:require [cljs.test :refer [deftest use-fixtures testing is]]
             [frontend.handler.repo :as repo-handler]
-            [frontend.test.helper :as test-helper]
+            [frontend.test.helper :as test-helper :refer [load-test-files]]
             [logseq.graph-parser.cli :as gp-cli]
             [logseq.graph-parser.test.docs-graph-helper :as docs-graph-helper]
+            [logseq.graph-parser.util.block-ref :as block-ref]
+            [frontend.db.model :as model]
             [frontend.db.conn :as conn]))
 
 (use-fixtures :each {:before test-helper/start-test-db!
@@ -19,3 +21,44 @@
         db (conn/get-db test-helper/test-db)]
 
     (docs-graph-helper/docs-graph-assertions db (map :file/path files))))
+
+(deftest parse-files-and-load-to-db-with-block-refs-on-reload
+  (testing "Refs to blocks on a page are retained if that page is reloaded"
+    (let [test-uuid "16c90195-6a03-4b3f-839d-095a496d9acd"
+          target-page-content (str "- target block\n  id:: " test-uuid)
+          referring-page-content (str "- " (block-ref/->block-ref test-uuid))]
+      (load-test-files [{:file/path "pages/target.md"
+                         :file/content target-page-content}
+                        {:file/path "pages/referrer.md"
+                         :file/content referring-page-content}])
+      (is (= [(parse-uuid test-uuid)] (model/get-all-referenced-blocks-uuid)))
+
+      (load-test-files [{:file/path "pages/target.md"
+                         :file/content target-page-content}])
+      (is (= [(parse-uuid test-uuid)] (model/get-all-referenced-blocks-uuid))))))
+
+(deftest parse-files-and-load-to-db-with-page-rename
+  (testing
+    "Reload a file when the disk contents result in the file having a new page name"
+    (let [test-uuid "16c90195-6a03-4b3f-839d-095a496d9efc"
+          target-page-content (str "- target block\n  id:: " test-uuid)
+          referring-page-content (str "- " (block-ref/->block-ref test-uuid))
+          update-referring-page-content (str "title:: updatedPage\n- " (block-ref/->block-ref test-uuid))
+          get-page-block-count (fn [page-name]
+                                 (let [page-id (:db/id (model/get-page page-name))]
+                                   (if (some? page-id)
+                                     (model/get-page-blocks-count test-helper/test-db page-id)
+                                     0)))]
+      (load-test-files [{:file/path "pages/target.md"
+                         :file/content target-page-content}
+                        {:file/path "pages/referrer.md"
+                         :file/content referring-page-content}])
+      (is (= [(parse-uuid test-uuid)] (model/get-all-referenced-blocks-uuid)))
+      (is (= 1 (get-page-block-count "referrer")))
+      (is (= 0 (get-page-block-count "updatedPage")))
+
+      (load-test-files [{:file/path "pages/referrer.md"
+                         :file/content update-referring-page-content}])
+      (is (= [(parse-uuid test-uuid)] (model/get-all-referenced-blocks-uuid)))
+      (is (= 0 (get-page-block-count "referrer")))
+      (is (= 2 (get-page-block-count "updatedPage"))))))

src/test/frontend/util/text_test.cljs

@@ -57,3 +57,37 @@
     '(false false false false false false true true true true true true)
     (map #(text-util/wrapped-by? "prop::value" % "::" "") (take 12 (range)))
     ))
+
+
+(deftest test-cut-by
+  []
+  (are [x y] (= x y)
+    ["" "" ""]
+    (text-util/cut-by "[[]]" "[[" "]]")
+
+    ["" "abc" ""]
+    (text-util/cut-by "[[abc]]" "[[" "]]")
+
+    ["012 " "6" " [[2]]"]
+    (text-util/cut-by "012 [[6]] [[2]]" "[[" "]]")
+
+    ["" "prop" "value"]
+    (text-util/cut-by "prop::value" "" "::")
+
+    ["prop" "" "value"]
+    (text-util/cut-by "prop::value" "::" "")
+
+    ["some " "content" " here"]
+    (text-util/cut-by "some $pfts>$content$pfts<$ here" "$pfts>$" "$pfts<$")
+
+    ["some " "content$pft" nil]
+    (text-util/cut-by "some $pfts>$content$pft" "$pfts>$" "$pfts<$")
+
+    ["some $pf" nil nil]
+    (text-util/cut-by "some $pf" "$pfts>$" "$pfts<$")
+
+    ["" "content" ""]
+    (text-util/cut-by "$pfts>$content$pfts<$" "$pfts>$" "$pfts<$")
+    
+    ["" "content$p" nil]
+    (text-util/cut-by "$pfts>$content$p" "$pfts>$" "$pfts<$")))

templates/config.edn

@@ -295,8 +295,5 @@
  ;;   :file/name-format :triple-lowbar
  ;;     ;use triple underscore `___` for slash `/` in page title
  ;;     ;use Percent-encoding for other invalid characters
- ;;   :file/name-format :legacy
- ;;     ;use Percent-encoding for slash and other invalid characters
- ;;     ;parse `.` in file name as slash `/` in page title
  :file/name-format :triple-lowbar
  }