Merge branch 'feat/db' into refactor/pipeline-worker

.carve/ignore

@@ -94,7 +94,5 @@ frontend.ui/_emoji-init-data
 frontend.worker.rtc.op-mem-layer/_sync-loop
 ;; Used by shadow.cljs
 frontend.db-worker/init
-;; For defonce
-frontend.persist-db.browser/_do_not_reload_worker
 ;; WIP fn, remove when it's ready
-frontend.worker.rtc.asset-sync/<loop-for-assets-sync
+frontend.worker.rtc.asset-sync/<loop-for-assets-sync

deps/graph-parser/src/logseq/graph_parser.cljs

@@ -8,8 +8,7 @@
             [logseq.graph-parser.extract :as extract]
             [logseq.common.util :as common-util]
             [logseq.common.config :as common-config]
-            [logseq.db :as ldb]
-            [logseq.db.sqlite.util :as sqlite-util]))
+            [logseq.db :as ldb]))
 
 (defn- retract-blocks-tx
   [blocks retain-uuids]
@@ -124,13 +123,24 @@ Options available:
   [db property-name]
   (:block/uuid (d/entity db [:block/name (common-util/page-name-sanity-lc (name property-name))])))
 
+(defn add-missing-timestamps
+  "Add updated-at or created-at timestamps if they doesn't exist"
+  [block]
+  (let [updated-at (date-time-util/time-ms)
+        block (cond-> block
+                (nil? (:block/updated-at block))
+                (assoc :block/updated-at updated-at)
+                (nil? (:block/created-at block))
+                (assoc :block/created-at updated-at))]
+    block))
+
 (defn- update-block-with-invalid-tags
   [block]
   (if (seq (:block/tags block))
     (update block :block/tags
             (fn [tags]
               (mapv #(-> %
-                         sqlite-util/block-with-timestamps
+                         add-missing-timestamps
                          (merge {:block/journal? false
                                  :block/format :markdown
                                  :block/uuid (d/squuid)}))
@@ -174,7 +184,7 @@ Options available:
                      (fn [refs]
                        (mapv #(assoc % :block/format :markdown) refs)))
              block')))
-        sqlite-util/block-with-timestamps
+        add-missing-timestamps
         ;; FIXME: Remove when properties are supported
         (assoc :block/properties {})
         ;; TODO: org-mode content needs to be handled
@@ -226,7 +236,7 @@ Options available:
                                  (if-not (:block/original-name m)
                                    (assoc m :block/original-name (:block/name m))
                                    m)))
-                              sqlite-util/block-with-timestamps
+                              add-missing-timestamps
                               ;; TODO: org-mode content needs to be handled
                               (assoc :block/format :markdown)
                               (dissoc :block/properties-text-values :block/properties-order :block/invalid-properties

src/main/frontend/components/editor.cljs

@@ -642,10 +642,10 @@
             :visibility "hidden"
             :top 0
             :left 0}}
-   (let [content (str content "0")]
-     (for [[idx c] (map-indexed
-                    vector
-                    (string/split content ""))]
+   (let [content (str content "0")
+         graphemes (util/split-graphemes content)
+         graphemes-char-index (reductions #(+ %1 (count %2)) 0 graphemes)]
+     (for [[idx c] (zipmap graphemes-char-index graphemes)]
        (if (= c "\n")
          [:span {:id (str "mock-text_" idx)
                  :key idx} "0" [:br]]

src/main/frontend/components/imports.cljs

@@ -28,7 +28,8 @@
             [promesa.core :as p]
             [borkdude.rewrite-edn :as rewrite]
             [rum.core :as rum]
-            [frontend.handler.repo :as repo-handler]))
+            [frontend.handler.repo :as repo-handler]
+            [frontend.handler.common.config-edn :as config-edn-common-handler]))
 
 ;; Can't name this component as `frontend.components.import` since shadow-cljs
 ;; will complain about it.
@@ -211,7 +212,7 @@
       (p/then (fn [content]
                 (let [migrated-content (-> (reduce rewrite/dissoc
                                                    (rewrite/parse-string (str content))
-                                                   [:preferred-format :property/separated-by-commas])
+                                                   (keys config-edn-common-handler/file-only-config))
                                            str)]
                   (p/do!
                    (db-editor-handler/save-file! "logseq/config.edn" migrated-content))

src/main/frontend/handler/common/config_edn.cljs

@@ -86,6 +86,29 @@ nested keys or positional errors e.g. tuples"
       :else
       (validate-config-map parsed-body schema path))))
 
+(def file-only-config
+  "File only config that is deprecated in DB graphs"
+  {:preferred-format
+   "is not used in DB graphs as there is only markdown mode."
+   :property/separated-by-commas
+   "is not used in DB graphs"
+   :property-pages/enabled?
+   "is not used in DB graphs as all properties have pages"
+   :property-pages/excludelist
+   "is not used in DB graphs"
+   :hidden
+   "is not used in DB graphs"
+   :org-mode/insert-file-link?
+   "is not used in DB graphs"
+   :block-hidden-properties
+   "is not used in DB graphs as hiding a property is done in its configuration"
+   :ignored-page-references-keywords
+   "is not used in DB graphs"
+   :file/name-format
+   "is not used in DB graphs"
+   :feature/enable-block-timestamps?
+   "is not used in DB graphs as it is always enabled"})
+
 (defn detect-deprecations
   "Detects config keys that will or have been deprecated"
   [path content {:keys [db-graph?]}]
@@ -96,10 +119,7 @@ nested keys or positional errors e.g. tuples"
                    "is no longer supported. Please use '/' and report bugs on it."}
                    db-graph?
                    (merge
-                    {:preferred-format
-                     "is not used in DB graphs as there is only markdown mode."
-                     :property/separated-by-commas
-                     "is not used in DB graphs"}))]
+                    file-only-config))]
     (cond
       (= body ::failed-to-detect)
       (log/info :msg "Skip deprecation check since config is not valid edn")

src/main/frontend/handler/editor.cljs

@@ -2596,7 +2596,7 @@
 (defn- move-cross-boundary-up-down
   [direction move-opts]
   (let [input (state/get-input)
-        line-pos (util/get-first-or-last-line-pos input)
+        line-pos (util/get-line-pos (.-value input) (util/get-selection-start input))
         repo (state/get-current-repo)
         f (case direction
             :up util/get-prev-block-non-collapsed

src/main/frontend/util.cljc

@@ -403,12 +403,49 @@
   (when input
     (.-selectionDirection input)))
 
-(defn get-first-or-last-line-pos
-  [input]
-  (let [pos (get-selection-start input)
-        value (.-value input)
-        last-newline-pos (or (string/last-index-of value \newline (dec pos)) -1)]
-    (- pos last-newline-pos 1)))
+#?(:cljs
+   (defn split-graphemes
+     [s]
+     (let [^js splitter (GraphemeSplitter.)]
+       (.splitGraphemes splitter s))))
+
+#?(:cljs
+   (defn get-graphemes-pos
+     "Return the length of the substrings in s between start and from-index.
+
+      multi-char count as 1, like emoji characters"
+     [s from-index]
+     (let [^js splitter (GraphemeSplitter.)]
+       (.countGraphemes splitter (subs s 0 from-index)))))
+
+#?(:cljs
+   (defn get-line-pos
+     "Return the length of the substrings in s between the last index of newline
+      in s searching backward from from-newline-index and from-newline-index.
+
+      multi-char count as 1, like emoji characters"
+     [s from-newline-index]
+     (let [^js splitter (GraphemeSplitter.)
+           last-newline-pos (string/last-index-of s \newline (dec from-newline-index))
+           before-last-newline-length (or last-newline-pos -1)
+           last-newline-content (subs s (inc before-last-newline-length) from-newline-index)]
+       (.countGraphemes splitter last-newline-content))))
+
+#?(:cljs
+   (defn get-text-range
+     "Return the substring of the first grapheme-num characters of s if first-line? is true,
+      otherwise return the substring of s before the last \n and the first grapheme-num characters.
+
+      grapheme-num treats multi-char as 1, like emoji characters"
+     [s grapheme-num first-line?]
+     (let [newline-pos (if first-line?
+                         0
+                         (inc (or (string/last-index-of s \newline) -1)))
+           ^js splitter (GraphemeSplitter.)
+           ^js newline-graphemes (.splitGraphemes splitter (subs s newline-pos))
+           ^js newline-graphemes (.slice newline-graphemes 0 grapheme-num)
+           content (.join newline-graphemes "")]
+       (subs s 0 (+ newline-pos (count content))))))
 
 #?(:cljs
    (defn stop [e]
@@ -666,7 +703,7 @@
    (defn safe-dec-current-pos-from-end
      [input current-pos]
      (if-let [len (and (string? input) (.-length input))]
-       (when-let [input (and (>= len 2) (<= current-pos len)
+       (if-let [input (and (>= len 2) (<= current-pos len)
                              (.substring input (max (- current-pos 20) 0) current-pos))]
          (try
            (let [^js splitter (GraphemeSplitter.)
@@ -674,7 +711,8 @@
              (- current-pos (.-length (.pop input))))
            (catch :default e
              (js/console.error e)
-             (dec current-pos))))
+             (dec current-pos)))
+         (dec current-pos))
        (dec current-pos))))
 
 #?(:cljs
@@ -682,7 +720,7 @@
    (defn safe-inc-current-pos-from-start
      [input current-pos]
      (if-let [len (and (string? input) (.-length input))]
-       (when-let [input (and (>= len 2) (<= current-pos len)
+       (if-let [input (and (>= len 2) (<= current-pos len)
                              (.substr input current-pos 20))]
          (try
            (let [^js splitter (GraphemeSplitter.)
@@ -690,7 +728,8 @@
              (+ current-pos (.-length (.shift input))))
            (catch :default e
              (js/console.error e)
-             (inc current-pos))))
+             (inc current-pos)))
+         (inc current-pos))
        (inc current-pos))))
 
 #?(:cljs
@@ -1254,10 +1293,11 @@
              scroll-top  (.-scrollTop main-node)
 
              current-pos (get-selection-start el)
+             grapheme-pos (get-graphemes-pos (.-value (.textContent el)) current-pos)
              mock-text   (some-> (gdom/getElement "mock-text")
                                  gdom/getChildren
                                  array-seq
-                                 (nth-safe current-pos))
+                                 (nth-safe grapheme-pos))
              offset-top   (and mock-text (.-offsetTop mock-text))
              offset-height (and mock-text (.-offsetHeight mock-text))
 

src/main/frontend/util/cursor.cljs

@@ -31,12 +31,13 @@
   ([input] (get-caret-pos input (util/get-selection-start input)))
   ([input pos]
    (when input
-     (let [rect (bean/->clj (.. input (getBoundingClientRect) (toJSON)))]
+     (let [rect (bean/->clj (.. input (getBoundingClientRect) (toJSON)))
+           grapheme-pos (util/get-graphemes-pos (.-value input) pos)]
        (try
          (some-> (gdom/getElement "mock-text")
                  gdom/getChildren
                  array-seq
-                 (util/nth-safe pos)
+                 (util/nth-safe grapheme-pos)
                  mock-char-pos
                  (assoc :rect rect))
          (catch :default e
@@ -71,9 +72,8 @@
   ([input n]
    (when input
      (let [{:keys [pos]} (get-caret-pos input)
-           pos (if (and (= n 1) (not (zero? pos)))
-                 (or (util/safe-inc-current-pos-from-start (.-value input) pos)
-                     (inc pos))
+           pos (if (= n 1)
+                 (util/safe-inc-current-pos-from-start (.-value input) pos)
                  (+ pos n))]
        (move-cursor-to input pos)))))
 
@@ -85,8 +85,7 @@
      (let [{:keys [pos]} (get-caret-pos input)
            pos (if (= n 1)
                  (util/safe-dec-current-pos-from-end (.-value input) pos)
-                 (- pos n))
-           pos (max 0 (or pos (dec pos)))]
+                 (- pos n))]
        (move-cursor-to input pos)))))
 
 (defn- get-input-content&pos

src/test/frontend/util_test.cljs

@@ -13,6 +13,7 @@
     (is (= 3 (util/safe-dec-current-pos-from-end "abc😀d" 5)))
     (is (= 3 (util/safe-dec-current-pos-from-end "abc😀" 5)))
     (is (= 0 (util/safe-dec-current-pos-from-end "😀" 2)))
+    (is (= 0 (util/safe-dec-current-pos-from-end "a" 1)))
     (is (= 4 (util/safe-dec-current-pos-from-end "abcde" 5)))
     (is (= 1 (util/safe-dec-current-pos-from-end "中文" 2))))
 
@@ -20,8 +21,40 @@
     (is (= 5 (util/safe-inc-current-pos-from-start "abc😀d" 3)))
     (is (= 2 (util/safe-inc-current-pos-from-start "😀" 0)))
     (is (= 2 (util/safe-inc-current-pos-from-start "abcde" 1)))
+    (is (= 1 (util/safe-inc-current-pos-from-start "a" 0)))
     (is (= 1 (util/safe-inc-current-pos-from-start "中文" 0)))))
 
+(deftest test-get-line-pos
+  (testing "get-line-pos"
+    (is (= 3 (util/get-line-pos "abcde" 3)))
+    (is (= 4 (util/get-line-pos "abcd\ne" 4)))
+    (is (= 0 (util/get-line-pos "abcd\ne" 5)))
+    (is (= 4 (util/get-line-pos "abc😀d" 5)))
+    (is (= 1 (util/get-line-pos "abc\nde" 5)))
+    (is (= 1 (util/get-line-pos "abc\n😀d" 6)))
+    (is (= 2 (util/get-line-pos "ab\nc😀d" 6)))
+    (is (= 1 (util/get-line-pos "abc\nde\nf" 5)))
+    (is (= 1 (util/get-line-pos "abc\n😀d\ne" 6)))
+    (is (= 2 (util/get-line-pos "ab\nc😀d\ne" 6)))))
+
+(deftest test-get-text-range
+  (testing "get-text-range"
+    (is (= "" (util/get-text-range "abcdefg" 0 true)))
+    (is (= "" (util/get-text-range "abcdefg" 0 false)))
+    (is (= "abcdefg" (util/get-text-range "abcdefg" 10 true)))
+    (is (= "abcdefg" (util/get-text-range "abcdefg" 10 false)))
+    (is (= "abc" (util/get-text-range "abcdefg" 3 true)))
+    (is (= "abc" (util/get-text-range "abcdefg" 3 false)))
+    (is (= "abc" (util/get-text-range "abcdefg\nhijklmn" 3 true)))
+    (is (= "abcdefg\nhij" (util/get-text-range "abcdefg\nhijklmn" 3 false)))
+    (is (= "abcdefg\nhijklmn" (util/get-text-range "abcdefg\nhijklmn" 10 false)))
+    (is (= "abcdefg\nhijklmn\nopq" (util/get-text-range "abcdefg\nhijklmn\nopqrst" 3 false)))
+    (is (= "a😀b" (util/get-text-range "a😀bcdefg" 3 true)))
+    (is (= "a😀b" (util/get-text-range "a😀bcdefg" 3 false)))
+    (is (= "a😀b" (util/get-text-range "a😀bcdefg\nhijklmn" 3 true)))
+    (is (= "a😀bcdefg\nhij" (util/get-text-range "a😀bcdefg\nhijklmn" 3 false)))
+    (is (= "a😀bcdefg\nh😀i" (util/get-text-range "a😀bcdefg\nh😀ijklmn" 3 false)))))
+
 (deftest test-memoize-last
   (testing "memoize-last add test"
     (let [actual-ops (atom 0)