Merge branch 'feat/db' into refactor/worker-search

deps.edn

@@ -4,7 +4,7 @@
   rum/rum                               {:mvn/version "0.12.9"}
 
   datascript/datascript                 {:git/url "https://github.com/logseq/datascript" ;; fork
-                                         :sha     "15234c650df20844ec089c1738d28c00eccacd6e"}
+                                         :sha     "29985da9844e37c89bba12f9fce70f94aed16ba6"}
 
   datascript-transit/datascript-transit {:mvn/version "0.3.0"}
   borkdude/rewrite-edn                  {:mvn/version "0.4.7"}

deps/db/deps.edn

@@ -1,7 +1,7 @@
 {:deps
  ;; External deps should be kept in sync with https://github.com/logseq/nbb-logseq/blob/main/bb.edn
  {datascript/datascript {:git/url "https://github.com/logseq/datascript" ;; fork
-                         :sha     "15234c650df20844ec089c1738d28c00eccacd6e"}
+                         :sha     "29985da9844e37c89bba12f9fce70f94aed16ba6"}
   com.cognitect/transit-cljs   {:mvn/version "0.8.280"}
   cljs-bean/cljs-bean          {:mvn/version "1.5.0"}}
  :aliases

deps/db/src/logseq/db/sqlite/db.cljs

@@ -96,5 +96,6 @@
 
 (defn transact!
   [repo tx-data tx-meta]
-  (when-let [conn (get-conn repo)]
-    (d/transact! conn tx-data tx-meta)))
+  (if-let [conn (get-conn repo)]
+    (d/transact! conn tx-data tx-meta)
+    (throw (ex-info (str "Failed to transact! No db connection found for " repo) {}))))

deps/outliner/deps.edn

@@ -1,7 +1,7 @@
 {:deps
  ;; External deps should be kept in sync with https://github.com/logseq/nbb-logseq/blob/main/bb.edn
  {datascript/datascript {:git/url "https://github.com/logseq/datascript" ;; fork
-                         :sha     "15234c650df20844ec089c1738d28c00eccacd6e"}
+                         :sha     "29985da9844e37c89bba12f9fce70f94aed16ba6"}
   logseq/db             {:local/root "../db"}
   com.cognitect/transit-cljs {:mvn/version "0.8.280"}}
  :aliases

externs.js

@@ -142,6 +142,7 @@ dummy.DOCUMENT_TYPE = function() {};
 dummy.ELEMENT = function() {};
 dummy.TEXT = function() {};
 dummy.isAbsolute = function() {};
+dummy._address = function() {};
 
 var utils = {}
 utils.withFileTypes = true;

scripts/src/logseq/tasks/dev/db_and_file_graphs.clj

@@ -35,8 +35,7 @@
    "src/main/frontend/components/property.cljs"
    "src/main/frontend/components/property"
    "src/main/frontend/components/db_based"
-   ;; TODO: Enable this when run-export-periodically is deleted or moved out of the ns
-   #_"src/main/frontend/persist_db.cljs"
+   "src/main/frontend/persist_db.cljs"
    "src/main/frontend/persist_db"
    "src/electron/electron/db.cljs"])
 

src/main/frontend/components/cmdk.cljs

@@ -410,7 +410,7 @@
   (let [command (some-> state state->highlighted-item :source-command)]
     (when-let [action (:action command)]
       (action)
-      (when-not (contains? #{:graph/open :graph/remove :ui/toggle-settings :go/flashcards} (:id command))
+      (when-not (contains? #{:graph/open :graph/remove :dev/replace-graph-with-db-file :ui/toggle-settings :go/flashcards} (:id command))
         (state/close-modal!)))))
 
 (defmethod handle-action :create [_ state _event]

src/main/frontend/components/imports.cljs

@@ -5,6 +5,7 @@
             [frontend.ui :as ui]
             [frontend.context.i18n :refer [t]]
             [frontend.components.svg :as svg]
+            [frontend.components.repo :as repo]
             [frontend.handler.route :as route-handler]
             [frontend.handler.ui :as ui-handler]
             [frontend.handler.notification :as notification]
@@ -12,6 +13,7 @@
             [clojure.string :as string]
             [goog.object :as gobj]
             [frontend.components.onboarding.setups :as setups]
+            [frontend.util.fs :as fs-util]
             [frontend.util.text :as text-util]
             [frontend.util :as util]))
 
@@ -125,7 +127,9 @@
   (rum/local "" ::input)
   [state sqlite-input-e opts]
   (let [*input (::input state)
-        on-submit #(lsq-import-handler sqlite-input-e (assoc opts :graph-name @*input))]
+        on-submit #(if (fs-util/include-reserved-chars? @*input)
+                     (repo/invalid-graph-name-warning)
+                     (lsq-import-handler sqlite-input-e (assoc opts :graph-name @*input)))]
     [:div.container
      [:div.sm:flex.sm:items-start
       [:div.mt-3.text-center.sm:mt-0.sm:text-left

src/main/frontend/components/repo.cljs

@@ -263,32 +263,35 @@
         (when (seq repos)
           (ui/dropdown-with-links render-content links links-header))))))
 
+(defn invalid-graph-name-warning
+  []
+  (notification/show!
+   [:div
+    [:p "Graph name can't contain following reserved characters:"]
+    [:ul
+     [:li "< (less than)"]
+     [:li "> (greater than)"]
+     [:li ": (colon)"]
+     [:li "\" (double quote)"]
+     [:li "/ (forward slash)"]
+     [:li "\\ (backslash)"]
+     [:li "| (vertical bar or pipe)"]
+     [:li "? (question mark)"]
+     [:li "* (asterisk)"]
+     [:li "# (hash)"]
+     ;; `+` is used to encode path that includes `:` or `/`
+     [:li "+ (plus)"]]]
+   :warning false))
+
 (rum/defcs new-db-graph <
   (rum/local "" ::graph-name)
   [state]
   (let [*graph-name (::graph-name state)
         new-db-f (fn []
                    (when-not (string/blank? @*graph-name)
-                     (if (or
-                          (fs-util/include-reserved-chars? @*graph-name)
-                          (string/includes? @*graph-name "+"))
-                       (notification/show!
-                        [:div
-                         [:p "Graph name can't contain following reserved characters:"]
-                         [:ul
-                          [:li "< (less than)"]
-                          [:li "> (greater than)"]
-                          [:li ": (colon)"]
-                          [:li "\" (double quote)"]
-                          [:li "/ (forward slash)"]
-                          [:li "\\ (backslash)"]
-                          [:li "| (vertical bar or pipe)"]
-                          [:li "? (question mark)"]
-                          [:li "* (asterisk)"]
-                          [:li "# (hash)"]
-                          ;; `+` is used to encode path that includes `:` or `/`
-                          [:li "+ (plus)"]]]
-                        :warning false)
+                     (if (or (fs-util/include-reserved-chars? @*graph-name)
+                             (string/includes? @*graph-name "+"))
+                       (invalid-graph-name-warning)
                        (do
                          (repo-handler/new-db! @*graph-name)
                          (state/close-modal!)))))]

src/main/frontend/components/select.cljs

@@ -13,6 +13,7 @@
             [rum.core :as rum]
             [frontend.config :as config]
             [frontend.handler.repo :as repo-handler]
+            [frontend.handler.common.developer :as dev-common-handler]
             [reitit.frontend.easy :as rfe]
             [clojure.string :as string]))
 
@@ -203,7 +204,20 @@
                              :id (config/get-repo-dir url)
                              :graph url
                              :original-graph original-graph}))))
-    :on-chosen #(repo-handler/remove-repo! (:original-graph %))}})
+    :on-chosen #(repo-handler/remove-repo! (:original-graph %))}
+   :db-graph-replace
+   {:items-fn (fn []
+                (let [current-repo (state/get-current-repo)]
+                  (->> (state/get-repos)
+                      (remove (fn [{:keys [url]}]
+                                ;; Can't replace current graph as ui wouldn't reload properly
+                                (or (= url current-repo) (not (config/db-based-graph? url)))))
+                      (map (fn [{:keys [url] :as original-graph}]
+                             {:value (text-util/get-graph-name-from-path url)
+                              :id (config/get-repo-dir url)
+                              :graph url
+                              :original-graph original-graph})))))
+    :on-chosen #(dev-common-handler/import-chosen-graph (:graph %))}})
 
 (rum/defc select-modal < rum/reactive
   []

src/main/frontend/db_worker.cljs

@@ -12,7 +12,6 @@
             [clojure.string :as string]
             [cljs-bean.core :as bean]
             [frontend.worker.search :as search]
-            [frontend.util :as util]
             [logseq.db.sqlite.util :as sqlite-util]))
 
 (defonce *sqlite (atom nil))
@@ -57,7 +56,7 @@
             base-url (str js/self.location.protocol "//" js/self.location.host)
             sqlite-wasm-url (if electron?
                               (js/URL. "sqlite3.wasm" (.. js/location -href))
-                              (str base-url "/js/"))
+                              (str base-url (string/replace js/self.location.pathname "db-worker.js" "")))
             sqlite (sqlite3InitModule (clj->js {:url sqlite-wasm-url
                                                 :print js/console.log
                                                 :printErr js/console.error}))]
@@ -105,14 +104,13 @@
   [repo _opts]
   (reify IStorage
     (-store [_ addr+data-seq delete-addrs]
-      (util/profile
-       (str "SQLite store addr+data count: " (count addr+data-seq))
-       (let [data (map
-                   (fn [[addr data]]
-                     #js {:$addr addr
-                          :$content (pr-str data)})
-                   addr+data-seq)]
-         (upsert-addr-content! repo data delete-addrs))))
+      (prn :debug (str "SQLite store addr+data count: " (count addr+data-seq)))
+      (let [data (map
+                  (fn [[addr data]]
+                    #js {:$addr addr
+                         :$content (pr-str data)})
+                  addr+data-seq)]
+        (upsert-addr-content! repo data delete-addrs)))
 
     (-restore [_ addr]
       (restore-data-from-addr repo addr))))
@@ -258,16 +256,14 @@
   (transact
    [_this repo tx-data tx-meta]
    (when-let [conn (get-datascript-conn repo)]
-     (util/profile
-      "DB transact!"
-      (try
-        (let [tx-data (edn/read-string tx-data)
-              tx-meta (edn/read-string tx-meta)]
-          (d/transact! conn tx-data tx-meta)
-          nil)
-        (catch :default e
-          (prn :debug :error)
-          (js/console.error e))))))
+     (try
+       (let [tx-data (edn/read-string tx-data)
+             tx-meta (edn/read-string tx-meta)]
+         (d/transact! conn tx-data tx-meta)
+         nil)
+       (catch :default e
+         (prn :debug :error)
+         (js/console.error e)))))
 
   (getInitialData
    [_this repo]

src/main/frontend/handler/common/developer.cljs

@@ -8,7 +8,9 @@
             [frontend.util.page :as page-util]
             [frontend.handler.db-based.property.util :as db-pu]
             [frontend.format.mldoc :as mldoc]
-            [frontend.config :as config]))
+            [frontend.config :as config]
+            [frontend.persist-db :as persist-db]
+            [promesa.core :as p]))
 
 ;; Fns used between menus and commands
 (defn show-entity-data
@@ -69,3 +71,12 @@
       (if (get-in page-data [:block/file :file/content])
         (show-content-ast (get-in page-data [:block/file :file/content]) (:block/format page-data))
         (notification/show! "No page found" :warning)))))
+
+(defn import-chosen-graph
+  [repo]
+  (p/let [_ (persist-db/<unsafe-delete repo)
+          _ (persist-db/<fetch-init-data repo)]
+    (notification/show! "Graph updated!" :success)))
+
+(defn ^:export replace-graph-with-db-file []
+  (state/set-state! :ui/open-select :db-graph-replace))

src/main/frontend/modules/shortcut/config.cljs

@@ -548,6 +548,10 @@
                                              :file-graph? true
                                              :fn       commit/show-commit-modal!}
 
+   :dev/replace-graph-with-db-file           {:binding  []
+                                              :inactive (or (not (util/electron?)) (not (state/developer-mode?)))
+                                              :fn       :frontend.handler.common.developer/replace-graph-with-db-file}
+
    :dev/show-block-data                     {:binding  []
                                              :inactive (not (state/developer-mode?))
                                              :fn       :frontend.handler.common.developer/show-block-data}
@@ -758,6 +762,7 @@
           :dev/show-block-ast
           :dev/show-page-data
           :dev/show-page-ast
+          :dev/replace-graph-with-db-file
           :ui/cycle-color
           :ui/cycle-color-off])
         (with-meta {:before m/enable-when-not-editing-mode!}))
@@ -941,6 +946,7 @@
      :dev/show-block-ast
      :dev/show-page-data
      :dev/show-page-ast
+     :dev/replace-graph-with-db-file
      :ui/clear-all-notifications]
 
     :shortcut.category/plugins

src/main/frontend/persist_db/browser.cljs

@@ -82,7 +82,9 @@
           tx-meta' (pr-str tx-meta)]
       (p/do!
        (ipc/ipc :db-transact repo tx-data' tx-meta')
-       (when sqlite (.transact sqlite repo tx-data' tx-meta'))
+       (if sqlite
+         (.transact sqlite repo tx-data' tx-meta')
+         (notification/show! "Latest change was not saved! Please restart the application." :error))
        nil)))
 
   (<fetch-initial-data [_this repo _opts]

src/main/frontend/util/fs.cljs

@@ -73,6 +73,7 @@
   (when-let [repo-dir (config/get-repo-dir repo-url)]
     (fs/read-file repo-dir file-rpath)))
 
+;; Update repo/invalid-graph-name-warning if characters change
 (def multiplatform-reserved-chars ":\\*\\?\"<>|\\#\\\\")
 
 (def reserved-chars-pattern

src/resources/dicts/en.edn

@@ -818,4 +818,5 @@
   :dev/show-block-ast              "(Dev) Show block AST"
   :dev/show-page-data              "(Dev) Show page data"
   :dev/show-page-ast               "(Dev) Show page AST"
+  :dev/replace-graph-with-db-file  "(Dev) Replace graph with its db.sqlite file"
   :window/close                    "Close window"}}