db wip

src/electron/electron/core.cljs

@@ -1,6 +1,7 @@
 (ns electron.core
   (:require [electron.handler :as handler]
             [electron.search :as search]
+            [electron.db :as db]
             [electron.updater :refer [init-updater] :as updater]
             [electron.utils :refer [*win mac? linux? dev? get-win-from-sender
                                     decode-protected-assets-schema-path get-graph-name send-to-renderer]
@@ -266,6 +267,7 @@
                (js-utils/disableXFrameOptions win)
 
                (search/ensure-search-dir!)
+               (db/ensure-graphs-dir!)
 
                (search/open-dbs!)
 

src/electron/electron/db.cljs

@@ -0,0 +1,138 @@
+(ns electron.db
+  "SQLite db"
+  (:require ["path" :as node-path]
+            ["fs-extra" :as fs]
+            ["better-sqlite3" :as sqlite3]
+            [clojure.string :as string]
+            ["electron" :refer [app]]
+            [electron.logger :as logger]
+            [medley.core :as medley]
+            [electron.utils :as utils]
+            [cljs-bean.core :as bean]))
+
+;; use built-in blocks to represent db schema, config, custom css, custom js, etc.
+
+(defonce databases (atom nil))
+
+(defn close!
+  []
+  (when @databases
+    (doseq [[_ database] @databases]
+      (.close database))
+    (reset! databases nil)))
+
+(defn sanitize-db-name
+  [db-name]
+  (-> db-name
+      (string/replace "/" "_")
+      (string/replace "\\" "_")
+      (string/replace ":" "_"))) ;; windows
+
+(defn get-db
+  [repo]
+  (get @databases (sanitize-db-name repo)))
+
+(declare delete-db!)
+
+(defn prepare
+  [^object db sql db-name]
+  (when db
+    (try
+      (.prepare db sql)
+      (catch :default e
+        (logger/error (str "SQLite prepare failed: " e ": " db-name))
+        (throw e)))))
+
+(defn create-blocks-table!
+  [db db-name]
+  (let [stmt (prepare db "CREATE TABLE IF NOT EXISTS blocks (
+                        id INTEGER PRIMARY KEY,
+                        page INTEGER,
+                        name TEXT,
+                        uuid TEXT NOT NULL,
+                        content TEXT,
+                        serialized_edn TEXT,
+                        journal_day INTEGER,
+                        core_data INTEGER,
+                        created_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL,
+                        updated_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL
+                        )"
+                      db-name)]
+    (.run ^object stmt)))
+
+;; ~/logseq
+(defn get-graphs-dir
+  []
+  (let [path (.getPath ^object app "home")]
+    (node-path/join path "logseq" "graphs")))
+
+(defn ensure-graphs-dir!
+  []
+  (fs/ensureDirSync (get-graphs-dir)))
+
+(defn get-db-full-path
+  [db-name]
+  (let [db-name (sanitize-db-name db-name)
+        dir (get-graphs-dir)]
+    [db-name (node-path/join dir db-name)]))
+
+(defn open-db!
+  [db-name]
+  (let [[db-sanitized-name db-full-path] (get-db-full-path db-name)]
+    (try (let [db (sqlite3 db-full-path nil)]
+           (create-blocks-table! db db-name)
+           (swap! databases assoc db-sanitized-name db))
+         (catch :default e
+           (logger/error (str e ": " db-name))
+           ;; (fs/unlinkSync db-full-path)
+           ))))
+
+(defn- clj-list->sql
+  "Turn clojure list into SQL list
+   '(1 2 3 4)
+   ->
+   \"('1','2','3','4')\""
+  [ids]
+  (str "(" (->> (map (fn [id] (str "'" id "'")) ids)
+                (string/join ", ")) ")"))
+
+(defn upsert-blocks!
+  [repo blocks]
+  (if-let [db (get-db repo)]
+    (let [insert (prepare db "INSERT INTO blocks (id, page, name, uuid, content, serialized_edn, journal_day, core_data, created_at, updated_at) VALUES (@id, @page, @name, @uuid, @content, @serialized_edn, @journal_day, @core_data, @created_at, @updated_at) ON CONFLICT (id) DO UPDATE SET (page, name, uuid, content, serialized_edn, journal_day, core_data, created_at, updated_at) = (@page, @name, @uuid, @content, @serialized_edn, @journal_day, @core_data, @created_at, @updated_at)" repo)
+          insert-many (.transaction ^object db
+                                    (fn [blocks]
+                                      (doseq [block blocks]
+                                        (.run ^object insert block))))]
+      (insert-many blocks))
+    (do
+      (open-db! repo)
+      (upsert-blocks! repo blocks))))
+
+(defn delete-blocks!
+  [repo ids]
+  (when-let [db (get-db repo)]
+    (let [sql (str "DELETE from blocks WHERE id IN " (clj-list->sql ids))
+          stmt (prepare db sql repo)]
+      (.run ^object stmt))))
+
+
+;; Initial data:
+;; All pages and block ids
+;; latest 3 journals
+;; core data such as config, custom css/js
+;; current page, sidebar blocks
+
+(defn get-initial-data!
+  [repo]
+  (when-let [db (get-db repo)]
+    (let [sql "select * from blocks"
+          stmt (prepare db sql repo)]
+      (.all ^object stmt))))
+
+(defn get-all-data
+  [repo]
+  (when-let [db (get-db repo)]
+    (let [sql "select * from blocks"
+          stmt (prepare db sql repo)]
+      (.all ^object stmt))))

src/electron/electron/handler.cljs

@@ -23,6 +23,7 @@
             [electron.logger :as logger]
             [electron.plugin :as plugin]
             [electron.search :as search]
+            [electron.db :as db]
             [electron.server :as server]
             [electron.shell :as shell]
             [electron.state :as state]
@@ -329,6 +330,38 @@
 ;; ^^^^
 ;; Search related IPCs End
 
+;; DB related IPCs start
+
+(defmethod handle :db-new [_window [_ repo]]
+  (db/open-db! repo))
+
+(defmethod handle :db-transact-data [_window [_ repo data-str]]
+  (let [data (reader/read-string data-str)
+        {:keys [blocks deleted-block-ids tx-data]} data]
+    ;; TODO: store files
+    (when (seq deleted-block-ids)
+      (db/delete-blocks! repo deleted-block-ids))
+    (when (seq blocks)
+      (let [blocks' (mapv
+                      (fn [b]
+                        {:id (:db/id b)
+                         :page (:db/id (:block/page b))
+                         :name (:block/name b)
+                         :uuid (str (:block/uuid b))
+                         :content (:block/content b)
+                         :serialized_edn (pr-str b)
+                         :journal_day (:block/journal-day b)
+                         :core_data 0
+                         :created_at (or (:block/created-at b) (utils/time-ms))
+                         :updated_at (or (:block/updated-at b) (utils/time-ms))})
+                      blocks)]
+        (db/upsert-blocks! repo (bean/->js blocks'))))))
+
+(defmethod handle :get-initial-data [_ [_ repo _opts]]
+  (db/get-all-data repo))
+
+;; DB related IPCs End
+
 (defn clear-cache!
   [window]
   (let [graphs-dir (get-graphs-dir)]

src/electron/electron/search.cljs

@@ -201,6 +201,7 @@
 
 (defn upsert-blocks!
   [repo blocks]
+  (prn "debug-search-blocks" blocks)
   (if-let [db (get-db repo)]
     ;; TODO: what if a CONFLICT on uuid
     ;; Should update all values on id conflict

src/electron/electron/utils.cljs

@@ -7,7 +7,9 @@
             [electron.configs :as cfgs]
             [electron.logger :as logger]
             [cljs-bean.core :as bean]
-            [promesa.core :as p]))
+            [promesa.core :as p]
+            [cljs-time.coerce :as tc]
+            [cljs-time.core :as t]))
 
 (defonce *win (atom nil)) ;; The main window
 
@@ -274,3 +276,7 @@
     (catch :default _
       (println "decodeURIComponent failed: " uri)
       uri)))
+
+(defn time-ms
+  []
+  (tc/to-long (t/now)))

src/main/frontend/components/repo.cljs

@@ -186,6 +186,8 @@
               (if (or (nfs-handler/supported?) (mobile-util/native-platform?))
                 {:title (t :new-graph) :options {:on-click #(state/pub-event! [:graph/setup-a-repo])}}
                 {:title (t :new-graph) :options {:href (rfe/href :repos)}}) ;; Brings to the repos page for showing fallback message
+              {:title (str (t :new-graph) "- DB version")
+               :options {:on-click #(state/pub-event! [:graph/new-db-graph])}}
               {:title (t :all-graphs) :options {:href (rfe/href :repos)}}
               refresh-link
               reindex-link
@@ -241,3 +243,18 @@
                                                       (ui/icon "refresh")]))]))]
         (when (seq repos)
           (ui/dropdown-with-links render-content links links-header))))))
+
+(rum/defcs new-db-graph <
+  (rum/local "" ::graph-name)
+  [state]
+  (let [*graph-name (::graph-name state)]
+    [:div.new-graph.p-4
+     [:h1.title "Create new graph: "]
+     [:input.form-input.mb-4 {:value @*graph-name
+                              :auto-focus true
+                              :on-change #(reset! *graph-name (util/evalue %))}]
+     (ui/button
+       "Submit"
+       :on-click (fn []
+                   (when-not (string/blank? @*graph-name)
+                     (repo-handler/new-db! @*graph-name))))]))

src/main/frontend/config.cljs

@@ -354,6 +354,7 @@
 (defonce idb-db-prefix "logseq-db/")
 (defonce local-db-prefix "logseq_local_")
 (defonce local-handle "handle")
+(defonce db-version-prefix "logseq_db_")
 
 (defn local-db?
   [s]

src/main/frontend/db.cljs

@@ -16,7 +16,11 @@
             [frontend.state :as state]
             [frontend.util :as util]
             [promesa.core :as p]
-            [electron.ipc :as ipc]))
+            [electron.ipc :as ipc]
+            [clojure.string :as string]
+            [frontend.config :as config]
+            [clojure.edn :as edn]
+            [cljs-bean.core :as bean]))
 
 (import-vars
  [frontend.db.conn
@@ -174,12 +178,30 @@
                 (conn/reset-conn! db-conn db)))]
     (d/transact! db-conn [{:schema/version db-schema/version}])))
 
+(defn restore-graph-from-sqlite!
+  "Load graph from SQLite"
+  [repo]
+  (p/let [db-name (datascript-db repo)
+          db-conn (d/create-conn db-schema/schema)
+          _ (swap! conns assoc db-name db-conn)
+          data (ipc/ipc :get-initial-data repo)
+          data (bean/->clj data)
+          blocks (map (comp edn/read-string :serialized_edn) data)]
+
+    ;; TODO: Store schema in sqlite
+    ;; (db-migrate/migrate attached-db)
+
+    (d/transact! db-conn [{:schema/version db-schema/version}])
+    (d/transact! db-conn blocks)))
+
 (defn restore-graph!
   "Restore db from serialized db cache"
   [repo]
-  (p/let [db-name (datascript-db repo)
-          stored (db-persist/get-serialized-graph db-name)]
-    (restore-graph-from-text! repo stored)))
+  (if (string/starts-with? repo config/db-version-prefix)
+    (restore-graph-from-sqlite! repo)
+    (p/let [db-name (datascript-db repo)
+           stored (db-persist/get-serialized-graph db-name)]
+     (restore-graph-from-text! repo stored))))
 
 (defn restore!
   [repo]

src/main/frontend/db/react.cljs

@@ -393,3 +393,8 @@
               (js/console.error error)))))
       (recur))
     chan))
+
+(defn db-graph?
+  "Whether the current graph is db-only"
+  [graph]
+  (= "db" (sub-key-value :db/type)))

src/main/frontend/handler.cljs

@@ -42,7 +42,8 @@
             [goog.object :as gobj]
             [lambdaisland.glogi :as log]
             [promesa.core :as p]
-            [frontend.mobile.core :as mobile]))
+            [frontend.mobile.core :as mobile]
+            [frontend.db.react :as db-react]))
 
 (defn set-global-error-notification!
   []
@@ -63,9 +64,11 @@
   (let [f (fn []
             #_:clj-kondo/ignore
             (let [repo (state/get-current-repo)]
-              (when (and (not (state/nfs-refreshing?))
-                         (not (contains? (:file/unlinked-dirs @state/state)
-                                         (config/get-repo-dir repo))))
+              (when (or
+                     (db-react/db-graph? repo)
+                     (and (not (state/nfs-refreshing?))
+                          (not (contains? (:file/unlinked-dirs @state/state)
+                                          (config/get-repo-dir repo)))))
                 ;; Don't create the journal file until user writes something
                 (page-handler/create-today-journal!))))]
     (f)

src/main/frontend/handler/events.cljs

@@ -23,6 +23,7 @@
             [frontend.components.shell :as shell]
             [frontend.components.whiteboard :as whiteboard]
             [frontend.components.user.login :as login]
+            [frontend.components.repo :as repo]
             [frontend.config :as config]
             [frontend.context.i18n :refer [t]]
             [frontend.db :as db]
@@ -867,6 +868,12 @@
        {:label "graph-setup"})
       (page-handler/ls-dir-files! st/refresh! opts'))))
 
+(defmethod handle :graph/new-db-graph [[_ opts]]
+  (state/set-modal!
+   repo/new-db-graph
+   {:id :new-db-graph
+    :label "graph-setup"}))
+
 (defmethod handle :file/alter [[_ repo path content]]
   (p/let [_ (file-handler/alter-file repo path content {:from-disk? true})]
     (ui-handler/re-render-root!)))

src/main/frontend/handler/repo.cljs

@@ -31,7 +31,8 @@
             [frontend.mobile.util :as mobile-util]
             [medley.core :as medley]
             [logseq.common.path :as path]
-            [logseq.common.config :as common-config]))
+            [logseq.common.config :as common-config]
+            [frontend.db.react :as react]))
 
 ;; Project settings should be checked in two situations:
 ;; 1. User changes the config.edn directly in logseq.com (fn: alter-file)
@@ -531,3 +532,24 @@
   ;; FIXME: Call electron that the graph is loaded, an ugly implementation for redirect to page when graph is restored
   [graph]
   (ipc/ipc "graphReady" graph))
+
+;; New DB implementation
+(defn new-db!
+  [graph]
+  ;; TODO: check whether graph exists first
+  (p/let [full-graph-name (str config/db-version-prefix graph)
+          _ (start-repo-db-if-not-exists! full-graph-name)
+          _ (state/add-repo! {:url full-graph-name})
+          _ (ipc/ipc :db-new graph)
+          initial-data [(react/kv :db/type "db")
+                        {:file/path (str "logseq/" "config.edn")
+                         :file/content config/config-default-content}
+                        {:file/path (str "logseq/" "custom.css")
+                         :file/content ""}
+                        {:file/path (str "logseq/" "custom.js")
+                         :file/content ""}]
+          _ (db/transact! full-graph-name initial-data)
+          _ (repo-config-handler/set-repo-config-state! full-graph-name config/config-default-content)
+          ;; TODO: handle global graph
+          _ (state/pub-event! [:page/create (date/today) {:redirect? false}])]
+    (prn "New db created: " full-graph-name)))

src/main/frontend/modules/outliner/datascript.cljc

@@ -4,6 +4,7 @@
   #?(:cljs (:require [datascript.core :as d]
                      [frontend.db.conn :as conn]
                      [frontend.db :as db]
+                     [frontend.db.react :as react]
                      [frontend.modules.outliner.pipeline :as pipelines]
                      [frontend.modules.editor.undo-redo :as undo-redo]
                      [frontend.state :as state]
@@ -54,7 +55,8 @@
 #?(:cljs
    (defn transact!
      [txs opts before-editor-cursor]
-     (let [txs (remove-nil-from-transaction txs)
+     (let [repo (state/get-current-repo)
+           txs (remove-nil-from-transaction txs)
            txs (map (fn [m] (if (map? m)
                               (dissoc m
                                       :block/children :block/meta :block/top? :block/bottom? :block/anchor
@@ -62,8 +64,10 @@
                               m)) txs)]
        (when (and (seq txs)
                   (not (:skip-transact? opts))
-                  (not (contains? (:file/unlinked-dirs @state/state)
-                                  (config/get-repo-dir (state/get-current-repo)))))
+                  (if (react/db-graph? repo)
+                    true
+                    (not (contains? (:file/unlinked-dirs @state/state)
+                                    (config/get-repo-dir repo)))))
 
          ;; (prn "[DEBUG] Outliner transact:")
          ;; (frontend.util/pprint txs)

src/main/frontend/modules/outliner/pipeline.cljs

@@ -7,11 +7,15 @@
             [frontend.db.react :as react]
             [frontend.db :as db]
             [clojure.set :as set]
-            [datascript.core :as d]))
+            [datascript.core :as d]
+            [electron.ipc :as ipc]
+            [promesa.core :as p]))
 
 (defn updated-page-hook
   [tx-report page]
-  (when-not (get-in tx-report [:tx-meta :created-from-journal-template?])
+  (when (and
+         (not (react/db-graph? (state/get-current-repo)))
+         (not (get-in tx-report [:tx-meta :created-from-journal-template?])))
     (file/sync-to-file page (:outliner-op (:tx-meta tx-report)))))
 
 ;; TODO: it'll be great if we can calculate the :block/path-refs before any
@@ -82,6 +86,14 @@
                          children-refs))))
                   blocks))))))
 
+(defn- filter-deleted-blocks
+  [datoms]
+  (keep
+   (fn [d]
+     (when (and (= :block/uuid (:a d)) (false? (:added d)))
+       (:e d)))
+   datoms))
+
 (defn invoke-hooks
   [tx-report]
   (let [tx-meta (:tx-meta tx-report)]
@@ -101,11 +113,23 @@
                            ;; merge
                            (assoc tx-report :tx-data (concat (:tx-data tx-report) refs-tx-data')))
                          tx-report)
-            importing? (:graph/importing @state/state)]
+            importing? (:graph/importing @state/state)
+            deleted-block-ids (set (filter-deleted-blocks (:tx-data tx-report)))]
 
         (when-not importing?
           (react/refresh! repo tx-report'))
 
+        (let [upsert-blocks (remove (fn [b] (contains? deleted-block-ids (:db/id b))) blocks)
+              datoms-data (map (fn [d]
+                                 [(:e d) (:a d) (:v d) (:tx d) (:added d)]) (:tx-data tx-report))]
+          (p/let [ipc-result (ipc/ipc :db-transact-data repo
+                                      (pr-str
+                                       {:blocks upsert-blocks
+                                        :deleted-block-ids deleted-block-ids
+                                        :datoms datoms-data}))]
+            ;; TODO: disable edit when transact failed to avoid future data-loss
+            (prn "DB transact result: " ipc-result)))
+
         (when-not (:delete-files? tx-meta)
           (doseq [p (seq pages)]
             (updated-page-hook tx-report p)))
@@ -116,5 +140,6 @@
                    (<= (count blocks) 1000))
           (state/pub-event! [:plugin/hook-db-tx
                              {:blocks  blocks
+                              :deleted-block-ids deleted-block-ids
                               :tx-data (:tx-data tx-report)
                               :tx-meta (:tx-meta tx-report)}]))))))

src/main/frontend/search.cljs

@@ -249,8 +249,10 @@
                                      blocks-result)
                              (map search-db/block->index)
                              (remove nil?))
+          added (set (map :id blocks-to-add))
           blocks-to-remove-set (->> (remove :added blocks)
                                     (map :e)
+                                    (remove added)
                                     (set))]
       {:blocks-to-remove-set blocks-to-remove-set
        :blocks-to-add        blocks-to-add})))