fix: don't delete rows from kvs if they're still used by other rows

deps.edn

@@ -3,8 +3,9 @@
  {org.clojure/clojure                   {:mvn/version "1.11.1"}
   rum/rum                               {:mvn/version "0.12.9"}
 
-  datascript/datascript                 {:git/url "https://github.com/logseq/datascript" ;; fork
-                                         :sha     "8c7168b0c29ae2dc3c3efb5d28188916e991bde1"}
+  datascript/datascript                 {:local/root "../../Datascript"}
+  ;; datascript/datascript                 {:git/url "https://github.com/logseq/datascript" ;; fork
+  ;;                                        :sha     "8c7168b0c29ae2dc3c3efb5d28188916e991bde1"}
 
   datascript-transit/datascript-transit {:mvn/version "0.3.0"}
   borkdude/rewrite-edn                  {:mvn/version "0.4.7"}

deps/db/deps.edn

@@ -1,7 +1,8 @@
 {:deps
  ;; These deps are kept in sync with https://github.com/logseq/nbb-logseq/blob/main/bb.edn
- {datascript/datascript {:git/url "https://github.com/logseq/datascript" ;; fork
-                         :sha     "8c7168b0c29ae2dc3c3efb5d28188916e991bde1"}
+ {datascript/datascript                 {:local/root "../../../../Datascript"}
+  ;; datascript/datascript {:git/url "https://github.com/logseq/datascript" ;; fork
+  ;;                        :sha     "8c7168b0c29ae2dc3c3efb5d28188916e991bde1"}
   datascript-transit/datascript-transit {:mvn/version "0.3.0"
                                          :exclusions [datascript/datascript]}
   cljs-bean/cljs-bean         {:mvn/version "1.5.0"}

deps/db/src/logseq/db/sqlite/common_db.cljs

@@ -282,7 +282,7 @@
 (defn create-kvs-table!
   "Creates a sqlite table for use with datascript.storage if one doesn't exist"
   [sqlite-db]
-  (.exec sqlite-db "create table if not exists kvs (addr INTEGER primary key, content TEXT)"))
+  (.exec sqlite-db "create table if not exists kvs (addr INTEGER primary key, content TEXT, addresses JSON)"))
 
 (defn get-storage-conn
   "Given a datascript storage, returns a datascript connection for it"

deps/outliner/deps.edn

@@ -1,7 +1,8 @@
 {:deps
  ;; External deps should be kept in sync with https://github.com/logseq/nbb-logseq/blob/main/bb.edn
- {datascript/datascript {:git/url "https://github.com/logseq/datascript" ;; fork
-                         :sha     "8c7168b0c29ae2dc3c3efb5d28188916e991bde1"}
+ {datascript/datascript                 {:local/root "../../../../Datascript"}
+  ;; datascript/datascript {:git/url "https://github.com/logseq/datascript" ;; fork
+  ;;                        :sha     "8c7168b0c29ae2dc3c3efb5d28188916e991bde1"}
   logseq/db             {:local/root "../db"}
   logseq/graph-parser   {:local/root "../db"}
   com.cognitect/transit-cljs {:mvn/version "0.8.280"}

src/main/frontend/worker/db/migrate.cljs

@@ -117,8 +117,9 @@
                                            :else
                                            (first types))
                                          types)]
-                              [[:db/retract id :block/type]
-                               [:db/add id :block/type type]]))))
+                              (when type
+                                [[:db/retract id :block/type]
+                                [:db/add id :block/type type]])))))
         schema (:schema db)]
     (ldb/transact! conn new-type-tx {:db-migrate? true})
     (d/reset-schema! conn (update schema :block/type #(assoc % :db/cardinality :db.cardinality/one)))

src/main/frontend/worker/db_worker.cljs

@@ -5,6 +5,7 @@
             [cljs-bean.core :as bean]
             [clojure.edn :as edn]
             [clojure.string :as string]
+            [clojure.set :as set]
             [datascript.core :as d]
             [datascript.storage :refer [IStorage]]
             [frontend.common.file.core :as common-file]
@@ -88,6 +89,29 @@
   [^js pool data]
   (.importDb ^js pool repo-path data))
 
+(comment
+  (defn- gc-kvs-table!
+    [^Object db]
+    (let [schema (some->> (.exec db #js {:sql "select content from kvs where addr = 0"
+                                         :rowMode "array"})
+                          bean/->clj
+                          ffirst
+                          sqlite-util/transit-read)
+          result (->> (.exec db #js {:sql "select addr, addresses from kvs"
+                                     :rowMode "array"})
+                      bean/->clj
+                      (map (fn [[addr addresses]]
+                             [addr (bean/->clj (js/JSON.parse addresses))])))
+          used-addresses (set (concat (mapcat second result)
+                                      [0 1 (:eavt schema) (:avet schema) (:aevt schema)]))
+          unused-addresses (set/difference (set (map first result)) used-addresses)]
+      (when unused-addresses
+        (prn :debug :db-gc :unused-addresses unused-addresses)
+        (.transaction db (fn [tx]
+                           (doseq [addr unused-addresses]
+                             (.exec tx #js {:sql "Delete from kvs where addr = ?"
+                                            :bind #js [addr]}))))))))
+
 (defn upsert-addr-content!
   "Upsert addr+data-seq"
   [repo data delete-addrs & {:keys [client-ops-db?] :or {client-ops-db? false}}]
@@ -95,37 +119,51 @@
     (assert (some? db) "sqlite db not exists")
     (.transaction db (fn [tx]
                        (doseq [item data]
-                         (.exec tx #js {:sql "INSERT INTO kvs (addr, content) values ($addr, $content) on conflict(addr) do update set content = $content"
-                                        :bind item}))
-
-                       (doseq [addr delete-addrs]
-                         (.exec db #js {:sql "Delete from kvs where addr = ?"
-                                        :bind #js [addr]}))))))
+                         (.exec tx #js {:sql "INSERT INTO kvs (addr, content, addresses) values ($addr, $content, $addresses) on conflict(addr) do update set content = $content, addresses = $addresses"
+                                        :bind item}))))
+    (when (seq delete-addrs)
+      (.transaction db (fn [tx]
+                         (prn :debug :delete-addrs delete-addrs)
+                         (doseq [addr delete-addrs]
+                           (.exec tx #js {:sql "Delete from kvs WHERE addr = ? AND NOT EXISTS (SELECT 1 FROM json_each(addresses) WHERE value = ?);"
+                                          :bind #js [addr]})))))))
 
 (defn restore-data-from-addr
   [repo addr & {:keys [client-ops-db?] :or {client-ops-db? false}}]
   (let [^Object db (worker-state/get-sqlite-conn repo (if client-ops-db? :client-ops :db))]
     (assert (some? db) "sqlite db not exists")
-    (when-let [content (-> (.exec db #js {:sql "select content from kvs where addr = ?"
-                                          :bind #js [addr]
-                                          :rowMode "array"})
-                           ffirst)]
-      (try
-        (let [data (sqlite-util/transit-read content)]
-          (if-let [addresses (:addresses data)]
-            (assoc data :addresses (bean/->js addresses))
-            data))
-        (catch :default _e              ; TODO: remove this once db goes to test
-          (edn/read-string content))))))
+    (when-let [result (-> (.exec db #js {:sql "select content, addresses from kvs where addr = ?"
+                                         :bind #js [addr]
+                                         :rowMode "array"})
+                          first)]
+      (let [[content addresses] (bean/->clj result)
+            addresses (when addresses
+                        (js/JSON.parse addresses))
+            data (sqlite-util/transit-read content)]
+        (if (and addresses (map? data))
+          (assoc data :addresses addresses)
+          data)))))
 
 (defn new-sqlite-storage
   [repo _opts]
   (reify IStorage
     (-store [_ addr+data-seq delete-addrs]
-      (let [data (map
+      (let [used-addrs (set (mapcat
+                             (fn [[addr data]]
+                               (cons addr
+                                     (when (map? data)
+                                       (:addresses data))))
+                             addr+data-seq))
+            delete-addrs (remove used-addrs delete-addrs)
+            data (map
                   (fn [[addr data]]
-                    #js {:$addr addr
-                         :$content (sqlite-util/transit-write data)})
+                    (let [data' (if (map? data) (dissoc data :addresses) data)
+                          addresses (when (map? data)
+                                      (when-let [addresses (:addresses data)]
+                                        (js/JSON.stringify (bean/->js addresses))))]
+                      #js {:$addr addr
+                           :$content (sqlite-util/transit-write data')
+                           :$addresses addresses}))
                   addr+data-seq)]
         (upsert-addr-content! repo data delete-addrs)))
 
@@ -136,10 +174,22 @@
   [repo]
   (reify IStorage
     (-store [_ addr+data-seq delete-addrs]
-      (let [data (map
+      (let [used-addrs (set (mapcat
+                             (fn [[addr data]]
+                               (cons addr
+                                     (when (map? data)
+                                       (:addresses data))))
+                             addr+data-seq))
+            delete-addrs (remove used-addrs delete-addrs)
+            data (map
                   (fn [[addr data]]
-                    #js {:$addr addr
-                         :$content (sqlite-util/transit-write data)})
+                    (let [data' (if (map? data) (dissoc data :addresses) data)
+                          addresses (when (map? data)
+                                      (when-let [addresses (:addresses data)]
+                                        (js/JSON.stringify (bean/->js addresses))))]
+                      #js {:$addr addr
+                           :$content (sqlite-util/transit-write data')
+                           :$addresses addresses}))
                   addr+data-seq)]
         (upsert-addr-content! repo data delete-addrs :client-ops-db? true)))
 
@@ -201,7 +251,8 @@
       (let [schema (sqlite-util/get-schema repo)
             conn (sqlite-common-db/get-storage-conn storage schema)
             client-ops-conn (sqlite-common-db/get-storage-conn client-ops-storage client-op/schema-in-db)
-            initial-data-exists? (d/entity @conn :logseq.class/Root)]
+            initial-data-exists? (and (d/entity @conn :logseq.class/Root)
+                                      (= "db" (:kv/value (d/entity @conn :logseq.kv/db-type))))]
         (swap! *datascript-conns assoc repo conn)
         (swap! *client-ops-conns assoc repo client-ops-conn)
         (when (and (sqlite-util/db-based-graph? repo) (not initial-data-exists?))
@@ -209,9 +260,12 @@
                 initial-data (sqlite-create-graph/build-db-initial-data config)]
             (d/transact! conn initial-data {:initial-db? true})))
 
-        (when-not (ldb/page-exists? @conn common-config/views-page-name "hidden")
-          (ldb/create-views-page! conn))
+        (try
+          (when-not (ldb/page-exists? @conn common-config/views-page-name "hidden")
+           (ldb/create-views-page! conn))
+          (catch :default _e))
 
+        ;; (gc-kvs-table! db)
         (db-migrate/migrate conn search-db)
 
         (db-listener/listen-db-changes! repo conn)))))