fix: broken db

src/main/frontend/db/restore.cljs

@@ -10,10 +10,10 @@
 
 (defn restore-graph!
   "Restore db from SQLite"
-  [repo]
+  [repo & {:as opts}]
   (state/set-state! :graph/loading? true)
   (p/let [start-time (t/now)
-          data (persist-db/<fetch-init-data repo)
+          data (persist-db/<fetch-init-data repo opts)
           _ (assert (some? data) "No data found when reloading db")
           {:keys [schema initial-data]} (dt/read-transit-str data)
           conn (try

src/main/frontend/handler/import.cljs

@@ -233,7 +233,7 @@
      (p/do!
       (persist-db/<import-db graph buffer)
       (state/add-repo! {:url graph})
-      (repo-handler/restore-and-setup-repo! graph)
+      (repo-handler/restore-and-setup-repo! graph {:import-type "sqlite"})
       (state/set-current-repo! graph)
       (persist-db/<export-db graph {})
       (db/transact! graph (sqlite-util/import-tx :sqlite-db))

src/main/frontend/handler/repo.cljs

@@ -66,10 +66,10 @@
 (defn restore-and-setup-repo!
   "Restore the db of a graph from the persisted data, and setup. Create a new
   conn, or replace the conn in state with a new one."
-  [repo]
+  [repo & {:as opts}]
   (p/do!
    (state/set-db-restoring! true)
-   (db-restore/restore-graph! repo)
+   (db-restore/restore-graph! repo opts)
    (repo-config-handler/restore-repo-config! repo)
    (when (config/global-config-enabled?)
      (global-config-handler/restore-global-config!))

src/main/frontend/persist_db/browser.cljs

@@ -190,13 +190,13 @@
     (when-let [^js sqlite @*worker]
       (.releaseAccessHandles sqlite repo)))
 
-  (<fetch-initial-data [_this repo _opts]
+  (<fetch-initial-data [_this repo opts]
     (when-let [^js sqlite @*worker]
       (-> (p/let [db-exists? (.dbExists sqlite repo)
                   disk-db-data (when-not db-exists? (ipc/ipc :db-get repo))
                   _ (when disk-db-data
                       (.importDb sqlite repo disk-db-data))
-                  _ (.createOrOpenDB sqlite repo (ldb/write-transit-str {}))]
+                  _ (.createOrOpenDB sqlite repo (ldb/write-transit-str opts))]
             (.getInitialData sqlite repo))
           (p/catch sqlite-error-handler))))
 

src/main/frontend/worker/db/migrate.cljs

@@ -16,7 +16,8 @@
             [logseq.common.uuid :as common-uuid]
             [clojure.string :as string]
             [logseq.db.frontend.content :as db-content]
-            [logseq.common.util.page-ref :as page-ref]))
+            [logseq.common.util.page-ref :as page-ref]
+            [datascript.impl.entity :as de]))
 
 ;; TODO: fixes/rollback
 ;; Frontend migrations
@@ -553,6 +554,23 @@
   (when (< db-schema/version max-schema-version)
     (js/console.warn (str "Current db schema-version is " db-schema/version ", max available schema-version is " max-schema-version))))
 
+(defn- ensure-built-in-class-exists!
+  [conn]
+  (let [classes' [:logseq.class/Property :logseq.class/Tag :logseq.class/Page :logseq.class/Journal :logseq.class/Whiteboard]
+        new-classes (->> (select-keys db-class/built-in-classes classes')
+                         ;; class already exists, this should never happen
+                         (remove (fn [[k _]] (d/entity @conn k)))
+                         (into {})
+                         (#(sqlite-create-graph/build-initial-classes* % {}))
+                         (map (fn [b] (assoc b :logseq.property/built-in? true))))
+        new-class-idents (keep (fn [class]
+                                 (when-let [db-ident (:db/ident class)]
+                                   {:db/ident db-ident})) new-classes)
+        tx-data (concat new-class-idents new-classes)]
+    (when (seq tx-data)
+      (d/transact! conn tx-data {:fix-db? true
+                                 :db-migrate? true}))))
+
 (defn- upgrade-version!
   [conn search-db db-based? version {:keys [properties classes fix]}]
   (let [db @conn
@@ -584,6 +602,73 @@
     (ldb/transact! conn tx-data' {:db-migrate? true})
     (println "DB schema migrated to" version)))
 
+(defn fix-path-refs!
+  [conn]
+  (let [data (keep
+              (fn [d]
+                (when (not (de/entity? (d/entity @conn (:v d))))
+                  [:db/retract (:e d) (:a d) (:v d)]))
+              (d/datoms @conn :avet :block/path-refs))]
+    (when (seq data)
+      (ldb/transact! conn data {:fix-db? true
+                                :db-migrate? true}))))
+
+(defn fix-missing-title!
+  [conn]
+  (let [data (->>
+              (mapcat
+               (fn [d]
+                 (let [entity (d/entity @conn (:e d))]
+                   [(when-not (:block/title entity)
+                      [:db/add (:e d) :block/title (:v d)])
+                    (when-not (:block/uuid entity)
+                      [:db/add (:e d) :block/uuid (d/squuid)])
+                    (when-not (:block/format entity)
+                      [:db/add (:e d) :block/format :markdown])]))
+               (d/datoms @conn :avet :block/name))
+              (remove nil?))]
+    (when (seq data)
+      (ldb/transact! conn data {:fix-db? true
+                                :db-migrate? true}))))
+
+(defn fix-block-timestamps!
+  [conn]
+  (let [data (map
+              (fn [d]
+                (let [entity (d/entity @conn (:e d))]
+                  (when (or (nil? (:block/created-at entity))
+                            (nil? (:block/updated-at entity)))
+                    (-> (select-keys entity [:db/id :block/created-at :block/updated-at])
+                        sqlite-util/block-with-timestamps))))
+              (d/datoms @conn :avet :block/uuid))]
+    (when (seq data)
+      (ldb/transact! conn data {:fix-db? true
+                                :db-migrate? true}))))
+
+(defn fix-properties!
+  [conn]
+  (let [schema (:schema @conn)
+        wrong-properties (filter (fn [[k v]]
+                                   (and (int? k) (not (qualified-ident? v)))) schema)
+        data (map (fn [[k _v]]
+                    [:db/retract k :db/valueType]) wrong-properties)]
+    (when (seq data)
+      (ldb/transact! conn data {:fix-db? true
+                                :db-migrate? true})
+      (d/reset-schema! conn (apply dissoc schema (keys wrong-properties))))))
+
+(defn fix-missing-page-tag!
+  [conn]
+  (let [data (keep
+              (fn [d]
+                (let [entity (d/entity @conn (:e d))]
+                  (when-not (:block/tags entity)
+                    [:db/add (:e d) :block/tags :logseq.class/Page])))
+              (d/datoms @conn :avet :block/name))]
+    (when (seq data)
+      (ldb/transact! conn data {:fix-db? true
+                                :db-migrate? true}))))
+
 (defn migrate
   "Migrate 'frontend' datascript schema and data. To add a new migration,
   add an entry to schema-version->updates and bump db-schema/version"
@@ -606,14 +691,28 @@
                                 (when (and (< version-in-db v) (<= v db-schema/version))
                                   [v updates]))
                               schema-version->updates)]
+            (fix-path-refs! conn)
+            (fix-missing-title! conn)
+            (fix-properties! conn)
+            (fix-block-timestamps! conn)
             (println "DB schema migrated from" version-in-db)
             (doseq [[v m] updates]
-              (upgrade-version! conn search-db db-based? v m)))
+              (upgrade-version! conn search-db db-based? v m))
+            (fix-missing-page-tag! conn))
           (catch :default e
             (prn :error (str "DB migration failed to migrate to " db-schema/version " from " version-in-db ":"))
             (js/console.error e)
             (throw e)))))))
 
+(defn fix-db!
+  [conn]
+  (ensure-built-in-class-exists! conn)
+  (fix-path-refs! conn)
+  (fix-missing-title! conn)
+  (fix-properties! conn)
+  (fix-block-timestamps! conn)
+  (fix-missing-page-tag! conn))
+
 ;; Backend migrations
 ;; ==================
 

src/main/frontend/worker/db_worker.cljs

@@ -37,7 +37,8 @@
             [logseq.outliner.op :as outliner-op]
             [me.tonsky.persistent-sorted-set :as set :refer [BTSet]]
             [promesa.core :as p]
-            [shadow.cljs.modern :refer [defclass]]))
+            [shadow.cljs.modern :refer [defclass]]
+            [datascript.impl.entity :as de]))
 
 (defonce *sqlite worker-state/*sqlite)
 (defonce *sqlite-conns worker-state/*sqlite-conns)
@@ -108,7 +109,7 @@
 
 (defn- rebuild-db-from-datoms!
   "Persistent-sorted-set has been broken, used addresses can't be found"
-  [datascript-conn sqlite-db]
+  [datascript-conn sqlite-db import-type]
   (let [datoms (get-all-datoms-from-sqlite-db sqlite-db)
         db (d/init-db [] db-schema/schema-for-db-based-graph
                       {:storage (storage/storage @datascript-conn)})
@@ -117,10 +118,12 @@
                              [:db/add (:e d) (:a d) (:v d) (:t d)]) datoms))]
     (prn :debug :rebuild-db-from-datoms :datoms-count (count datoms))
     ;; export db first
-    (worker-util/post-message :notification ["The SQLite db will be exported to avoid any data-loss." :warning false])
-    (worker-util/post-message :export-current-db [])
+    (when-not import-type
+      (worker-util/post-message :notification ["The SQLite db will be exported to avoid any data-loss." :warning false])
+      (worker-util/post-message :export-current-db []))
     (.exec sqlite-db #js {:sql "delete from kvs"})
-    (d/reset-conn! datascript-conn db)))
+    (d/reset-conn! datascript-conn db)
+    (db-migrate/fix-db! datascript-conn)))
 
 (comment
   (defn- gc-kvs-table!
@@ -327,7 +330,7 @@
           (db-migrate/migrate conn search-db)
           (catch :default _e
             (when db-based?
-              (rebuild-db-from-datoms! conn db))))
+              (rebuild-db-from-datoms! conn db import-type))))
 
         (db-listener/listen-db-changes! repo (get @*datascript-conns repo))))))