fix: invalid data

/Validate current graph will try to fix those errors.

fixes https://github.com/logseq/db-test/issues/205.

deps/db/script/validate_client_db.cljs

@@ -20,7 +20,7 @@
   (let [ent-maps (db-malli-schema/update-properties-in-ents db ent-maps*)
         explainer (db-validate/get-schema-explainer closed-maps)]
     (if-let [explanation (binding [db-malli-schema/*db-for-validate-fns* db]
-                           (->> ent-maps explainer not-empty))]
+                           (->> (map (fn [e] (dissoc e :db/id)) ent-maps) explainer not-empty))]
       (do
         (if group-errors
           (let [ent-errors (db-validate/group-errors-by-entity db ent-maps (:errors explanation))]

deps/db/src/logseq/db/frontend/malli_schema.cljs

@@ -194,7 +194,7 @@
 (defn datoms->entities
   "Returns a vec of entity maps given :eavt datoms"
   [datoms]
-  (mapv (fn [[db-id m]] (with-meta m {:db/id db-id}))
+  (mapv (fn [[db-id m]] (assoc m :db/id db-id))
         (datoms->entity-maps datoms)))
 
 (assert (every? #(re-find #"^(block|logseq\.)" (namespace %)) db-property/db-attribute-properties)
@@ -393,6 +393,7 @@
   [:map
    [:block/uuid :uuid]
    [:block/created-at :int]
+   [:block/updated-at {:optional true} :int]
    [:logseq.property.history/block :int]
    [:logseq.property.history/property :int]
    [:logseq.property.history/ref-value {:optional true} :int]
@@ -441,10 +442,7 @@
   "A block has content and a page"
   [:or
    normal-block
-   closed-value-block
-   whiteboard-block
-   property-value-block
-   property-history-block])
+   whiteboard-block])
 
 (def asset-block
   "A block tagged with #Asset"
@@ -504,6 +502,16 @@
                                              :asset-block
                                              (:file/path d)
                                              :file-block
+                                             (:logseq.property.history/block d)
+                                             :property-history-block
+
+                                             (:block/closed-value-property d)
+                                             :closed-value-block
+
+                                             (and (:logseq.property/created-from-property d)
+                                                  (:property.value/content d))
+                                             :property-value-block
+
                                              (:block/uuid d)
                                              :block
                                              (= (:db/ident d) :logseq.property/empty-placeholder)
@@ -515,6 +523,9 @@
     :class class-page
     :hidden hidden-page
     :normal-page normal-page
+    :property-history-block property-history-block
+    :closed-value-block closed-value-block
+    :property-value-block property-value-block
     :block block
     :asset-block asset-block
     :file-block file-block

deps/db/src/logseq/db/frontend/validate.cljs

@@ -48,7 +48,7 @@
                                                             [(:db/ident p) v])
                                                           properties)))
                     data {:entity-map m'
-                          :errors (me/humanize (explainer [m]))}]
+                          :errors (me/humanize (explainer [(dissoc m :db/id)]))}]
                 (try
                   (pprint/pprint data)
                   (catch :default _e
@@ -63,11 +63,8 @@
   (->> errors
        (group-by #(-> % :in first))
        (map (fn [[idx errors']]
-              {:entity (let [ent (get ent-maps idx)
-                             db-id (:db/id (meta ent))]
+              {:entity (let [ent (get ent-maps idx)]
                          (cond-> ent
-                           db-id
-                           (assoc :db/id db-id)
                            ;; Provide additional page info for debugging
                            (:block/page ent)
                            (update :block/page
@@ -102,7 +99,9 @@
                   #(dissoc % :block.temp/fully-loaded?)
                   (db-malli-schema/update-properties-in-ents db ent-maps*))
         errors (binding [db-malli-schema/*db-for-validate-fns* db]
-                 (-> ent-maps closed-db-schema-explainer :errors))]
+                 (-> (map (fn [e]
+                            (dissoc e :db/id))
+                          ent-maps) closed-db-schema-explainer :errors))]
     (cond-> {:datom-count (count datoms)
              :entities ent-maps*}
       (some? errors)
@@ -123,4 +122,4 @@
      :properties properties-count
      ;; Objects that aren't classes or properties
      :objects (- (count (d/datoms db :avet :block/tags)) classes-count properties-count)
-     :property-pairs (count (mapcat #(-> % db-property/properties (dissoc :block/tags)) entities))}))
+     :property-pairs (count (mapcat #(-> % db-property/properties (dissoc :block/tags)) entities))}))

src/main/frontend/worker/db/migrate.cljs

@@ -476,8 +476,8 @@
               [:db/retract (:e d) :logseq.task/deadline]))
           datoms))))))
 
-(defn- remove-block-format-from-db
-  [conn _search-db]
+(defn- remove-block-format-from-db!
+  [conn]
   (let [db @conn]
     (when (ldb/db-based-graph? db)
       (let [datoms (d/datoms db :avet :block/uuid)
@@ -486,8 +486,7 @@
                        [:db/retract (:e d) :block/format])
                      datoms)]
         (ldb/transact! conn tx-data {:db-migrate? true})
-        (d/reset-schema! conn (dissoc (:schema db) :block/format))
-        []))))
+        (d/reset-schema! conn (dissoc (:schema db) :block/format))))))
 
 (defn- remove-duplicated-contents-page
   [conn _search-db]
@@ -625,7 +624,6 @@
    [56 {:properties [:logseq.property/enable-history?
                      :logseq.property.history/block :logseq.property.history/property
                      :logseq.property.history/ref-value :logseq.property.history/scalar-value]}]
-   [57 {:fix remove-block-format-from-db}]
    [58 {:fix remove-duplicated-contents-page}]])
 
 (let [max-schema-version (apply max (map first schema-version->updates))]
@@ -736,9 +734,7 @@
                    [(when-not (:block/title entity)
                       [:db/add (:e d) :block/title (:v d)])
                     (when-not (:block/uuid entity)
-                      [:db/add (:e d) :block/uuid (d/squuid)])
-                    (when-not (:block/format entity)
-                      [:db/add (:e d) :block/format :markdown])]))
+                      [:db/add (:e d) :block/uuid (d/squuid)])]))
                (d/datoms @conn :avet :block/name))
               (remove nil?))]
     (when (seq data)
@@ -807,6 +803,7 @@
                               schema-version->updates)]
             (fix-path-refs! conn)
             (fix-missing-title! conn)
+            (remove-block-format-from-db! conn)
             (fix-properties! conn)
             (fix-block-timestamps! conn)
             (println "DB schema migrated from" version-in-db)
@@ -819,14 +816,112 @@
             (js/console.error e)
             (throw e)))))))
 
+(defn fix-invalid-data!
+  [conn invalid-entity-ids]
+  (let [db @conn
+        tx-data (->>
+                 (mapcat
+                  (fn [id]
+                    (let [entity (d/entity db id)
+                         ;; choice not included in closed values
+                          wrong-choice (keep
+                                        (fn [[k v]]
+                                          (if (= "block.temp" (namespace k))
+                                            [:db/retract (:db/id entity) k]
+                                            (when-let [property (d/entity db k)]
+                                              (let [closed-values (:property/closed-values property)]
+                                                (when (seq closed-values)
+                                                  (if (and (de/entity? v)
+                                                           (not (contains? (set (map :db/id closed-values)) (:db/id v))))
+                                                    [:db/retractEntity (:db/id v)]
+                                                    [:db/retract (:db/id entity) k]))))))
+                                        (into {} entity))
+                          eid (:db/id entity)
+                          fix (cond
+                                (= #{:block/tx-id} (set (keys entity)))
+                                [[:db/retractEntity (:db/id entity)]]
+
+                                (and (seq (:block/refs entity))
+                                     (not (or (:block/title entity) (:block/content entity) (:property.value/content entity))))
+                                [[:db/retractEntity (:db/id entity)]]
+
+                                (:logseq.property.node/type entity)
+                                [[:db/retract eid :logseq.property.node/type]
+                                 [:db/retractEntity :logseq.property.node/type]
+                                 [:db/add eid :logseq.property.node/display-type (:logseq.property.node/type entity)]]
+
+                                (and (:db/cardinality entity) (not (ldb/property? entity)))
+                                [[:db/add eid :block/tags :logseq.class/Property]
+                                 [:db/retract eid :block/tags :logseq.class/Page]]
+
+                                (when-let [schema (:block/schema entity)]
+                                  (or (:cardinality schema) (:classes schema)))
+                                (let [schema (:block/schema entity)]
+                                  [[:db/add eid :block/schema (dissoc schema :cardinality :classes)]])
+
+                                (and (:logseq.property.asset/type entity)
+                                     (or (nil? (:logseq.property.asset/checksum entity))
+                                         (nil? (:logseq.property.asset/size entity))))
+                                [[:db/retractEntity eid]]
+
+                                ;; add missing :db/ident for classes && properties
+                                (and (ldb/class? entity) (nil? (:db/ident entity)))
+                                [[:db/add (:db/id entity) :db/ident (db-class/create-user-class-ident-from-name (:block/title entity))]]
+
+                                (and (ldb/property? entity) (nil? (:db/ident entity)))
+                                [[:db/add (:db/id entity) :db/ident (db-property/create-user-property-ident-from-name (:block/title entity))]]
+
+                                ;; remove #Page for classes/properties/journals
+                                (and (ldb/internal-page? entity) (or (ldb/class? entity) (ldb/property? entity) (ldb/journal? entity)))
+                                [[:db/retract (:db/id entity) :block/tags :logseq.class/Page]]
+
+                                ;; remove file entities
+                                (and (:file/path entity)
+                                     (not (contains? #{"logseq/custom.css" "logseq/config.js"  "logseq/config.edn"} (:file/path entity))))
+                                [[:db/retractEntity (:db/id entity)]]
+
+                                (:block/properties-order entity)
+                                [[:db/retract (:db/id entity) :block/properties-order]]
+
+                                (:block/macros entity)
+                                [[:db/retract (:db/id entity) :block/macros]]
+
+                                (and (seq (:block/tags entity)) (not (every? ldb/class? (:block/tags entity))))
+                                (let [tags (remove ldb/class? (:block/tags entity))]
+                                  (map
+                                   (fn [tag]
+                                     {:db/id (:db/id tag)
+                                      :db/ident (or (:db/ident tag) (db-class/create-user-class-ident-from-name (:block/title entity)))
+                                      :block/tags :logseq.class/Tag})
+                                   tags))
+                                :else
+                                nil)]
+                      (into fix wrong-choice)))
+                  invalid-entity-ids)
+                 distinct)]
+    (when (seq tx-data)
+      (d/transact! conn tx-data {:fix-db? true}))))
+
 (defn fix-db!
-  [conn]
-  (ensure-built-in-data-exists! conn)
-  (fix-path-refs! conn)
-  (fix-missing-title! conn)
-  (fix-properties! conn)
-  (fix-block-timestamps! conn)
-  (fix-missing-page-tag! conn))
+  [conn & {:keys [invalid-entity-ids]}]
+  (when (ldb/db-based-graph? @conn)
+    (try
+      (ensure-built-in-data-exists! conn)
+      (remove-block-format-from-db! conn)
+      (fix-path-refs! conn)
+      (fix-missing-title! conn)
+      (fix-properties! conn)
+      (fix-block-timestamps! conn)
+      (fix-missing-page-tag! conn)
+      (when (seq invalid-entity-ids)
+        (fix-invalid-data! conn invalid-entity-ids))
+
+      ;; TODO: remove this after RTC db fixed
+      (let [data (deprecate-logseq-user-ns conn nil)]
+        (when (seq data)
+          (d/transact! conn data {:fix-db? true})))
+      (catch :default e
+        (js/console.error e)))))
 
 ;; Backend migrations
 ;; ==================

src/main/frontend/worker/db/validate.cljs

@@ -9,13 +9,15 @@
     (if errors
       (do
         (worker-util/post-message :log [:db-invalid :error
-                                        {:msg (str "Validation detected " (count errors) " invalid block(s):")
-                                         :counts (assoc (db-validate/graph-counts db entities) :datoms datom-count)}])
-        (prn errors)
+                                        {:msg "Validation errors"
+                                         :errors errors}])
         (worker-util/post-message :notification
                                   [(str "Validation detected " (count errors) " invalid block(s). These blocks may be buggy when you interact with them. See the javascript console for more.")
                                    :warning false]))
 
       (worker-util/post-message :notification
                                 [(str "Your graph is valid! " (assoc (db-validate/graph-counts db entities) :datoms datom-count))
-                                 :success false]))))
+                                 :success false]))
+    {:errors errors
+     :datom-count datom-count
+     :invalid-entity-ids (distinct (map (fn [e] (:db/id (:entity e))) errors))}))

src/main/frontend/worker/db_worker.cljs

@@ -912,7 +912,9 @@
   (validate-db
    [_this repo]
    (when-let [conn (worker-state/get-datascript-conn repo)]
-     (worker-db-validate/validate-db @conn)))
+     (let [result (worker-db-validate/validate-db @conn)]
+       (db-migrate/fix-db! conn {:invalid-entity-ids (:invalid-entity-ids result)})
+       nil)))
 
   (dangerousRemoveAllDbs
    [this repo]

src/main/frontend/worker/pipeline.cljs

@@ -144,21 +144,22 @@
                        (rebuild-block-refs repo tx-report* blocks'))
           refs-tx-report (when (seq block-refs)
                            (ldb/transact! conn block-refs {:pipeline-replace? true}))
-          replace-tx (concat
+          replace-tx (let [db-after (or (:db-after refs-tx-report) (:db-after tx-report*))]
+                       (concat
                       ;; block path refs
-                      (when (seq blocks')
-                        (let [db-after (or (:db-after refs-tx-report) (:db-after tx-report*))
-                              blocks' (keep (fn [b] (d/entity db-after (:db/id b))) blocks')]
-                          (compute-block-path-refs-tx tx-report* blocks')))
+                        (when (seq blocks')
+                          (let [blocks' (keep (fn [b] (d/entity db-after (:db/id b))) blocks')]
+                            (compute-block-path-refs-tx tx-report* blocks')))
 
                        ;; update block/tx-id
-                      (let [updated-blocks (remove (fn [b] (contains? (set deleted-block-uuids) (:block/uuid b)))
-                                                   (concat pages blocks))
-                            tx-id (get-in (or refs-tx-report tx-report*) [:tempids :db/current-tx])]
-                        (keep (fn [b]
-                                (when-let [db-id (:db/id b)]
-                                  {:db/id db-id
-                                   :block/tx-id tx-id})) updated-blocks)))
+                        (let [updated-blocks (remove (fn [b] (contains? (set deleted-block-uuids) (:block/uuid b)))
+                                                     (concat pages blocks))
+                              tx-id (get-in (or refs-tx-report tx-report*) [:tempids :db/current-tx])]
+                          (keep (fn [b]
+                                  (when-let [db-id (:db/id b)]
+                                    (when (:block/uuid (d/entity db-after db-id))
+                                      {:db/id db-id
+                                       :block/tx-id tx-id}))) updated-blocks))))
           tx-report' (if (seq replace-tx)
                        (ldb/transact! conn replace-tx {:pipeline-replace? true})
                        (do