validate entities that are changed per transact!

This should help us catch bugs due to invalid data fairly quickly.
This only runs when in dev?
part of LOG-2818

deps/db/script/validate_client_db.cljs

@@ -86,16 +86,6 @@
         (js/process.exit 1))
       (println "Valid!"))))
 
-(defn- datoms->entity-maps
-  "Returns entity maps for given :eavt datoms"
-  [datoms]
-  (->> datoms
-       (reduce (fn [acc m]
-                 (if (contains? db-schema/card-many-attributes (:a m))
-                   (update acc (:e m) update (:a m) (fnil conj #{}) (:v m))
-                   (update acc (:e m) assoc (:a m) (:v m))))
-               {})))
-
 (def spec
   "Options spec"
   {:help {:alias :h
@@ -119,7 +109,7 @@
                  (js/process.exit 1)))
         conn (sqlite-cli/read-graph db-name)
         datoms (d/datoms @conn :eavt)
-        ent-maps (datoms->entity-maps datoms)]
+        ent-maps (db-malli-schema/datoms->entity-maps datoms)]
     (println "Read graph" (str db-name " with " (count datoms) " datoms, "
                                (count ent-maps) " entities and "
                                (count (mapcat :block/properties (vals ent-maps))) " properties"))

deps/db/src/logseq/db/frontend/malli_schema.cljs

@@ -2,12 +2,13 @@
   "Malli schemas and fns for logseq.db.frontend.*"
   (:require [clojure.walk :as walk]
             [datascript.core :as d]
+            [logseq.db.frontend.schema :as db-schema]
             [logseq.db.frontend.property :as db-property]
             [logseq.db.frontend.property.type :as db-property-type]))
 
 ;; Helper fns
 ;; ==========
-(defn validate-property-value
+(defn- validate-property-value
   "Validates the value in a property tuple. The property value can be one or
   many of a value to validated"
   [prop-type schema-fn val]
@@ -41,6 +42,16 @@
           %)
        ents))
 
+(defn datoms->entity-maps
+  "Returns entity maps for given :eavt datoms"
+  [datoms]
+  (->> datoms
+       (reduce (fn [acc m]
+                 (if (contains? db-schema/card-many-attributes (:a m))
+                   (update acc (:e m) update (:a m) (fnil conj #{}) (:v m))
+                   (update acc (:e m) assoc (:a m) (:v m))))
+               {})))
+
 ;; Malli schemas
 ;; =============
 ;; These schemas should be data vars to remain as simple and reusable as possible

src/main/frontend/modules/outliner/datascript.cljs

@@ -11,8 +11,11 @@
             [clojure.string :as string]
             [frontend.util :as util]
             [logseq.graph-parser.util.block-ref :as block-ref]
+            [logseq.db.frontend.malli-schema :as db-malli-schema]
             [frontend.db.fix :as db-fix]
-            [frontend.handler.file-based.property.util :as property-util]))
+            [frontend.handler.file-based.property.util :as property-util]
+            [cljs.pprint :as pprint]
+            [malli.core :as m]))
 
 (defn new-outliner-txs-state [] (atom []))
 
@@ -22,10 +25,29 @@
    (instance? cljs.core/Atom state)
    (coll? @state)))
 
+(defn- validate-db!
+  [{:keys [db-after tx-data tx-meta]}]
+  (let [changed-ids (->> tx-data (map :e) distinct)
+        ent-maps* (->> changed-ids (mapcat #(d/datoms db-after :eavt %)) db-malli-schema/datoms->entity-maps vals)
+        ent-maps (vec (db-malli-schema/update-properties-in-ents ent-maps*))
+        db-schema (db-malli-schema/update-properties-in-schema db-malli-schema/DB db-after)]
+    (js/console.log "changed eids:" changed-ids tx-meta)
+    (when-let [errors (->> ent-maps
+                           (m/explain db-schema)
+                           :errors)]
+      (js/console.error "Invalid datascript entities detected amongst changed entity ids:" changed-ids)
+      (pprint/pprint {:errors errors})
+      (pprint/pprint {:entity-maps ent-maps})
+      ;; (js/alert "Invalid DB!")
+      )))
+
 (defn after-transact-pipelines
   [repo {:keys [_db-before _db-after _tx-data _tempids tx-meta] :as tx-report}]
   (when-not config/test?
     (pipelines/invoke-hooks tx-report)
+    ;; Skip tx with update-tx-ids? because they are immediately followed by the original block tx
+    (when (and config/dev? (not (:update-tx-ids? tx-meta)))
+      (validate-db! tx-report))
 
     (when (or (:outliner/transact? tx-meta)
               (:outliner-op tx-meta)
@@ -118,7 +140,7 @@
       (concat txs retracted-tx'))
     txs))
 
-(defn validate-db!
+(defn fix-db!
   [{:keys [db-before db-after tx-data]}]
   (let [changed-pages (->> (filter (fn [d] (contains? #{:block/left :block/parent} (:a d))) tx-data)
                            (map :e)
@@ -168,7 +190,7 @@
               rs (db/transact! repo txs (assoc opts :outliner/transact? true))
               tx-id (get-tx-id rs)]
           ;; TODO: disable this when db is stable
-          (when (and config/dev? (not util/node-test?)) (validate-db! rs))
+          (when (and config/dev? (not util/node-test?)) (fix-db! rs))
           (state/update-state! :history/tx->editor-cursor
                                (fn [m] (assoc m tx-id before-editor-cursor)))