fix: graph counts after validation are wrong

after type->tags migration. Also update recommended validate cli options
to default to true. Also clean up graph counts spread across two
validators and importer. The property-pairs count in the UI were way off
because they were only counting pairs that are validated by malli-schema

deps/db/script/validate_client_db.cljs

@@ -4,7 +4,6 @@
   (:require [logseq.db.sqlite.cli :as sqlite-cli]
             [logseq.db.frontend.malli-schema :as db-malli-schema]
             [logseq.db.frontend.validate :as db-validate]
-            [logseq.db.frontend.property :as db-property]
             [datascript.core :as d]
             [clojure.string :as string]
             [nbb.core :as nbb]
@@ -13,8 +12,7 @@
             ["path" :as node-path]
             ["os" :as os]
             [cljs.pprint :as pprint]
-            [malli.error :as me]
-            [logseq.db :as ldb]))
+            [malli.error :as me]))
 
 (defn validate-client-db
   "Validate datascript db as a vec of entity maps"
@@ -57,12 +55,15 @@
   {:help {:alias :h
           :desc "Print help"}
    :humanize {:alias :H
+              :default true
               :desc "Humanize errors as an alternative to -v"}
    :verbose {:alias :v
              :desc "Print more info"}
    :closed-maps {:alias :c
+                 :default true
                  :desc "Validate maps marked with closed as :closed"}
    :group-errors {:alias :g
+                  :default true
                   :desc "Groups errors by their entity id"}})
 
 (defn- validate-graph [graph-dir options]
@@ -77,14 +78,9 @@
                     (js/process.exit 1)))
         datoms (d/datoms @conn :eavt)
         ent-maps (db-malli-schema/datoms->entities datoms)]
-    (println "Read graph" (str db-name " with " (count datoms) " datoms, "
-                               (count ent-maps) " entities, "
-                               (count (filter :block/name ent-maps)) " pages, "
-                               (count (filter :block/title ent-maps)) " blocks, "
-                               (count (filter ldb/class? ent-maps)) " tags, "
-                               (count (filter #(seq (:block/tags %)) ent-maps)) " objects, "
-                               (count (filter ldb/property? ent-maps)) " properties and "
-                               (count (mapcat db-property/properties ent-maps)) " property pairs"))
+    (println "Read graph" (str db-name " with counts: "
+                               (pr-str (assoc (db-validate/graph-counts @conn ent-maps)
+                                              :datoms (count datoms)))))
     (validate-client-db @conn ent-maps options)))
 
 (defn -main [argv]

deps/db/src/logseq/db/frontend/validate.cljs

@@ -2,6 +2,7 @@
   "Validate frontend db for DB graphs"
   (:require [datascript.core :as d]
             [logseq.db.frontend.malli-schema :as db-malli-schema]
+            [logseq.db.frontend.property :as db-property]
             [malli.core :as m]
             [malli.error :as me]
             [malli.util :as mu]
@@ -103,8 +104,23 @@
         errors (binding [db-malli-schema/*db-for-validate-fns* db]
                  (-> ent-maps closed-db-schema-explainer :errors))]
     (cond-> {:datom-count (count datoms)
-             :entities ent-maps}
+             :entities ent-maps*}
       (some? errors)
       (assoc :errors (map #(-> (dissoc % :errors-by-type)
                                (update :errors (fn [errs] (me/humanize {:errors errs}))))
                           (group-errors-by-entity db ent-maps errors))))))
+
+(defn graph-counts
+  "Calculates graph-wide counts given a graph's db and its entities from :eavt datoms"
+  [db entities]
+  (let [classes-count (count (d/datoms db :avet :block/tags :logseq.class/Tag))
+        properties-count (count (d/datoms db :avet :block/tags :logseq.class/Property))]
+    {:entities (count entities)
+     :pages (count (filter :block/name entities))
+     ;; Nodes that aren't pages
+     :blocks (count (filter :block/parent entities))
+     :classes classes-count
+     :properties properties-count
+     ;; Objects that aren't classes or properties
+     :objects (- (count (d/datoms db :avet :block/tags)) classes-count properties-count)
+     :property-pairs (count (mapcat #(-> % db-property/properties (dissoc :block/tags)) entities))}))

docs/dev-practices.md

@@ -365,7 +365,7 @@ These tasks are specific to database graphs. For these tasks there is a one time
 
   ```sh
   # One or more graphs can be validated e.g.
-  $ bb dev:validate-db test-db schema -c -g
+  $ bb dev:validate-db test-db schema
   Read graph test-db with 1572 datoms, 220 entities and 13 properties
   Valid!
   Read graph schema with 26105 datoms, 2320 entities and 3168 properties

src/main/frontend/components/imports.cljs

@@ -31,8 +31,7 @@
             [logseq.shui.dialog.core :as shui-dialog]
             [logseq.shui.form.core :as form-core]
             [lambdaisland.glogi :as log]
-            [logseq.db.frontend.validate :as db-validate]
-            [logseq.db :as ldb]))
+            [logseq.db.frontend.validate :as db-validate]))
 
 ;; Can't name this component as `frontend.components.import` since shadow-cljs
 ;; will complain about it.
@@ -278,16 +277,6 @@
 
                           (shui/button {:type "submit" :class "right-0 mt-3"} "Submit")]))])
 
-(defn- counts-from-entities
-  [entities]
-  {:entities (count entities)
-   :pages (count (filter :block/name entities))
-   :blocks (count (filter :block/title entities))
-   :classes (count (filter ldb/class? entities))
-   :objects (count (filter #(seq (:block/tags %)) entities))
-   :properties (count (filter ldb/property? entities))
-   :property-values (count (mapcat :block/properties entities))})
-
 (defn- validate-imported-data
   [db import-state files]
   (when-let [org-files (seq (filter #(= "org" (path/file-ext (:path %))) files))]
@@ -320,12 +309,12 @@
     (if errors
       (do
         (log/error :import-errors {:msg (str "Import detected " (count errors) " invalid block(s):")
-                                   :counts (assoc (counts-from-entities entities) :datoms datom-count)})
+                                   :counts (assoc (db-validate/graph-counts db entities) :datoms datom-count)})
         (pprint/pprint errors)
         (notification/show! (str "Import detected " (count errors) " invalid block(s). These blocks may be buggy when you interact with them. See the javascript console for more.")
                             :warning false))
       (log/info :import-valid {:msg "Valid import!"
-                               :counts (assoc (counts-from-entities entities) :datoms datom-count)}))))
+                               :counts (assoc (db-validate/graph-counts db entities) :datoms datom-count)}))))
 
 (defn- show-notification [{:keys [msg level ex-data]}]
   (if (= :error level)

src/main/frontend/worker/db/validate.cljs

@@ -2,19 +2,8 @@
   "Validate db"
   (:require [cljs.pprint :as pprint]
             [frontend.worker.util :as worker-util]
-            [logseq.db :as ldb]
             [logseq.db.frontend.validate :as db-validate]))
 
-(defn- counts-from-entities
-  [entities]
-  {:entities (count entities)
-   :pages (count (filter :block/name entities))
-   :blocks (count (filter :block/title entities))
-   :classes (count (filter ldb/class? entities))
-   :objects (count (filter #(seq (:block/tags %)) entities))
-   :properties (count (filter ldb/property? entities))
-   :property-values (count (mapcat :block/properties entities))})
-
 (defn validate-db
   [db]
   (let [{:keys [errors datom-count entities]} (db-validate/validate-db! db)]
@@ -22,12 +11,12 @@
       (do
         (worker-util/post-message :log [:db-invalid :error
                                         {:msg (str "Validation detected " (count errors) " invalid block(s):")
-                                         :counts (assoc (counts-from-entities entities) :datoms datom-count)}])
+                                         :counts (assoc (db-validate/graph-counts db entities) :datoms datom-count)}])
         (pprint/pprint errors)
         (worker-util/post-message :notification
                                   [(str "Validation detected " (count errors) " invalid block(s). These blocks may be buggy when you interact with them. See the javascript console for more.")
                                    :warning false]))
 
       (worker-util/post-message :notification
-                                [(str "Your graph is valid! " (assoc (counts-from-entities entities) :datoms datom-count))
-                                 :success false]))))
+                                [(str "Your graph is valid! " (assoc (db-validate/graph-counts db entities) :datoms datom-count))
+                                 :success false]))))