add bb dev:lint-test-deps to run lint and tests for deps

bb.edn

@@ -159,6 +159,9 @@
   dev:lint-and-test
   logseq.tasks.dev/lint-and-test
 
+  dev:lint-test-deps
+  logseq.tasks.dev.lint-test-deps/run
+
   dev:e2e-basic-test
   logseq.tasks.dev/e2e-basic-test
 

deps/db-sync/.carve/config.edn

@@ -0,0 +1,5 @@
+{:paths ["src"]
+ :api-namespaces [;; CLI entry points
+                  logseq.db-sync.node.find-checksum-drift
+                  logseq.db-sync.node.recompute-log-checksum]
+ :report {:format :ignore}}

deps/db/src/logseq/db.cljs

@@ -106,20 +106,46 @@
                  (or @*debounce-fn d/store))]
          (f @conn))))))
 
+(defn- should-validate-tx?
+  [conn db tx-meta]
+  (and (entity-plus/db-based-graph? db)
+       (not
+        (or (:rtc-download-graph? tx-meta)
+            (:reset-conn! tx-meta)
+            (:initial-db? tx-meta)
+            (:skip-validate-db? tx-meta false)
+            ;; used by `batch-transact-with-temp-conn!`
+            (:skip-validate-db? @conn)
+            (:logseq.graph-parser.exporter/new-graph? tx-meta)))))
+
+(defn- tx-report-pipeline-data
+  [tx-report]
+  (map (fn [[e a v t]]
+         [e a v t])
+       (:tx-data tx-report)))
+
+(defn- invalid-tx-debug-data
+  [tx-meta tx-data errors tx-report]
+  {:tx-meta tx-meta
+   :tx-data tx-data
+   :errors errors
+   :pipeline-tx-data (tx-report-pipeline-data tx-report)})
+
+(defn- throw-invalid-tx!
+  [tx-meta tx-data errors tx-report]
+  ;; notify ui
+  (when-let [f @*transact-invalid-callback]
+    (f tx-report errors))
+  (let [debug-data (invalid-tx-debug-data tx-meta tx-data errors tx-report)]
+    (prn :debug :invalid-data debug-data)
+    (throw (ex-info "DB write failed with invalid data" debug-data))))
+
 (defn- transact-sync
   [conn tx-data tx-meta]
   (try
     (let [db @conn
-          db-based? (entity-plus/db-based-graph? db)]
-      (if (and db-based?
-               (not
-                (or (:rtc-download-graph? tx-meta)
-                    (:reset-conn! tx-meta)
-                    (:initial-db? tx-meta)
-                    (:skip-validate-db? tx-meta false)
-                    ;; used by `batch-transact-with-temp-conn!`
-                    (:skip-validate-db? @conn)
-                    (:logseq.graph-parser.exporter/new-graph? tx-meta))))
+          validate? (should-validate-tx? conn db tx-meta)]
+      (if validate?
         (let [tx-report* (d/with db tx-data tx-meta)
               pipeline-f @*transact-pipeline-fn
               tx-report (if pipeline-f (pipeline-f tx-report*) tx-report*)
@@ -135,25 +161,7 @@
               (dc/run-callbacks conn tx-report))
 
             :else
-            (do
-              ;; notify ui
-              (when-let [f @*transact-invalid-callback]
-                (f tx-report errors))
-              (prn :debug :invalid-data
-                   {:tx-meta tx-meta
-                    :tx-data tx-data
-                    :errors errors
-                    :pipeline-tx-data (map
-                                       (fn [[e a v t]]
-                                         [e a v t])
-                                       (:tx-data tx-report))})
-              (throw (ex-info "DB write failed with invalid data" {:tx-meta tx-meta
-                                                                   :tx-data tx-data
-                                                                   :errors errors
-                                                                   :pipeline-tx-data (map
-                                                                                      (fn [[e a v t]]
-                                                                                        [e a v t])
-                                                                                      (:tx-data tx-report))}))))
+            (throw-invalid-tx! tx-meta tx-data errors tx-report))
           tx-report)
         (d/transact! conn tx-data tx-meta)))
     (catch :default e

deps/outliner/src/logseq/outliner/op/construct.cljc

@@ -1186,125 +1186,147 @@
                     (numeric-id-in-ref-value? v)))
              block)))
 
+(defn- stale-numeric-id-in-page-ops?
+  [db op args]
+  (case op
+    :save-block
+    (let [[block _opts] args]
+      (numeric-id-in-block-ref-attrs? db block))
+
+    :insert-blocks
+    (let [[blocks target-id _opts] args]
+      (or (some #(numeric-id-in-block-ref-attrs? db %) blocks)
+          (unresolved-numeric-entity-id? target-id)))
+
+    :create-page
+    (let [[_title opts] args]
+      (unresolved-numeric-entity-id? (:uuid opts)))
+
+    :rename-page
+    (let [[page-uuid _new-title] args]
+      (unresolved-numeric-entity-id? page-uuid))
+
+    :delete-page
+    (let [[page-uuid _opts] args]
+      (unresolved-numeric-entity-id? page-uuid))
+
+    :restore-recycled
+    (let [[root-id] args]
+      (unresolved-numeric-entity-id? root-id))
+
+    :apply-template
+    (let [[template-id target-id _opts] args]
+      (or (unresolved-numeric-entity-id? template-id)
+          (unresolved-numeric-entity-id? target-id)))
+
+    :recycle-delete-permanently
+    (let [[root-id] args]
+      (unresolved-numeric-entity-id? root-id))
+
+    nil))
+
+(defn- stale-numeric-id-in-structure-ops?
+  [op args]
+  (case op
+    :move-blocks
+    (let [[ids target-id _opts] args]
+      (or (unresolved-numeric-entity-id-coll? ids)
+          (unresolved-numeric-entity-id? target-id)))
+
+    :move-blocks-up-down
+    (let [[ids _up?] args]
+      (unresolved-numeric-entity-id-coll? ids))
+
+    :indent-outdent-blocks
+    (let [[ids _indent? _opts] args]
+      (unresolved-numeric-entity-id-coll? ids))
+
+    :delete-blocks
+    (let [[ids _opts] args]
+      (unresolved-numeric-entity-id-coll? ids))
+
+    nil))
+
+(defn- stale-numeric-id-in-property-ops?
+  [op args]
+  (case op
+    :set-block-property
+    (let [[block-id property-id _v] args]
+      (or (unresolved-numeric-entity-id? block-id)
+          (unresolved-numeric-entity-id? property-id)))
+
+    :remove-block-property
+    (let [[block-id property-id] args]
+      (or (unresolved-numeric-entity-id? block-id)
+          (unresolved-numeric-entity-id? property-id)))
+
+    :batch-set-property
+    (let [[block-ids property-id _v _opts] args]
+      (or (unresolved-numeric-entity-id-coll? block-ids)
+          (unresolved-numeric-entity-id? property-id)))
+
+    :batch-remove-property
+    (let [[block-ids property-id] args]
+      (or (unresolved-numeric-entity-id-coll? block-ids)
+          (unresolved-numeric-entity-id? property-id)))
+
+    :delete-property-value
+    (let [[block-id property-id _property-value] args]
+      (or (unresolved-numeric-entity-id? block-id)
+          (unresolved-numeric-entity-id? property-id)))
+
+    :batch-delete-property-value
+    (let [[block-ids property-id _property-value] args]
+      (or (unresolved-numeric-entity-id-coll? block-ids)
+          (unresolved-numeric-entity-id? property-id)))
+
+    :create-property-text-block
+    (let [[block-id property-id _value _opts] args]
+      (or (unresolved-numeric-entity-id? block-id)
+          (unresolved-numeric-entity-id? property-id)))
+
+    :class-add-property
+    (let [[class-id property-id] args]
+      (or (unresolved-numeric-entity-id? class-id)
+          (unresolved-numeric-entity-id? property-id)))
+
+    :class-remove-property
+    (let [[class-id property-id] args]
+      (or (unresolved-numeric-entity-id? class-id)
+          (unresolved-numeric-entity-id? property-id)))
+
+    :delete-closed-value
+    (let [[property-id value-block-id] args]
+      (or (unresolved-numeric-entity-id? property-id)
+          (unresolved-numeric-entity-id? value-block-id)))
+
+    nil))
+
+(defn- stale-numeric-id-in-schema-ops?
+  [op args]
+  (case op
+    :upsert-property
+    (let [[property-id _schema _opts] args]
+      (unresolved-numeric-entity-id? property-id))
+
+    :upsert-closed-value
+    (let [[property-id _opts] args]
+      (unresolved-numeric-entity-id? property-id))
+
+    :add-existing-values-to-closed-values
+    (let [[property-id _values] args]
+      (unresolved-numeric-entity-id? property-id))
+
+    nil))
+
 (defn- stale-numeric-id-in-op?
   [db [op args]]
-  (if (= :transact op)
-    false
-    (case op
-      :save-block
-      (let [[block _opts] args]
-        (numeric-id-in-block-ref-attrs? db block))
-
-      :insert-blocks
-      (let [[blocks target-id _opts] args]
-        (or (some #(numeric-id-in-block-ref-attrs? db %) blocks)
-            (unresolved-numeric-entity-id? target-id)))
-
-      :create-page
-      (let [[_title opts] args]
-        (unresolved-numeric-entity-id? (:uuid opts)))
-
-      :rename-page
-      (let [[page-uuid _new-title] args]
-        (unresolved-numeric-entity-id? page-uuid))
-
-      :delete-page
-      (let [[page-uuid _opts] args]
-        (unresolved-numeric-entity-id? page-uuid))
-
-      :restore-recycled
-      (let [[root-id] args]
-        (unresolved-numeric-entity-id? root-id))
-
-      :apply-template
-      (let [[template-id target-id _opts] args]
-        (or (unresolved-numeric-entity-id? template-id)
-            (unresolved-numeric-entity-id? target-id)))
-
-      :move-blocks
-      (let [[ids target-id _opts] args]
-        (or (unresolved-numeric-entity-id-coll? ids)
-            (unresolved-numeric-entity-id? target-id)))
-
-      :move-blocks-up-down
-      (let [[ids _up?] args]
-        (unresolved-numeric-entity-id-coll? ids))
-
-      :indent-outdent-blocks
-      (let [[ids _indent? _opts] args]
-        (unresolved-numeric-entity-id-coll? ids))
-
-      :delete-blocks
-      (let [[ids _opts] args]
-        (unresolved-numeric-entity-id-coll? ids))
-
-      :set-block-property
-      (let [[block-id property-id _v] args]
-        (or (unresolved-numeric-entity-id? block-id)
-            (unresolved-numeric-entity-id? property-id)))
-
-      :remove-block-property
-      (let [[block-id property-id] args]
-        (or (unresolved-numeric-entity-id? block-id)
-            (unresolved-numeric-entity-id? property-id)))
-
-      :batch-set-property
-      (let [[block-ids property-id _v _opts] args]
-        (or (unresolved-numeric-entity-id-coll? block-ids)
-            (unresolved-numeric-entity-id? property-id)))
-
-      :batch-remove-property
-      (let [[block-ids property-id] args]
-        (or (unresolved-numeric-entity-id-coll? block-ids)
-            (unresolved-numeric-entity-id? property-id)))
-
-      :delete-property-value
-      (let [[block-id property-id _property-value] args]
-        (or (unresolved-numeric-entity-id? block-id)
-            (unresolved-numeric-entity-id? property-id)))
-
-      :batch-delete-property-value
-      (let [[block-ids property-id _property-value] args]
-        (or (unresolved-numeric-entity-id-coll? block-ids)
-            (unresolved-numeric-entity-id? property-id)))
-
-      :create-property-text-block
-      (let [[block-id property-id _value _opts] args]
-        (or (unresolved-numeric-entity-id? block-id)
-            (unresolved-numeric-entity-id? property-id)))
-
-      :class-add-property
-      (let [[class-id property-id] args]
-        (or (unresolved-numeric-entity-id? class-id)
-            (unresolved-numeric-entity-id? property-id)))
-
-      :class-remove-property
-      (let [[class-id property-id] args]
-        (or (unresolved-numeric-entity-id? class-id)
-            (unresolved-numeric-entity-id? property-id)))
-
-      :upsert-property
-      (let [[property-id _schema _opts] args]
-        (unresolved-numeric-entity-id? property-id))
-
-      :upsert-closed-value
-      (let [[property-id _opts] args]
-        (unresolved-numeric-entity-id? property-id))
-
-      :add-existing-values-to-closed-values
-      (let [[property-id _values] args]
-        (unresolved-numeric-entity-id? property-id))
-
-      :delete-closed-value
-      (let [[property-id value-block-id] args]
-        (or (unresolved-numeric-entity-id? property-id)
-            (unresolved-numeric-entity-id? value-block-id)))
-
-      :recycle-delete-permanently
-      (let [[root-id] args]
-        (unresolved-numeric-entity-id? root-id))
-
-      false)))
+  (and (not= :transact op)
+       (boolean
+        (or (stale-numeric-id-in-page-ops? db op args)
+            (stale-numeric-id-in-structure-ops? op args)
+            (stale-numeric-id-in-property-ops? op args)
+            (stale-numeric-id-in-schema-ops? op args)))))
 
 (defn- assert-no-stale-numeric-ids!
   [db ops stage]

scripts/src/logseq/tasks/dev/lint_test_deps.clj

@@ -0,0 +1,107 @@
+(ns logseq.tasks.dev.lint-test-deps
+  "Runs lint/test tasks for selected deps with readable progress output."
+  (:require [babashka.process :refer [shell]]))
+
+(def ^:private dep-plan
+  [{:dep "deps/common"
+    :steps [{:runner :bb :name "lint:large-vars" :cmd "lint:large-vars"}
+            {:runner :bb :name "lint:carve" :cmd "lint:carve"}
+            {:runner :bb :name "lint:ns-docstrings" :cmd "lint:ns-docstrings"}
+            {:runner :cmd :name "yarn test (-e long)" :cmd "yarn test -e long"}]}
+   {:dep "deps/db"
+    :steps [{:runner :bb :name "lint:large-vars" :cmd "lint:large-vars"}
+            {:runner :bb :name "lint:carve" :cmd "lint:carve"}
+            {:runner :bb :name "lint:ns-docstrings" :cmd "lint:ns-docstrings"}
+            {:runner :bb :name "lint:rules" :cmd "lint:rules"}
+            {:runner :cmd :name "yarn test (-e long)" :cmd "yarn test -e long"}]}
+   {:dep "deps/db-sync"
+    :steps [{:runner :bb :name "lint:large-vars" :cmd "lint:large-vars"}
+            {:runner :bb :name "lint:carve" :cmd "lint:carve"}
+            ;; {:runner :bb :name "lint:ns-docstrings" :cmd "lint:ns-docstrings"}
+            ;; {:runner :bb :name "lint:minimize-public-vars" :cmd "lint:minimize-public-vars"}
+            ]}
+   {:dep "."
+    :steps [{:runner :bb :name "dev:db-sync-test" :cmd "dev:db-sync-test"}]}
+   {:dep "deps/outliner"
+    :steps [{:runner :bb :name "lint:large-vars" :cmd "lint:large-vars"}
+            {:runner :bb :name "lint:carve" :cmd "lint:carve"}
+            {:runner :bb :name "lint:ns-docstrings" :cmd "lint:ns-docstrings"}
+            {:runner :bb :name "lint:minimize-public-vars" :cmd "lint:minimize-public-vars"}
+            {:runner :cmd :name "yarn test (-e long)" :cmd "yarn test -e long"}]}
+   {:dep "deps/graph-parser"
+    :steps [{:runner :bb :name "lint:large-vars" :cmd "lint:large-vars"}
+            {:runner :bb :name "lint:carve" :cmd "lint:carve"}
+            {:runner :bb :name "lint:ns-docstrings" :cmd "lint:ns-docstrings"}
+            ;; {:runner :bb :name "lint:minimize-public-vars" :cmd "lint:minimize-public-vars"}
+            {:runner :cmd :name "yarn test (-e long)" :cmd "yarn test -e long"}]}
+   {:dep "deps/cli"
+    :steps [{:runner :bb :name "lint:large-vars" :cmd "lint:large-vars"}
+            {:runner :bb :name "lint:carve" :cmd "lint:carve"}
+            {:runner :bb :name "lint:ns-docstrings" :cmd "lint:ns-docstrings"}
+            {:runner :bb :name "lint:minimize-public-vars" :cmd "lint:minimize-public-vars"}
+            {:runner :cmd :name "yarn test (-e long)" :cmd "yarn test -e long"}]}
+   {:dep "deps/publish"
+    :steps [{:runner :bb :name "lint:large-vars" :cmd "lint:large-vars"}
+            {:runner :bb :name "lint:carve" :cmd "lint:carve"}
+            {:runner :bb :name "lint:ns-docstrings" :cmd "lint:ns-docstrings"}
+            {:runner :skip :name "tests" :reason "no test script in deps/publish/package.json"}]}
+   {:dep "deps/publishing"
+    :steps [{:runner :bb :name "lint:large-vars" :cmd "lint:large-vars"}
+            {:runner :bb :name "lint:carve" :cmd "lint:carve"}
+            {:runner :bb :name "lint:ns-docstrings" :cmd "lint:ns-docstrings"}
+            {:runner :bb :name "lint:minimize-public-vars" :cmd "lint:minimize-public-vars"}]}])
+
+(defn- run-step!
+  [dep {:keys [runner name cmd reason]}]
+  (case runner
+    :skip
+    (do
+      (println (str "\n==> [" dep "] " name " (skip)"))
+      (println (str "SKIP[" dep "] " name " - " reason))
+      {:dep dep :task name :exit 0 :skipped true})
+
+    :bb
+    (do
+      (println (str "\n==> [" dep "] bb " cmd))
+      (let [result (shell {:dir dep :continue true} (str "bb " cmd))
+            success? (zero? (:exit result))]
+        (println (str (if success? "OK  " "ERR ")
+                      "[" dep "] "
+                      name
+                      (when-not success?
+                        (str " (exit " (:exit result) ")"))))
+        {:dep dep :task name :exit (:exit result)}))
+
+    :cmd
+    (do
+      (println (str "\n==> [" dep "] " cmd))
+      (let [result (shell {:dir dep :continue true} cmd)
+            success? (zero? (:exit result))]
+        (println (str (if success? "OK  " "ERR ")
+                      "[" dep "] "
+                      name
+                      (when-not success?
+                        (str " (exit " (:exit result) ")"))))
+        {:dep dep :task name :exit (:exit result)}))))
+
+(defn run
+  "Lint and test selected deps projects with per-step progress and summary."
+  [& _]
+  (println "Starting deps lint/test suite...")
+  (let [results (vec
+                 (mapcat (fn [{:keys [dep steps]}]
+                           (map #(run-step! dep %) steps))
+                         dep-plan))
+        skipped-count (count (filter :skipped results))
+        failures (filter #(pos? (:exit %)) results)]
+    (println "\nSummary:")
+    (println (str "  Total tasks: " (count results)))
+    (println (str "  Passed: " (- (count results) skipped-count (count failures))))
+    (println (str "  Skipped: " skipped-count))
+    (println (str "  Failed: " (count failures)))
+    (when (seq failures)
+      (println "\nFailed tasks:")
+      (doseq [{:keys [dep task exit]} failures]
+        (println (str "  - [" dep "] " task " (exit " exit ")")))
+      (System/exit 1))
+    (println "\nAll deps lint/test tasks passed.")))