enhance: improve error handling for exports + imports

Display user friendly notification when unexpected failures occur.
Also actively validate export to ensure it can even import

deps/db/src/logseq/db/sqlite/build.cljs

@@ -121,7 +121,7 @@
                      [property-map v])))))
        (db-property-build/build-property-values-tx-m new-block)))
 
-(defn extract-content-refs
+(defn- extract-content-refs
   "Extracts basic refs from :block/title like `[[foo]]` or `[[UUID]]`. Can't
   use db-content/get-matched-ids because of named ref support.  Adding more ref
   support would require parsing each block with mldoc and extracting with
@@ -352,20 +352,6 @@
                              ((fn [x] (update-vals x #(mapv second %)))))]
     props-to-values))
 
-(defn- validate-options
-  [{:keys [properties] :as options}]
-  (when-let [errors (->> options (m/explain Options) me/humanize)]
-    (println "The build-blocks-tx has the following options errors:")
-    (pprint/pprint errors)
-    (throw (ex-info "Options validation failed" {:errors errors})))
-  (when-not (:auto-create-ontology? options)
-    (let [used-properties (get-used-properties-from-options options)
-          undeclared-properties (-> (set (keys used-properties))
-                                    (set/difference (set (keys properties)))
-                                    ((fn [x] (remove db-property/logseq-property? x))))]
-      (assert (empty? undeclared-properties)
-              (str "The following properties used in EDN were not declared in :properties: " undeclared-properties)))))
-
 ;; TODO: How to detect these idents don't conflict with existing? :db/add?
 (defn- create-all-idents
   [properties classes graph-namespace]
@@ -484,7 +470,7 @@
       (println "Building additional pages from content refs:" (pr-str (mapv #(get-in % [:page :block/title]) new-pages-from-refs))))
     (concat pages-and-blocks new-pages-from-refs)))
 
-(defn add-new-pages-from-properties
+(defn- add-new-pages-from-properties
   [properties pages-and-blocks]
   (let [used-properties (get-used-properties-from-options {:pages-and-blocks pages-and-blocks :properties properties})
         existing-pages (->> pages-and-blocks (keep #(select-keys (:page %) [:build/journal :block/title])) set)
@@ -623,6 +609,34 @@
                              classes-tx
                              pages-and-blocks-tx))))
 
+;; Public API
+;; ==========
+
+(defn extract-from-blocks
+  "Given a vec of blocks and a fn which applied to a block returns a coll, this
+  returns the coll produced by applying f to all blocks including :build/children blocks"
+  [blocks f]
+  (let [apply-to-block-and-all-children
+        (fn apply-to-block-and-all-children [m f]
+          (into (f m)
+                (when-let [children (seq (:build/children m))]
+                  (mapcat #(apply-to-block-and-all-children % f) children))))]
+    (mapcat #(apply-to-block-and-all-children % f) blocks)))
+
+(defn validate-options
+  [{:keys [properties] :as options}]
+  (when-let [errors (->> options (m/explain Options) me/humanize)]
+    (println "The build-blocks-tx has the following options errors:")
+    (pprint/pprint errors)
+    (throw (ex-info "Options validation failed" {:errors errors})))
+  (when-not (:auto-create-ontology? options)
+    (let [used-properties (get-used-properties-from-options options)
+          undeclared-properties (-> (set (keys used-properties))
+                                    (set/difference (set (keys properties)))
+                                    ((fn [x] (remove db-property/logseq-property? x))))]
+      (assert (empty? undeclared-properties)
+              (str "The following properties used in EDN were not declared in :properties: " undeclared-properties)))))
+
 (defn ^:large-vars/doc-var build-blocks-tx
   "Given an EDN map for defining pages, blocks and properties, this creates a map
  with two keys of transactable data for use with d/transact!. The :init-tx key

deps/db/src/logseq/db/sqlite/export.cljs

@@ -1,7 +1,8 @@
 (ns logseq.db.sqlite.export
   "Builds sqlite.build EDN to represent nodes in a graph-agnostic way.
    Useful for exporting and importing across DB graphs"
-  (:require [datascript.core :as d]
+  (:require [clojure.set :as set]
+            [datascript.core :as d]
             [datascript.impl.entity :as de]
             [logseq.db :as ldb]
             [logseq.db.frontend.class :as db-class]
@@ -354,16 +355,48 @@
       (seq classes)
       (assoc :classes classes))))
 
+(defn- ensure-export-is-valid
+  "Checks that export map is usable by sqlite.build including checking that
+   all referenced properties and classes are defined"
+  [{:keys [classes properties pages-and-blocks] :as export-map}]
+  (sqlite-build/validate-options export-map)
+  (let [referenced-classes
+        (->> (concat (mapcat :build/property-classes (vals properties))
+                     (keep :class/parent (vals classes))
+                     (mapcat (comp :block/tags :page) pages-and-blocks)
+                     (mapcat #(sqlite-build/extract-from-blocks (:blocks %) :build/tags) pages-and-blocks))
+             (remove db-class/logseq-class?)
+             set)
+        undefined-classes (set/difference referenced-classes (set (keys classes)))
+        referenced-properties
+        (->> (concat (mapcat :build/class-properties (vals classes))
+              (mapcat (comp keys :build/properties :page) pages-and-blocks)
+                     (mapcat #(sqlite-build/extract-from-blocks (:blocks %) (comp keys :build/properties)) pages-and-blocks))
+             (remove db-property/logseq-property?)
+             set)
+        undefined-properties (set/difference referenced-properties (set (keys properties)))
+        undefined (cond-> {}
+                    (seq undefined-classes) (assoc :classes undefined-classes)
+                    (seq undefined-properties) (assoc :properties undefined-properties))]
+    ;; (prn :rclasses referenced-classes)
+    ;; (prn :rproperties referenced-properties)
+    (when (seq undefined)
+      (throw (ex-info (str "The following classes and properties are not defined: " (pr-str undefined))
+                      undefined)))))
+
 (defn build-export
   "Handles exporting db by given export-type"
   [db {:keys [export-type] :as options}]
-  (case export-type
-    :block
-    (build-block-export db (:block-id options))
-    :page
-    (build-page-export db (:page-id options))
-    :graph-ontology
-    (build-graph-ontology-export db)))
+  (let [export-map
+        (case export-type
+          :block
+          (build-block-export db (:block-id options))
+          :page
+          (build-page-export db (:page-id options))
+          :graph-ontology
+          (build-graph-ontology-export db))]
+    (ensure-export-is-valid (dissoc export-map ::block))
+    export-map))
 
 ;; Import fns
 ;; ==========

src/main/frontend/handler/db_based/export.cljs

@@ -59,9 +59,13 @@
     (if (= ::invalid-import export-map)
       (notification/show! "The submitted EDN data is invalid! Fix and try again." :warning)
       (let [{:keys [init-tx block-props-tx error] :as txs}
-            (sqlite-export/build-import export-map
-                                        (db/get-db)
-                                        (when block {:current-block block}))]
+            (try
+              (sqlite-export/build-import export-map
+                                          (db/get-db)
+                                          (when block {:current-block block}))
+              (catch :default e
+                (js/console.error "Import EDN error: " e)
+                {:error "An unexpected error occurred during import. See the javascript console for details."}))]
         (pprint/pprint txs)
         (if error
           (notification/show! error :error)

src/main/frontend/worker/db_worker.cljs

@@ -911,9 +911,16 @@
 
   (export-edn
    [_this repo options]
-   (let [conn (worker-state/get-datascript-conn repo)
-         result (sqlite-export/build-export @conn (ldb/read-transit-str options))]
-     (ldb/write-transit-str result)))
+   (let [conn (worker-state/get-datascript-conn repo)]
+     (try
+       (->> (ldb/read-transit-str options)
+            (sqlite-export/build-export @conn)
+            ldb/write-transit-str)
+       (catch :default e
+         (js/console.error "export-edn error: " e)
+         (worker-util/post-message :notification
+                                   ["An unexpected error occurred during export. See the javascript console for details."
+                                    :error])))))
 
   (dangerousRemoveAllDbs
    [this repo]