enhance: add :exclude-namespaces export option

Useful for graphs with existing ontologies to avoid noisy
and needless exporting. Also update related scripts and noted
that create script has always been able to also update

deps/db/script/create_graph.cljs

@@ -1,5 +1,6 @@
 (ns create-graph
-  "A script that creates a DB graph given a sqlite.build EDN file"
+  "A script that creates or updates a DB graph given a sqlite.build EDN file.
+   If the given graph already exists, the EDN file updates the graph."
   (:require ["fs" :as fs]
             ["os" :as os]
             ["path" :as node-path]
@@ -51,6 +52,7 @@
         [dir db-name] (get-dir-and-db-name graph-dir)
         sqlite-build-edn (merge (if (:import options) {} {:auto-create-ontology? true})
                                 (-> (resolve-path edn-path) fs/readFileSync str edn/read-string))
+        graph-exists? (fs/existsSync (node-path/join dir db-name))
         conn (outliner-cli/init-conn dir db-name {:classpath (cp/get-classpath) :import-type :cli/create-graph})
         {:keys [init-tx block-props-tx misc-tx] :as _txs}
         (if (:import options)
@@ -62,7 +64,7 @@
     (d/transact! conn init-tx)
     (when (seq block-props-tx) (d/transact! conn block-props-tx))
     (when (seq misc-tx) (d/transact! conn misc-tx))
-    (println "Created graph" (str db-name "!"))
+    (println (if graph-exists? "Updated graph" "Created graph") (str db-name "!"))
     (when (:validate options)
       (validate-db/validate-db @conn db-name {:group-errors true :closed-maps true :humanize true}))))
 

deps/db/script/diff_graphs.cljs

@@ -6,6 +6,7 @@
             [clojure.data :as data]
             [clojure.pprint :as pprint]
             [clojure.string :as string]
+            [logseq.common.config :as common-config]
             #_:clj-kondo/ignore
             [logseq.db.sqlite.cli :as sqlite-cli]
             [logseq.db.sqlite.export :as sqlite-export]
@@ -24,8 +25,11 @@
   "Options spec"
   {:help {:alias :h
           :desc "Print help"}
-   :timestamps {:alias :t
-                :desc "Include timestamps in export"}})
+   :exclude-namespaces {:alias :e
+                        :coerce #{}
+                        :desc "Namespaces to exclude from properties and classes"}
+   :include-timestamps? {:alias :t
+                         :desc "Include timestamps in export"}})
 
 (defn -main [args]
   (let [{options :opts args' :args} (cli/parse-args args {:spec spec})
@@ -36,10 +40,19 @@
             (js/process.exit 1))
         conn (apply sqlite-cli/open-db! (get-dir-and-db-name graph-dir))
         conn2 (apply sqlite-cli/open-db! (get-dir-and-db-name graph-dir2))
-        export-options {:include-timestamps? (:timestamps options)}
+        export-options (select-keys options [:include-timestamps? :exclude-namespaces])
         export-map (sqlite-export/build-export @conn {:export-type :graph :graph-options export-options})
         export-map2 (sqlite-export/build-export @conn2 {:export-type :graph :graph-options export-options})
-        diff (butlast (data/diff export-map export-map2))]
+        prepare-export-to-diff
+        (fn [m]
+          (-> m
+              (update :classes update-vals (fn [m]
+                                             (update m :build/class-properties sort)))
+              ;; TODO: fix built-in views
+              (update :pages-and-blocks (fn [pbs]
+                                          (vec (remove #(= (:block/title (:page %)) common-config/views-page-name) pbs))))))
+        diff (->> (data/diff (prepare-export-to-diff export-map) (prepare-export-to-diff export-map2))
+                  butlast)]
     (pprint/pprint diff)))
 
 (when (= nbb/*file* (nbb/invoked-file))

deps/db/script/export_graph.cljs

@@ -32,11 +32,14 @@
   "Options spec"
   {:help {:alias :h
           :desc "Print help"}
-   :timestamps {:alias :t
-                :desc "Include timestamps in export"}
+   :include-timestamps? {:alias :t
+                         :desc "Include timestamps in export"}
    :file {:alias :f
           :desc "Saves edn to file"}
-   :export-options {:alias :e
+   :exclude-namespaces {:alias :e
+                        :coerce #{}
+                        :desc "Namespaces to exclude from properties and classes"}
+   :export-options {:alias :E
                     :desc "Raw options map to pass to export"}})
 
 (defn -main [args]
@@ -48,7 +51,7 @@
             (js/process.exit 1))
         [dir db-name] (get-dir-and-db-name graph-dir)
         conn (sqlite-cli/open-db! dir db-name)
-        export-options (merge {:include-timestamps? (:timestamps options)}
+        export-options (merge (select-keys options [:include-timestamps? :exclude-namespaces])
                               (edn/read-string (:export-options options)))
         export-map (sqlite-export/build-export @conn {:export-type :graph :graph-options export-options})]
     (when (:file options)

deps/db/src/logseq/db/sqlite/build.cljs

@@ -334,7 +334,7 @@
    [:auto-create-ontology? {:optional true} :boolean]
    [:build-existing-tx? {:optional true} :boolean]])
 
-(defn- get-used-properties-from-options
+(defn get-used-properties-from-options
   "Extracts all used properties as a map of properties to their property values. Looks at properties
    from :build/properties and :build/class-properties. Properties from :build/class-properties have
    a ::no-value value"

deps/db/src/logseq/db/sqlite/export.cljs

@@ -2,6 +2,7 @@
   "Builds sqlite.build EDN to represent nodes in a graph-agnostic way.
    Useful for exporting and importing across DB graphs"
   (:require [clojure.set :as set]
+            [clojure.string :as string]
             [clojure.walk :as walk]
             [datascript.core :as d]
             [datascript.impl.entity :as de]
@@ -456,18 +457,26 @@
 
 (defn- build-graph-ontology-export
   "Exports a graph's tags and properties"
-  [db options]
-  (let [user-property-idents (d/q '[:find [?db-ident ...]
+  [db {:keys [exclude-namespaces] :as options}]
+  (let [exclude-regex (when (seq exclude-namespaces)
+                        (re-pattern (str "^("
+                                         (string/join "|" (map name exclude-namespaces))
+                                         ")(\\.|$)")))
+        user-property-idents (d/q '[:find [?db-ident ...]
                                     :where [?p :db/ident ?db-ident]
                                     [?p :block/tags :logseq.class/Property]
                                     (not [?p :logseq.property/built-in?])]
                                   db)
+        user-property-idents (if (seq exclude-namespaces)
+                               (remove #(re-find exclude-regex (namespace %)) user-property-idents)
+                               user-property-idents)
         properties (build-export-properties db user-property-idents (merge options {:include-properties? true}))
         class-ents (->> (d/q '[:find [?class ...]
                                :where [?class :block/tags :logseq.class/Tag]
                                (not [?class :logseq.property/built-in?])]
                              db)
-                        (map #(d/entity db %)))
+                        (map #(d/entity db %))
+                        (remove #(and (seq exclude-namespaces) (re-find exclude-regex (namespace (:db/ident %))))))
         classes
         (->> class-ents
              (map (fn [ent]
@@ -504,7 +513,10 @@
                                (build-page-export* db eid page-blocks* (merge options {:include-uuid-fn (constantly true)}))))
                            page-ids)
         pages-export (apply merge-export-maps page-exports)
-        pages-export' (assoc pages-export :pvalue-uuids (set (mapcat :pvalue-uuids page-exports)))]
+        pages-export' (-> pages-export
+                          (assoc :pvalue-uuids (set (mapcat :pvalue-uuids page-exports)))
+                          ;; TODO: Remove when exports in this fn no longer generate any ontologies
+                          (dissoc :classes :properties))]
     pages-export'))
 
 (defn- build-graph-files
@@ -537,9 +549,28 @@
    (sort-by #(or (get-in % [:page :block/title]) (get-in % [:page :block/title]))
             pages-and-blocks)))
 
+(defn- add-ontology-for-include-namespaces
+  "Adds :properties to export for given namespace parents. Current use case is for :exclude-namespaces
+   so no need to add :classes yet"
+  [db {::keys [auto-include-namespaces] :as graph-export}]
+  (let [include-regex (re-pattern (str "^("
+                                       (string/join "|" (map name auto-include-namespaces))
+                                       ")(\\.|$)"))
+        used-properties
+        (->> (sqlite-build/get-used-properties-from-options graph-export)
+             keys
+             (remove db-property/logseq-property?)
+             (filter #(re-find include-regex (namespace %)))
+             (map #(vector % (select-keys (d/entity db %) [:logseq.property/type :db/cardinality])))
+             (into {}))]
+    (merge-export-maps graph-export {:properties used-properties})))
+
 (defn- build-graph-export
   "Exports whole graph. Has the following options:
-   * :include-timestamps? - When set timestamps are included on all blocks"
+   * :include-timestamps? - When set timestamps are included on all blocks
+   * :exclude-namespaces - A set of parent namespaces to exclude from properties and classes.
+     This is useful for graphs seeded with an ontology e.g. schema.org as it eliminates noisy and needless
+     export+import"
   [db options*]
   (let [options (merge options* {:property-value-uuids? true})
         content-ref-uuids (get-graph-content-ref-uuids db)
@@ -548,7 +579,10 @@
         ontology-pvalue-uuids (set (concat (mapcat get-pvalue-uuids (vals (:properties ontology-export)))
                                            (mapcat get-pvalue-uuids (vals (:classes ontology-export)))))
         pages-export (build-graph-pages-export db options)
-        graph-export (merge-export-maps ontology-export pages-export)
+        graph-export* (merge-export-maps ontology-export pages-export)
+        graph-export (if (seq (:exclude-namespaces options))
+                         (assoc graph-export* ::auto-include-namespaces (:exclude-namespaces options))
+                         graph-export*)
         all-ref-uuids (set/union content-ref-uuids ontology-pvalue-uuids (:pvalue-uuids pages-export))
         files (build-graph-files db options)
         ;; Remove all non-ref uuids after all nodes are built.
@@ -601,11 +635,14 @@
 
 (defn- ensure-export-is-valid
   "Checks that export map is usable by sqlite.build including checking that
-   all referenced properties and classes are defined"
-  [export-map]
-  (sqlite-build/validate-options export-map)
+   all referenced properties and classes are defined. Checks related to properties and
+   classes are disabled when :exclude-namespaces is set because those checks can't be done"
+  [export-map {:keys [graph-options]}]
+  (when-not (seq (:exclude-namespaces graph-options)) (sqlite-build/validate-options export-map))
   (let [undefined-uuids (find-undefined-uuids export-map)
-        undefined (cond-> (find-undefined-classes-and-properties export-map)
+        undefined (cond-> {}
+                    (empty? (:exclude-namespaces graph-options))
+                    (merge (find-undefined-classes-and-properties export-map))
                     (seq undefined-uuids)
                     (assoc :uuids undefined-uuids))]
     (when (seq undefined)
@@ -627,7 +664,7 @@
           (build-graph-ontology-export db {})
           :graph
           (build-graph-export db (:graph-options options)))]
-    (ensure-export-is-valid (dissoc export-map ::block ::graph-files))
+    (ensure-export-is-valid (dissoc export-map ::block ::graph-files) options)
     export-map))
 
 ;; Import fns
@@ -711,13 +748,17 @@
   (let [export-map (if (and (::block export-map*) current-block)
                      (build-block-import-options current-block export-map*)
                      export-map*)
+        export-map' (if (seq (::auto-include-namespaces export-map*))
+                      (merge (select-keys export-map [::graph-files])
+                             (add-ontology-for-include-namespaces db export-map))
+                      export-map)
         property-conflicts (atom [])
-        export-map' (check-for-existing-entities db export-map property-conflicts)]
+        export-map'' (check-for-existing-entities db export-map' property-conflicts)]
     (if (seq @property-conflicts)
       (do
         (js/console.error :property-conflicts @property-conflicts)
         {:error (str "The following imported properties conflict with the current graph: "
                      (pr-str (mapv :property-id @property-conflicts)))})
-      (cond-> (sqlite-build/build-blocks-tx (dissoc export-map' ::graph-files))
-        (seq (::graph-files export-map'))
-        (assoc :misc-tx (::graph-files export-map'))))))
+      (cond-> (sqlite-build/build-blocks-tx (dissoc export-map'' ::graph-files))
+        (seq (::graph-files export-map''))
+        (assoc :misc-tx (::graph-files export-map''))))))

deps/db/test/logseq/db/sqlite/export_test.cljs

@@ -530,7 +530,7 @@
     (is (= (expand-classes (:classes original-data)) (:classes imported-nodes)))))
 
 (defn- build-original-graph-data
-  []
+  [& {:keys [exclude-namespaces?]}]
   (let [internal-block-uuid (random-uuid)
         favorited-uuid (random-uuid)
         block-pvalue-uuid (random-uuid)
@@ -546,7 +546,9 @@
          {:user.property/num {:logseq.property/type :number
                               :block/uuid property-uuid
                               :build/keep-uuid? true
-                              :build/properties {:user.property/node #{[:block/uuid property-pvalue-uuid]}}}
+                              :build/properties (if exclude-namespaces?
+                                                  {}
+                                                  {:user.property/node #{[:block/uuid property-pvalue-uuid]}})}
           :user.property/default-closed
           {:logseq.property/type :default
            :build/closed-values [{:value "joy" :uuid closed-value-uuid}
@@ -585,10 +587,11 @@
                      :build/tags [:user.class/MyClass]
                      :block/uuid block-pvalue-uuid
                      :build/keep-uuid? true}
-                    {:block/title "myclass object 2"
-                     :build/tags [:user.class/MyClass]
-                     :block/uuid property-pvalue-uuid
-                     :build/keep-uuid? true}
+                    (cond-> {:block/title "myclass object 2"
+                             :build/tags [:user.class/MyClass]}
+                      (not exclude-namespaces?)
+                      (merge {:block/uuid property-pvalue-uuid
+                              :build/keep-uuid? true}))
                     {:block/title "myclass object 3"
                      :build/tags [:user.class/MyClass]
                      :block/uuid page-pvalue-uuid
@@ -653,7 +656,7 @@
     (is (= (::sqlite-export/graph-files original-data) (::sqlite-export/graph-files imported-graph))
         "All :file/path entities are imported")))
 
-(deftest ^:focus2 import-graph-with-timestamps
+(deftest import-graph-with-timestamps
   (let [original-data* (build-original-graph-data)
         original-data (-> original-data*
                           (update :pages-and-blocks
@@ -688,4 +691,19 @@
     (is (= (expand-properties (:properties original-data)) (:properties imported-graph)))
     (is (= (expand-classes (:classes original-data)) (:classes imported-graph)))
     (is (= (::sqlite-export/graph-files original-data) (::sqlite-export/graph-files imported-graph))
-        "All :file/path entities are imported")))
+        "All :file/path entities are imported")))
+
+(deftest ^:focus2 import-graph-with-exclude-namespaces
+  (let [original-data (build-original-graph-data {:exclude-namespaces? true})
+        conn (db-test/create-conn-with-blocks (dissoc original-data ::sqlite-export/graph-files))
+        _ (d/transact! conn (::sqlite-export/graph-files original-data))
+        conn2 (db-test/create-conn-with-blocks
+               {:properties (update-vals (:properties original-data) #(dissoc % :build/properties))
+                :classes (update-vals (:classes original-data) #(dissoc % :build/properties))})
+        imported-graph (export-graph-and-import-to-another-graph conn conn2 {:exclude-namespaces #{:user}})]
+
+    ;; (cljs.pprint/pprint (butlast (clojure.data/diff (sort-pages-and-blocks (:pages-and-blocks original-data))
+    ;;                                                 (:pages-and-blocks imported-graph))))
+    (is (= (sort-pages-and-blocks (:pages-and-blocks original-data)) (:pages-and-blocks imported-graph)))
+    (is (= (::sqlite-export/graph-files original-data) (::sqlite-export/graph-files imported-graph))
+        "All :file/path entities are imported")))