fix: create properties graph and validate graph script

deps/db/script/validate_client_db.cljs

@@ -4,6 +4,7 @@
   (:require [logseq.db.sqlite.db :as sqlite-db]
             [logseq.db.frontend.malli-schema :as db-malli-schema]
             [logseq.db.frontend.validate :as db-validate]
+            [logseq.db.frontend.property :as db-property]
             [datascript.core :as d]
             [clojure.string :as string]
             [nbb.core :as nbb]
@@ -15,9 +16,8 @@
 
 (defn validate-client-db
   "Validate datascript db as a vec of entity maps"
-  [db ent-maps* {:keys [verbose group-errors closed-maps]}]
-  (let [ent-maps (vec (db-malli-schema/update-properties-in-ents ent-maps*))
-        schema (db-validate/update-schema db-malli-schema/DB db {:closed-schema? closed-maps})]
+  [db ent-maps {:keys [verbose group-errors closed-maps]}]
+  (let [schema (db-validate/update-schema db-malli-schema/DB db {:closed-schema? closed-maps})]
     (if-let [errors (->> ent-maps
                          (m/explain schema)
                          :errors)]
@@ -70,7 +70,7 @@
                                (count (filter #(contains? (:block/type %) "class") ent-maps)) " classes, "
                                (count (filter #(seq (:block/tags %)) ent-maps)) " objects, "
                                (count (filter #(contains? (:block/type %) "property") ent-maps)) " properties and "
-                               (count (mapcat :block/properties ent-maps)) " property values"))
+                               (count (mapcat db-property/properties ent-maps)) " property values"))
     (validate-client-db @conn ent-maps options)))
 
 (defn -main [argv]

deps/db/src/logseq/db/frontend/property.cljs

@@ -239,6 +239,14 @@
              (string/starts-with? k-name "logseq.task")
              (string/starts-with? k-name "user.property")))))
 
+(defn properties
+  "Fetch all properties of entity like :block/properties used to do.
+   Use this in deps because nbb can't use :block/properties from entity-plus"
+  [e]
+  (->> (into {} e)
+       (filter (fn [[k _]] (property? k)))
+       (into {})))
+
 ;; TODO: db ident should obey clojure's rules for keywords
 (defn get-db-ident-from-name
   [property-name]

deps/db/src/logseq/db/frontend/property/util.cljs

@@ -2,8 +2,7 @@
   "Util fns for building core property concepts"
   (:require [logseq.db.sqlite.util :as sqlite-util]
             [logseq.db.frontend.default :as default-db]
-            [datascript.core :as d]
-            [clojure.string :as string]))
+            [datascript.core :as d]))
 
 (defonce hidden-page-name-prefix "$$$")
 

scripts/src/logseq/tasks/db_graph/create_graph.cljs

@@ -9,12 +9,12 @@
             [logseq.db.frontend.property.util :as db-property-util]
             [logseq.outliner.cli.pipeline :as cli-pipeline]
             [logseq.common.util :as common-util]
-            [logseq.db :as ldb]
             [clojure.string :as string]
             [datascript.core :as d]
             ["fs" :as fs]
             ["path" :as node-path]
-            [nbb.classpath :as cp]))
+            [nbb.classpath :as cp]
+            [logseq.db.frontend.property :as db-property]))
 
 (defn- find-on-classpath [rel-path]
   (some (fn [dir]
@@ -54,20 +54,21 @@
   (if (vector? val)
     (case (first val)
       :page
-      (or (page-uuids (second val))
-          (throw (ex-info (str "No uuid for page '" (second val) "'") {:name (second val)})))
+      (if-let [page-uuid (page-uuids (second val))]
+        [:block/uuid page-uuid]
+        (throw (ex-info (str "No uuid for page '" (second val) "'") {:name (second val)})))
       :block
       (or (block-uuids (second val))
           (throw (ex-info (str "No uuid for block '" (second val) "'") {:name (second val)})))
       (throw (ex-info "Invalid property value type. Valid values are :block and :page" {})))
     val))
 
+;; TODO: Remove unused property-uuids
 (defn- ->block-properties-tx [properties {:keys [property-uuids] :as uuid-maps}]
   (->> properties
        (map
         (fn [[prop-name val]]
-          [(or (property-uuids prop-name)
-               (throw (ex-info "No uuid for property" {:name prop-name})))
+          [(db-property/get-db-ident-from-name (name prop-name))
             ;; set indicates a :many value
            (if (set? val)
              (set (map #(translate-property-value % uuid-maps) val))
@@ -100,12 +101,11 @@
      :page-uuids page-uuids
      :block-uuids block-uuids}))
 
+;; TODO: Remove unused property-db-ids
 (defn- build-property-refs [properties property-db-ids]
   (mapv
    (fn [prop-name]
-     {:db/id
-      (or (property-db-ids (name prop-name))
-          (throw (ex-info (str "No :db/id for property '" prop-name "'") {:property prop-name})))})
+     (db-property/get-db-ident-from-name (name prop-name)))
    (keys properties)))
 
 (def current-db-id (atom 0))
@@ -122,8 +122,8 @@
            :block/left {:db/id (or (:db/id last-block) page-id)}
            :block/parent {:db/id page-id}})
          (when (seq (:properties m))
-           {:block/properties (->block-properties-tx (:properties m) uuid-maps)
-            :block/refs (build-property-refs (:properties m) property-db-ids)})))
+           (assoc (->block-properties-tx (:properties m) uuid-maps)
+                  :block/refs (build-property-refs (:properties m) property-db-ids)))))
 
 (defn create-blocks-tx
   "Given an EDN map for defining pages, blocks and properties, this creates a
@@ -156,7 +156,7 @@
 
    This fn also takes an optional map arg which supports these keys:
    * :property-uuids - A map of property keyword names to uuids to provide ids for built-in properties"
-  [db {:keys [pages-and-blocks properties]} & {:as options}]
+  [{:keys [pages-and-blocks properties]} & {:as options}]
   (let [;; add uuids before tx for refs in :properties
         pages-and-blocks' (mapv (fn [{:keys [page blocks]}]
                                   (cond-> {:page (merge {:block/uuid (random-uuid)} page)}
@@ -171,25 +171,26 @@
                            (mapcat
                             (fn [[prop-name]]
                               (if (get-in properties [prop-name :closed-values])
-                                (db-property-util/build-closed-values
-                                 (keyword "user.property" (name prop-name))
-                                 prop-name
-                                 (get properties prop-name)
-                                 {:icon-id
-                                  (get-in options [:property-uuids :icon])
-                                  :translate-closed-page-value-fn
-                                  #(hash-map :block/uuid (translate-property-value (:value %) uuid-maps))
-                                  :property-attributes
-                                  {:db/id (or (property-db-ids (name prop-name))
-                                              (throw (ex-info "No :db/id for property" {:property prop-name})))}})
+                                (let [db-ident (db-property/get-db-ident-from-name (name prop-name))]
+                                  (db-property-util/build-closed-values
+                                   db-ident
+                                   prop-name
+                                   (assoc (get properties prop-name) :db/ident db-ident)
+                                   {:icon-id
+                                    (get-in options [:property-uuids :icon])
+                                    :translate-closed-page-value-fn
+                                    #(hash-map :block/uuid (translate-property-value (:value %) uuid-maps))
+                                    :property-attributes
+                                    {:db/id (or (property-db-ids (name prop-name))
+                                                (throw (ex-info "No :db/id for property" {:property prop-name})))}}))
                                 [(let [db-ident (keyword "user.property" (name prop-name))]
                                    (merge
                                     (sqlite-util/build-new-property db-ident prop-name (get-in properties [prop-name :block/schema]))
                                     {:db/id (or (property-db-ids (name prop-name))
                                                 (throw (ex-info "No :db/id for property" {:property prop-name})))}
                                     (when-let [props (not-empty (get-in properties [prop-name :properties]))]
-                                      {:block/properties (->block-properties-tx props uuid-maps)
-                                       :block/refs (build-property-refs props property-db-ids)})))]))
+                                      (assoc (->block-properties-tx props uuid-maps)
+                                             :block/refs (build-property-refs props property-db-ids)))))]))
                             property-uuids))
         pages-and-blocks-tx
         (vec
@@ -207,9 +208,10 @@
                    :block/format :markdown}
                   (dissoc page :properties)
                   (when (seq (:properties page))
-                    {:block/properties (->block-properties-tx (:properties page) uuid-maps)
-                     :block/refs (build-property-refs (:properties page) property-db-ids)
-                          ;; app doesn't do this yet but it should to link property to page
+                    (->block-properties-tx (:properties page) uuid-maps))
+                  (when (seq (:properties page))
+                    {:block/refs (build-property-refs (:properties page) property-db-ids)
+                     ;; app doesn't do this yet but it should to link property to page
                      :block/path-refs (build-property-refs (:properties page) property-db-ids)})))]
                ;; blocks tx
                (reduce (fn [acc m]
@@ -218,4 +220,7 @@
                        []
                        blocks))))
           pages-and-blocks'))]
-    (into pages-and-blocks-tx new-properties-tx)))
+    ;; Properties first b/c they have schema. Then pages b/c they can be referenced by blocks
+    (vec (concat new-properties-tx
+                 (filter :block/name pages-and-blocks-tx)
+                 (remove :block/name pages-and-blocks-tx)))))

scripts/src/logseq/tasks/db_graph/create_graph_with_inferred_properties.cljs

@@ -63,7 +63,7 @@
                         ((juxt node-path/dirname node-path/basename) graph-dir)
                         [(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
         conn (create-graph/init-conn dir db-name)
-        blocks-tx (create-graph/create-blocks-tx @conn (create-init-data))]
+        blocks-tx (create-graph/create-blocks-tx (create-init-data))]
     (println "Generating" (count (filter :block/name blocks-tx)) "pages and"
              (count (filter :block/content blocks-tx)) "blocks ...")
     (d/transact! conn blocks-tx)

scripts/src/logseq/tasks/db_graph/create_graph_with_large_sizes.cljs

@@ -65,7 +65,7 @@
                         [(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
         conn (create-graph/init-conn dir db-name)
         _ (println "Building tx ...")
-        blocks-tx (create-graph/create-blocks-tx @conn (create-init-data options))]
+        blocks-tx (create-graph/create-blocks-tx (create-init-data options))]
     (println "Built" (count blocks-tx) "tx," (count (filter :block/name blocks-tx)) "pages and"
              (count (filter :block/content blocks-tx)) "blocks ...")
     ;; Vary the chunking with page size up to a max to avoid OOM

scripts/src/logseq/tasks/db_graph/create_graph_with_properties.cljs

@@ -108,7 +108,16 @@
         {:block/content (str "{{query (property :page-closed " (page-ref/->page-ref (string/capitalize (get-closed-value :page-closed))) ")}}")}
         {:block/content (str "{{query (property :date " (page-ref/->page-ref (string/capitalize (date-journal-title today))) ")}}")}
         {:block/content (str "{{query (property :date-many " (page-ref/->page-ref (string/capitalize (date-journal-title yesterday))) ")}}")}
-        {:block/content (str "{{query (property :date-closed " (page-ref/->page-ref (string/capitalize (get-closed-value :date-closed))) ")}}")}]}
+        #_{:block/content (str "{{query (property :date-closed " (page-ref/->page-ref (string/capitalize (get-closed-value :date-closed))) ")}}")}]}
+
+      ;; Property values
+      ;; Needs to be before page property pages b/c they are referenced by them
+      {:page {:block/name "page 1"}
+       :blocks
+       [{:block/content "yee"}
+        {:block/content "haw"}]}
+      {:page {:block/name "page 2"}}
+      {:page {:block/name "page 3"}}
 
       ;; Page property pages and queries
       {:page {:block/name "default page" :properties {:default "yolo"}}}
@@ -126,7 +135,7 @@
       {:page {:block/name "date page" :properties {:date [:page (date-journal-title today)]}}}
       {:page {:block/name "date-many page" :properties {:date-many #{[:page (date-journal-title today)]
                                                                      [:page (date-journal-title yesterday)]}}}}
-      {:page {:block/name "date-closed page" :properties {:date-closed (random-page-closed-value :date-closed)}}}
+      #_{:page {:block/name "date-closed page" :properties {:date-closed (random-page-closed-value :date-closed)}}}
       {:page {:block/original-name "Page Property Queries"}
        :blocks
        [{:block/content "{{query (page-property :default \"yolo\")}}"}
@@ -143,15 +152,7 @@
         {:block/content (str "{{query (page-property :page-closed " (page-ref/->page-ref (string/capitalize (get-closed-value :page-closed))) ")}}")}
         {:block/content (str "{{query (page-property :date " (page-ref/->page-ref (string/capitalize (date-journal-title today))) ")}}")}
         {:block/content (str "{{query (page-property :date-many " (page-ref/->page-ref (string/capitalize (date-journal-title yesterday))) ")}}")}
-        {:block/content (str "{{query (page-property :date-closed " (page-ref/->page-ref (string/capitalize (get-closed-value :date-closed))) ")}}")}]}
-
-      ;; Property values
-      {:page {:block/name "page 1"}
-       :blocks
-       [{:block/content "yee"}
-        {:block/content "haw"}]}
-      {:page {:block/name "page 2"}}
-      {:page {:block/name "page 3"}}]
+        {:block/content (str "{{query (page-property :date-closed " (page-ref/->page-ref (string/capitalize (get-closed-value :date-closed))) ")}}")}]}]
 
      ;; Properties
      :properties
@@ -162,7 +163,7 @@
           (into (mapv #(vector (keyword (str (name %) "-closed"))
                                {:closed-values (closed-values-config (keyword (str (name %) "-closed")))
                                 :block/schema {:type %}})
-                      [:default :url :number :page :date]))
+                      [:default :url :number #_:page #_:date]))
           (into {}))}))
 
 (def spec
@@ -185,7 +186,6 @@
                         [(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
         conn (create-graph/init-conn dir db-name {:additional-config (:config options)})
         blocks-tx (create-graph/create-blocks-tx
-                   @conn
                    (create-init-data)
                    {:property-uuids {:icon (:block/uuid (d/entity @conn :logseq.property/icon))}})]
     (println "Generating" (count (filter :block/name blocks-tx)) "pages and"

scripts/src/logseq/tasks/db_graph/create_graph_with_schema_org.cljs

@@ -391,7 +391,7 @@
         conn (create-graph/init-conn dir db-name)
         init-data (create-init-data (d/q '[:find [?name ...] :where [?b :block/name ?name]] @conn)
                                     options)
-        blocks-tx (create-graph/create-blocks-tx @conn init-data)]
+        blocks-tx (create-graph/create-blocks-tx init-data)]
     (println "Generating" (str (count (filter :block/name blocks-tx)) " pages with "
                                (count (:pages-and-blocks init-data)) " classes and "
                                (count (:properties init-data)) " properties ..."))