Improve on create graph script

- Split out reusable fns to their own ns
- Created graph now initializes all the same data as in the app
- Had to make a copy of block-with-timestamps for nbb compatibility
  and started using it in a couple places that are easy to QA

deps/db/src/logseq/db/sqlite/util.cljs

@@ -57,3 +57,13 @@
                    [:block/uuid (str (some #(when (= (:db/id %) (:db/id v)) (:block/uuid %)) blocks))]]
                   [a v])))
          (transit/write t-writer))))
+
+(defn block-with-timestamps
+  "Copy of outliner-core/block-with-timestamps. Too basic to couple this to main app"
+  [block]
+  (let [updated-at (time-ms)
+        block (cond->
+               (assoc block :block/updated-at updated-at)
+                (nil? (:block/created-at block))
+                (assoc :block/created-at updated-at))]
+    block))

scripts/src/logseq/tasks/dev/create_graph.cljs → scripts/src/logseq/tasks/db_graph/create_graph.cljs

@@ -1,23 +1,26 @@
-(ns create-graph
-  "Script that generates a DB graph using an EDN format"
-  (:require [nbb.core :as nbb]
-            [logseq.db.sqlite.db :as sqlite-db]
+(ns logseq.tasks.db-graph.create-graph
+  "This ns provides fns to create a DB graph using EDN. See `init-conn` for
+  initializing a DB graph with a datascript connection that syncs to a sqlite DB
+  at the given directory. See `create-blocks-tx` for the EDN format to create a
+  graph. This ns can likely be used to also update graphs"
+  (:require [logseq.db.sqlite.db :as sqlite-db]
             [logseq.db.sqlite.util :as sqlite-util]
             [cljs-bean.core :as bean]
             [logseq.db :as ldb]
             [clojure.string :as string]
             [datascript.core :as d]
             ["fs" :as fs]
-            ["path" :as path]
-            ;; These namespaces should move to deps/
+            ["path" :as node-path]
+            [nbb.classpath :as cp]
+            ;; TODO: Move these namespaces to more stable deps/ namespaces
             [frontend.modules.datascript-report.core :as ds-report]
-            [frontend.modules.outliner.pipeline-util :as pipeline-util]))
+            [frontend.modules.outliner.pipeline-util :as pipeline-util]
+            [frontend.handler.common.repo :as repo-common-handler]))
 
 (defn- invoke-hooks
   "Modified copy frontend.modules.outliner.pipeline/invoke-hooks that doesn't
-  handle path-ref recalculation or persist to app db"
+  handle :block/path-refs recalculation"
   [{:keys [db-after] :as tx-report}]
-  ;; TODO: Add :block/path-refs to blocks
   (let [{:keys [blocks]} (ds-report/get-blocks-and-pages tx-report)
         deleted-block-uuids (set (pipeline-util/filter-deleted-blocks (:tx-data tx-report)))
         upsert-blocks (pipeline-util/build-upsert-blocks blocks deleted-block-uuids db-after)]
@@ -33,6 +36,36 @@
     (let [blocks' (mapv sqlite-util/ds->sqlite-block blocks)]
       (sqlite-db/upsert-blocks! db-name (bean/->js blocks')))))
 
+(defn- find-on-classpath [rel-path]
+  (some (fn [dir]
+          (let [f (node-path/join dir rel-path)]
+            (when (fs/existsSync f) f)))
+        (string/split (cp/get-classpath) #":")))
+
+(defn- setup-init-data
+  "Setup initial data same as frontend.handler.repo/create-db"
+  [conn]
+  ;; App doesn't persist :db/type but it does load it each time
+  (d/transact! conn [{:db/id -1 :db/ident :db/type :db/type "db"}])
+  (let [config-content (or (some-> (find-on-classpath "templates/config.edn") fs/readFileSync str)
+                           (do (println "Setting graph's config to empty since no templates/config.edn was found.")
+                               "{}"))]
+    (d/transact! conn (repo-common-handler/build-db-initial-data config-content))))
+
+(defn init-conn
+  "Create sqlite DB, initialize datascript connection and sync listener and then
+  transacts initial data"
+  [dir db-name]
+  (fs/mkdirSync (node-path/join dir db-name) #js {:recursive true})
+  (sqlite-db/open-db! dir db-name)
+  ;; Same order as frontend.db.conn/start!
+  (let [conn (ldb/start-conn :create-default-pages? false)]
+    (d/listen! conn :persist-to-sqlite (fn persist-to-sqlite [tx-report]
+                                         (update-sqlite-db db-name (invoke-hooks tx-report))))
+    (ldb/create-default-pages! conn)
+    (setup-init-data conn)
+    conn))
+
 (defn- translate-property-value
   "Translates a property value as needed. A value wrapped in vector indicates a reference type
    e.g. [:page \"some page\"]"
@@ -81,9 +114,29 @@
      :page-uuids page-uuids
      :block-uuids block-uuids}))
 
-(defn- create-blocks-tx
-  "For a map of pages to their blocks, this creates frontend blocks assuming only top level blocks
-   are desired. Anything more complex starts to recreate outliner namespaces"
+(defn create-blocks-tx
+  "Given an EDN map for defining pages, blocks and properties, this creates a
+  vector of transactable data for use with d/transact!. The EDN map is basic and
+  only supports defining blocks at the top level. The EDN map has the following keys:
+
+   * :pages-and-blocks - This is a vector of maps containing a :page key and optionally a :blocks
+     key when defining a page's blocks. More about each key:
+     * :page - This is a datascript attribute map e.g. `{:block/name \"foo\"}` .
+       :block/name is required and :properties can be passed to define page properties
+     * :blocks - This is a vec of datascript attribute maps e.g. `{:block/content \"bar\"}`.
+       :block/content is required and :properties can be passed to define block properties
+   * :properties - This is a map to configure properties where the keys are property names
+     and the values are maps of datascript attributes e.g. `{:block/schema {:type :checkbox}}`
+
+   The :properties for :pages-and-blocks is a map of property names to property
+   values.  Multiple property values for a many cardinality property are defined
+   as a set. The following property types are supported: :default, :url,
+   :checkbox, :number, :page and :block. :checkbox and :number values are written
+   as booleans and integers. :page and :block are references that are written as
+   vectors e.g. `[:page \"PAGE NAME\"]` and `[:block \"block content\"]`
+
+   Limitations:
+   * Page and block properties do not update :block/refs and :block/path-refs yet"
   [{:keys [pages-and-blocks properties]}]
   (let [;; add uuids before tx for refs in :properties
         pages-and-blocks' (mapv (fn [{:keys [page blocks]}]
@@ -136,87 +189,4 @@
                        []
                        blocks))))
           pages-and-blocks'))]
-    (into pages-and-blocks-tx new-properties-tx)))
-
-(defn- setup-db-graph
-  "Create sqlite DB and initialize datascript connection to sync to it"
-  [dir db-name]
-  (fs/mkdirSync (path/join dir db-name) #js {:recursive true})
-  (sqlite-db/open-db! dir db-name)
-  ;; Same order as frontend.db.conn/start!
-  (let [conn (ldb/start-conn :create-default-pages? false)]
-    (d/listen! conn :persist-to-sqlite (fn persist-to-sqlite [tx-report]
-                                         (update-sqlite-db db-name (invoke-hooks tx-report))))
-    (ldb/create-default-pages! conn)
-    conn))
-
-(defn- date-journal-title [date]
-  (let [title (.toLocaleString date "en-US" #js {:month "short" :day "numeric" :year "numeric"})
-        suffixes {1 "th" 21 "th" 31 "th" 2 "nd" 22 "nd" 3 "rd" 33 "rd"}]
-    (string/lower-case
-     (string/replace-first title #"(\d+)" (str "$1" (suffixes (.getDate date) "th"))))))
-
-(defn- date-journal-day [date]
-  (js/parseInt (str (.toLocaleString date "en-US" #js {:year "numeric"})
-                    (.toLocaleString date "en-US" #js {:month "2-digit"})
-                    (.toLocaleString date "en-US" #js {:day "2-digit"}))))
-
-(defn- create-init-data
-  []
-  {:pages-and-blocks
-   [{:page
-     (let [today (new js/Date)]
-       {:block/name (date-journal-title today) :block/journal? true :block/journal-day (date-journal-day today)})
-     :blocks
-     [{:block/content "[[Properties]]"}
-      {:block/content "[[Queries]]"}]}
-    {:page {:block/name "properties"}
-     :blocks
-     [{:block/content "default property block" :properties {:default "haha"}}
-      {:block/content "url property block" :properties {:url "https://logseq.com"}}
-      {:block/content "default-many property block" :properties {:default-many #{"woo" "hoo"}}}
-      {:block/content "url-many property block" :properties {:url-many #{"https://logseq.com" "https://docs.logseq.com"}}}
-      {:block/content "checkbox property block" :properties {:checkbox true}}
-      {:block/content "number property block" :properties {:number 5}}
-      {:block/content "number-many property block" :properties {:number-many #{5 10}}}
-      {:block/content "page property block" :properties {:page [:page "page 1"]}}
-      {:block/content "page-many property block" :properties {:page-many #{[:page "page 1"] [:page "page 2"]}}}
-      {:block/content "block property block" :properties {:block [:block "yee"]}}
-      {:block/content "block-many property block" :properties {:block-many #{[:block "yee"] [:block "haw"]}}}]}
-    {:page {:block/name "queries"}
-     :blocks
-     [{:block/content "{{query (property :default \"haha\")}}"}
-      {:block/content "{{query (property :url \"https://logseq.com\")}}"}
-      {:block/content "{{query (property :default-many \"woo\")}}"}
-      {:block/content "{{query (property :url-many \"https://logseq.com\")}}"}
-      {:block/content "{{query (property :checkbox true)}}"}
-      {:block/content "{{query (property :number 5)}}"}
-      {:block/content "{{query (property :number-many 10)}}"}
-      {:block/content "{{query (property :page \"Page 1\")}}"}
-      {:block/content "{{query (property :page-many \"Page 2\")}}"}]}
-    {:page {:block/name "page 1"}
-     :blocks
-     [{:block/content "yee"}
-      {:block/content "haw"}]}
-    {:page {:block/name "page 2"}}]
-   :properties
-   (->> [:default :url :checkbox :number :page :block]
-        (mapcat #(cond-> [[% {:block/schema {:type %}}]]
-                   (not= % :checkbox)
-                   (conj [(keyword (str (name %) "-many")) {:block/schema {:type % :cardinality :many}}])))
-        (into {}))})
-
-(defn -main [args]
-  (when (not= 1 (count args))
-    (println "Usage: $0 GRAPH-DIR")
-    (js/process.exit 1))
-  (let [[dir db-name] ((juxt path/dirname path/basename) (first args))
-        conn (setup-db-graph dir db-name)
-        blocks-tx (create-blocks-tx (create-init-data))]
-    (println "Generating" (count (filter :block/name blocks-tx)) "pages and"
-             (count (filter :block/content blocks-tx)) "blocks ...")
-    (d/transact! conn blocks-tx)
-    (println "Created graph" (str db-name "!"))))
-
-(when (= nbb/*file* (:file (meta #'-main)))
-  (-main *command-line-args*))
+    (into pages-and-blocks-tx new-properties-tx)))

scripts/src/logseq/tasks/db_graph/create_graph_with_properties.cljs

@@ -0,0 +1,79 @@
+(ns logseq.tasks.db-graph.create-graph-with-properties
+  "Script that generates all the permutations of property types and cardinality.
+   Also creates a page of queries that exercises most properties"
+  (:require [logseq.tasks.db-graph.create-graph :as create-graph]
+            [clojure.string :as string]
+            [datascript.core :as d]
+            ["path" :as node-path]
+            [nbb.core :as nbb]))
+
+(defn- date-journal-title [date]
+  (let [title (.toLocaleString date "en-US" #js {:month "short" :day "numeric" :year "numeric"})
+        suffixes {1 "th" 21 "th" 31 "th" 2 "nd" 22 "nd" 3 "rd" 33 "rd"}]
+    (string/lower-case
+     (string/replace-first title #"(\d+)" (str "$1" (suffixes (.getDate date) "th"))))))
+
+(defn- date-journal-day [date]
+  (js/parseInt (str (.toLocaleString date "en-US" #js {:year "numeric"})
+                    (.toLocaleString date "en-US" #js {:month "2-digit"})
+                    (.toLocaleString date "en-US" #js {:day "2-digit"}))))
+
+(defn- create-init-data
+  []
+  {:pages-and-blocks
+   [{:page
+     (let [today (new js/Date)]
+       {:block/name (date-journal-title today) :block/journal? true :block/journal-day (date-journal-day today)})
+     :blocks
+     [{:block/content "[[Properties]]"}
+      {:block/content "[[Queries]]"}]}
+    {:page {:block/name "properties"}
+     :blocks
+     [{:block/content "default property block" :properties {:default "haha"}}
+      {:block/content "url property block" :properties {:url "https://logseq.com"}}
+      {:block/content "default-many property block" :properties {:default-many #{"woo" "hoo"}}}
+      {:block/content "url-many property block" :properties {:url-many #{"https://logseq.com" "https://docs.logseq.com"}}}
+      {:block/content "checkbox property block" :properties {:checkbox true}}
+      {:block/content "number property block" :properties {:number 5}}
+      {:block/content "number-many property block" :properties {:number-many #{5 10}}}
+      {:block/content "page property block" :properties {:page [:page "page 1"]}}
+      {:block/content "page-many property block" :properties {:page-many #{[:page "page 1"] [:page "page 2"]}}}
+      {:block/content "block property block" :properties {:block [:block "yee"]}}
+      {:block/content "block-many property block" :properties {:block-many #{[:block "yee"] [:block "haw"]}}}]}
+    {:page {:block/name "queries"}
+     :blocks
+     [{:block/content "{{query (property :default \"haha\")}}"}
+      {:block/content "{{query (property :url \"https://logseq.com\")}}"}
+      {:block/content "{{query (property :default-many \"woo\")}}"}
+      {:block/content "{{query (property :url-many \"https://logseq.com\")}}"}
+      {:block/content "{{query (property :checkbox true)}}"}
+      {:block/content "{{query (property :number 5)}}"}
+      {:block/content "{{query (property :number-many 10)}}"}
+      {:block/content "{{query (property :page \"Page 1\")}}"}
+      {:block/content "{{query (property :page-many \"Page 2\")}}"}]}
+    {:page {:block/name "page 1"}
+     :blocks
+     [{:block/content "yee"}
+      {:block/content "haw"}]}
+    {:page {:block/name "page 2"}}]
+   :properties
+   (->> [:default :url :checkbox :number :page :block]
+        (mapcat #(cond-> [[% {:block/schema {:type %}}]]
+                   (not= % :checkbox)
+                   (conj [(keyword (str (name %) "-many")) {:block/schema {:type % :cardinality :many}}])))
+        (into {}))})
+
+(defn -main [args]
+  (when (not= 1 (count args))
+    (println "Usage: $0 GRAPH-DIR")
+    (js/process.exit 1))
+  (let [[dir db-name] ((juxt node-path/dirname node-path/basename) (first args))
+        conn (create-graph/init-conn dir db-name)
+        blocks-tx (create-graph/create-blocks-tx (create-init-data))]
+    (println "Generating" (count (filter :block/name blocks-tx)) "pages and"
+             (count (filter :block/content blocks-tx)) "blocks ...")
+    (d/transact! conn blocks-tx)
+    (println "Created graph" (str db-name "!"))))
+
+(when (= nbb/*file* (:file (meta #'-main)))
+  (-main *command-line-args*))

src/main/frontend/db/restore.cljs

@@ -14,10 +14,10 @@
             [goog.object :as gobj]
             [logseq.db.schema :as db-schema]
             [logseq.db.sqlite.restore :as sqlite-restore]
+            [logseq.db.sqlite.util :as sqlite-util]
             [promesa.core :as p]
             [frontend.util :as util]
             [cljs-time.core :as t]
-            [frontend.modules.outliner.core :as outliner-core]
             [logseq.graph-parser.property :as gp-property]))
 
 (defn- old-schema?
@@ -81,7 +81,7 @@
                       :block/name (util/page-name-sanity-lc k-name)
                       :block/uuid (:block/uuid property)
                       :block/type "property"}
-                     (outliner-core/block-with-timestamps
+                     (sqlite-util/block-with-timestamps
                       {:block/schema schema
                        :block/original-name (or original-name k-name)
                        :block/name (util/page-name-sanity-lc k-name)

src/main/frontend/handler/common/repo.cljs

@@ -0,0 +1,28 @@
+(ns ^:nbb-compatible frontend.handler.common.repo
+  (:require [datascript.core :as d]
+            [logseq.graph-parser.property :as gp-property]
+            [logseq.graph-parser.util :as gp-util]
+            [logseq.db.sqlite.util :as sqlite-util]))
+
+(defn build-db-initial-data
+  [config-content]
+  (let [initial-files [{:block/uuid (d/squuid)
+                        :file/path (str "logseq/" "config.edn")
+                        :file/content config-content}
+                       {:block/uuid (d/squuid)
+                        :file/path (str "logseq/" "custom.css")
+                        :file/content ""}
+                       {:block/uuid (d/squuid)
+                        :file/path (str "logseq/" "custom.js")
+                        :file/content ""}]
+        default-properties (map
+                            (fn [[k-keyword {:keys [schema original-name]}]]
+                              (let [k-name (name k-keyword)]
+                                (sqlite-util/block-with-timestamps
+                                 {:block/schema schema
+                                  :block/original-name (or original-name k-name)
+                                  :block/name (gp-util/page-name-sanity-lc k-name)
+                                  :block/uuid (d/squuid)
+                                  :block/type "property"})))
+                            gp-property/db-built-in-properties)]
+    (concat initial-files default-properties)))

src/main/frontend/handler/repo.cljs

@@ -12,6 +12,7 @@
             [frontend.handler.file :as file-handler]
             [frontend.handler.repo-config :as repo-config-handler]
             [frontend.handler.common.file :as file-common-handler]
+            [frontend.handler.common.repo :as repo-common-handler]
             [frontend.handler.route :as route-handler]
             [frontend.handler.ui :as ui-handler]
             [frontend.handler.global-config :as global-config-handler]
@@ -27,7 +28,6 @@
             [frontend.db.persist :as db-persist]
             [logseq.graph-parser :as graph-parser]
             [logseq.graph-parser.config :as gp-config]
-            [logseq.graph-parser.property :as gp-property]
             [electron.ipc :as ipc]
             [cljs-bean.core :as bean]
             [clojure.core.async :as async]
@@ -36,9 +36,7 @@
             [logseq.common.path :as path]
             [logseq.common.config :as common-config]
             [frontend.db.react :as react]
-            [frontend.db.listener :as db-listener]
-            [frontend.modules.outliner.core :as outliner-core]
-            [frontend.db.rtc.core :as rtc-core]))
+            [frontend.db.listener :as db-listener]))
 
 ;; Project settings should be checked in two situations:
 ;; 1. User changes the config.edn directly in logseq.com (fn: alter-file)
@@ -550,26 +548,7 @@
           _ (route-handler/redirect-to-home!)
           _ (db/transact! full-graph-name [(react/kv :db/type "db")]
                           {:skip-persist? true})
-          initial-files [{:block/uuid (db/new-block-id)
-                          :file/path (str "logseq/" "config.edn")
-                          :file/content config/config-default-content}
-                         {:block/uuid (db/new-block-id)
-                          :file/path (str "logseq/" "custom.css")
-                          :file/content ""}
-                         {:block/uuid (db/new-block-id)
-                          :file/path (str "logseq/" "custom.js")
-                          :file/content ""}]
-          default-properties (map
-                              (fn [[k-keyword {:keys [schema original-name]}]]
-                                (let [k-name (name k-keyword)]
-                                  (outliner-core/block-with-timestamps
-                                   {:block/schema schema
-                                    :block/original-name (or original-name k-name)
-                                    :block/name (util/page-name-sanity-lc k-name)
-                                    :block/uuid (db/new-block-id)
-                                    :block/type "property"})))
-                              gp-property/db-built-in-properties)
-          initial-data (concat initial-files default-properties)
+          initial-data (repo-common-handler/build-db-initial-data config/config-default-content)
           _ (db/transact! full-graph-name initial-data)
           _ (repo-config-handler/set-repo-config-state! full-graph-name config/config-default-content)
           ;; TODO: handle global graph

src/test/frontend/test/helper.cljs

@@ -5,7 +5,7 @@
             [frontend.db.conn :as conn]
             [clojure.string :as string]
             [clojure.set :as set]
-            [frontend.modules.outliner.core :as outliner-core]
+            [logseq.db.sqlite.util :as sqlite-util]
             [frontend.db :as db]
             [frontend.date :as date]
             [datascript.core :as d]
@@ -132,7 +132,7 @@
                               (into {}))
           ;; from upsert-property!
           new-properties-tx (mapv (fn [[prop-name uuid]]
-                                    (outliner-core/block-with-timestamps
+                                    (sqlite-util/block-with-timestamps
                                      {:block/uuid uuid
                                       :block/schema {:type :default}
                                       :block/original-name (name prop-name)
@@ -149,7 +149,7 @@
                           (map #(vector % (random-uuid)))
                           (into {}))
           page-tx (mapv (fn [[page-name uuid]]
-                          (outliner-core/block-with-timestamps
+                          (sqlite-util/block-with-timestamps
                            {:block/name (string/lower-case page-name)
                             :block/original-name page-name
                             :block/uuid uuid}))