fix: frontend tests shouldn't depend on sqlite3

frontend.test.helper was accidentally coupled to sqlite3 via
logseq.outliner.db-pipeline. Split out sqlite3 dependent fns to
logseq.outliner.cli so it's clear this shouldn't be coupled to frontend

deps/db/script/create_graph.cljs

@@ -1,6 +1,6 @@
 (ns create-graph
   "An example script that creates a DB graph given a sqlite.build EDN file"
-  (:require [logseq.outliner.db-pipeline :as db-pipeline]
+  (:require [logseq.outliner.cli :as outliner-cli]
             [clojure.string :as string]
             [clojure.edn :as edn]
             [datascript.core :as d]
@@ -27,8 +27,8 @@
                         [(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
         sqlite-build-edn (merge {:auto-create-ontology? true}
                                 (-> (resolve-path edn-path) fs/readFileSync str edn/read-string))
-        conn (db-pipeline/init-conn dir db-name {:classpath (cp/get-classpath)})
-        {:keys [init-tx block-props-tx]} (db-pipeline/build-blocks-tx sqlite-build-edn)]
+        conn (outliner-cli/init-conn dir db-name {:classpath (cp/get-classpath)})
+        {:keys [init-tx block-props-tx]} (outliner-cli/build-blocks-tx sqlite-build-edn)]
     (println "Generating" (count (filter :block/name init-tx)) "pages and"
              (count (filter :block/content init-tx)) "blocks ...")
     (d/transact! conn init-tx)

deps/graph-parser/script/db_import.cljs

@@ -12,7 +12,7 @@
             [babashka.cli :as cli]
             [logseq.graph-parser.exporter :as gp-exporter]
             [logseq.common.graph :as common-graph]
-            [logseq.outliner.db-pipeline :as db-pipeline]
+            [logseq.outliner.cli :as outliner-cli]
             [promesa.core :as p]))
 
 (defn- build-graph-files
@@ -107,7 +107,7 @@
                           ((juxt node-path/dirname node-path/basename) graph-dir'))
                         [(node-path/join (os/homedir) "logseq" "graphs") db-graph-dir])
         file-graph' (resolve-path file-graph)
-        conn (db-pipeline/init-conn dir db-name)
+        conn (outliner-cli/init-conn dir db-name)
         directory? (.isDirectory (fs/statSync file-graph'))]
     (p/do!
      (if directory?

deps/outliner/.carve/config.edn

@@ -1,6 +1,7 @@
 {:paths ["src"]
  :api-namespaces [logseq.outliner.datascript-report
                   logseq.outliner.pipeline
+                  logseq.outliner.cli
                   logseq.outliner.core
                   logseq.outliner.db-pipeline
                   logseq.outliner.property

deps/outliner/src/logseq/outliner/cli.cljs

@@ -0,0 +1,48 @@
+(ns ^:node-only logseq.outliner.cli
+  "Primary ns for outliner CLI fns"
+    (:require [clojure.string :as string]
+              [datascript.core :as d]
+              [logseq.db.sqlite.create-graph :as sqlite-create-graph]
+              [logseq.db.sqlite.build :as sqlite-build]
+              [logseq.db.sqlite.db :as sqlite-db]
+              [logseq.outliner.db-pipeline :as db-pipeline]
+              ["fs" :as fs]
+              ["path" :as node-path]))
+
+(defn- find-on-classpath [classpath rel-path]
+  (some (fn [dir]
+          (let [f (node-path/join dir rel-path)]
+            (when (fs/existsSync f) f)))
+        (string/split classpath #":")))
+
+(defn- setup-init-data
+  "Setup initial data same as frontend.handler.repo/create-db"
+  [conn {:keys [additional-config classpath]}]
+  (let [config-content
+        (cond-> (or (some-> (find-on-classpath classpath "templates/config.edn") fs/readFileSync str)
+                    (do (println "Setting graph's config to empty since no templates/config.edn was found.")
+                        "{}"))
+          additional-config
+          ;; TODO: Replace with rewrite-clj when it's available
+          (string/replace-first #"(:file/name-format :triple-lowbar)"
+                                (str "$1 "
+                                     (string/replace-first (str additional-config) #"^\{(.*)\}$" "$1"))))]
+    (d/transact! conn (sqlite-create-graph/build-db-initial-data config-content))))
+
+(defn init-conn
+  "Create sqlite DB, initialize datascript connection and sync listener and then
+  transacts initial data. Takes the following options:
+   * :additional-config - Additional config map to merge into repo config.edn
+   * :classpath - A java classpath string i.e. paths delimited by ':'. Used to find default config.edn
+     that comes with Logseq"
+  [dir db-name & [opts]]
+  (fs/mkdirSync (node-path/join dir db-name) #js {:recursive true})
+  ;; Same order as frontend.db.conn/start!
+  (let [conn (sqlite-db/open-db! dir db-name)]
+    (db-pipeline/add-listener conn)
+    (setup-init-data conn opts)
+    conn))
+
+(def build-blocks-tx
+  "An alias for build-blocks-tx to specify default options for this ns"
+  sqlite-build/build-blocks-tx)

deps/outliner/src/logseq/outliner/db_pipeline.cljs

@@ -1,19 +1,13 @@
-(ns ^:node-only logseq.outliner.db-pipeline
-  "This ns provides a datascript listener for DB graphs and helper fns that
-  build on top of it.  The listener adds additional changes that the frontend
-  also adds per transact.  Missing features from frontend.worker.pipeline including:
+(ns logseq.outliner.db-pipeline
+  "This ns provides a datascript listener for DB graphs that is useful for CLIs
+  and testing (since it doesn't assume a frontend worker exists). The listener adds
+  additional changes that the frontend also adds per transact.  Missing features
+  from frontend.worker.pipeline including:
    * Deleted blocks don't update effected :block/tx-id
    * Delete empty property parent"
-  (:require [clojure.string :as string]
-            [datascript.core :as d]
-            [logseq.db.sqlite.create-graph :as sqlite-create-graph]
-            [logseq.db.sqlite.build :as sqlite-build]
-            [logseq.db.sqlite.db :as sqlite-db]
+  (:require [datascript.core :as d]
             [logseq.outliner.datascript-report :as ds-report]
-            [logseq.outliner.pipeline :as outliner-pipeline]
-            ["fs" :as fs]
-            ["path" :as node-path]))
-
+            [logseq.outliner.pipeline :as outliner-pipeline]))
 
 (defn- rebuild-block-refs
   [{:keys [db-after]} blocks]
@@ -45,42 +39,4 @@
   "Adds a listener to the datascript connection to add additional changes from outliner.pipeline"
   [conn]
   (d/listen! conn :pipeline-updates (fn pipeline-updates [tx-report]
-                                      (invoke-hooks conn tx-report))))
-
-(defn- find-on-classpath [classpath rel-path]
-  (some (fn [dir]
-          (let [f (node-path/join dir rel-path)]
-            (when (fs/existsSync f) f)))
-        (string/split classpath #":")))
-
-(defn- setup-init-data
-  "Setup initial data same as frontend.handler.repo/create-db"
-  [conn {:keys [additional-config classpath]}]
-  (let [config-content
-        (cond-> (or (some-> (find-on-classpath classpath "templates/config.edn") fs/readFileSync str)
-                    (do (println "Setting graph's config to empty since no templates/config.edn was found.")
-                        "{}"))
-          additional-config
-          ;; TODO: Replace with rewrite-clj when it's available
-          (string/replace-first #"(:file/name-format :triple-lowbar)"
-                                (str "$1 "
-                                     (string/replace-first (str additional-config) #"^\{(.*)\}$" "$1"))))]
-    (d/transact! conn (sqlite-create-graph/build-db-initial-data config-content))))
-
-(defn init-conn
-  "Create sqlite DB, initialize datascript connection and sync listener and then
-  transacts initial data. Takes the following options:
-   * :additional-config - Additional config map to merge into repo config.edn
-   * :classpath - A java classpath string i.e. paths delimited by ':'. Used to find default config.edn
-     that comes with Logseq"
-  [dir db-name & [opts]]
-  (fs/mkdirSync (node-path/join dir db-name) #js {:recursive true})
-  ;; Same order as frontend.db.conn/start!
-  (let [conn (sqlite-db/open-db! dir db-name)]
-    (add-listener conn)
-    (setup-init-data conn opts)
-    conn))
-
-(def build-blocks-tx
-  "An alias for build-blocks-tx to specify default options for this ns"
-  sqlite-build/build-blocks-tx)
+                                      (invoke-hooks conn tx-report))))

scripts/src/logseq/tasks/db_graph/create_graph_with_large_sizes.cljs

@@ -1,6 +1,6 @@
 (ns logseq.tasks.db-graph.create-graph-with-large-sizes
   "Script that generates graphs at large sizes"
-  (:require [logseq.outliner.db-pipeline :as db-pipeline]
+  (:require [logseq.outliner.cli :as outliner-cli]
             [clojure.string :as string]
             [datascript.core :as d]
             [babashka.cli :as cli]
@@ -66,9 +66,9 @@
         [dir db-name] (if (string/includes? graph-dir "/")
                         ((juxt node-path/dirname node-path/basename) graph-dir)
                         [(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
-        conn (db-pipeline/init-conn dir db-name {:classpath (cp/get-classpath)})
+        conn (outliner-cli/init-conn dir db-name {:classpath (cp/get-classpath)})
         _ (println "Building tx ...")
-        {:keys [init-tx]} (db-pipeline/build-blocks-tx (create-init-data options))]
+        {:keys [init-tx]} (outliner-cli/build-blocks-tx (create-init-data options))]
     (println "Built" (count init-tx) "tx," (count (filter :block/original-name init-tx)) "pages and"
              (count (filter :block/content init-tx)) "blocks ...")
     ;; Vary the chunking with page size up to a max to avoid OOM

scripts/src/logseq/tasks/db_graph/create_graph_with_properties.cljs

@@ -2,7 +2,7 @@
   "Script that generates all the permutations of property types and cardinality.
    Also creates a page of queries that exercises most properties
    NOTE: This script is also used in CI to confirm graph creation works"
-  (:require [logseq.outliner.db-pipeline :as db-pipeline]
+  (:require [logseq.outliner.cli :as outliner-cli]
             [logseq.common.util.date-time :as date-time-util]
             [logseq.common.util.page-ref :as page-ref]
             [logseq.db.frontend.property.type :as db-property-type]
@@ -182,9 +182,9 @@
         [dir db-name] (if (string/includes? graph-dir "/")
                         ((juxt node-path/dirname node-path/basename) graph-dir)
                         [(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
-        conn (db-pipeline/init-conn dir db-name {:additional-config (:config options)
-                                                 :classpath (cp/get-classpath)})
-        {:keys [init-tx block-props-tx]} (db-pipeline/build-blocks-tx (create-init-data))
+        conn (outliner-cli/init-conn dir db-name {:additional-config (:config options)
+                                                  :classpath (cp/get-classpath)})
+        {:keys [init-tx block-props-tx]} (outliner-cli/build-blocks-tx (create-init-data))
         existing-names (set (map :v (d/datoms @conn :avet :block/original-name)))
         conflicting-names (set/intersection existing-names (set (keep :block/original-name init-tx)))]
     (when (seq conflicting-names)

scripts/src/logseq/tasks/db_graph/create_graph_with_schema_org.cljs

@@ -10,7 +10,7 @@
      * Some properties are skipped because they are superseded/deprecated or because they have a property
        type logseq doesnt' support yet
      * schema.org assumes no cardinality. For now, only :page properties are given a :cardinality :many"
-  (:require [logseq.outliner.db-pipeline :as db-pipeline]
+  (:require [logseq.outliner.cli :as outliner-cli]
             [logseq.common.util :as common-util]
             [logseq.db.frontend.property :as db-property]
             [clojure.string :as string]
@@ -396,11 +396,11 @@
         [dir db-name] (if (string/includes? graph-dir "/")
                         ((juxt node-path/dirname node-path/basename) graph-dir)
                         [(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
-        conn (db-pipeline/init-conn dir db-name {:additional-config (:config options)
-                                                 :classpath (cp/get-classpath)})
+        conn (outliner-cli/init-conn dir db-name {:additional-config (:config options)
+                                                  :classpath (cp/get-classpath)})
         init-data (create-init-data (d/q '[:find [?name ...] :where [?b :block/name ?name]] @conn)
                                     options)
-        {:keys [init-tx block-props-tx]} (db-pipeline/build-blocks-tx init-data)]
+        {:keys [init-tx block-props-tx]} (outliner-cli/build-blocks-tx init-data)]
     (println "Generating" (str (count (filter :block/name init-tx)) " pages with "
                                (count (:classes init-data)) " classes and "
                                (count (:properties init-data)) " properties ..."))