batch insert

deps/db-sync/src/logseq/db_sync/batch.cljs

@@ -0,0 +1,18 @@
+(ns logseq.db-sync.batch
+  (:require [clojure.string :as string]))
+
+(def ^:private max-sql-params 999)
+(def ^:private row-param-count 3)
+
+(defn rows->insert-batches
+  [table rows {:keys [max-params] :or {max-params max-sql-params}}]
+  (if (seq rows)
+    (let [max-rows (max 1 (js/Math.floor (/ max-params row-param-count)))
+          batches (partition-all max-rows rows)]
+      (mapv (fn [batch]
+              (let [placeholders (string/join "," (repeat (count batch) "(?, ?, ?)"))
+                    sql (str "insert or replace into " table " (addr, content, addresses) values " placeholders)
+                    args (vec (mapcat identity batch))]
+                {:sql sql :args args}))
+            batches))
+    []))

deps/db-sync/src/logseq/db_sync/worker.cljs

@@ -6,6 +6,7 @@
             [lambdaisland.glogi.console :as glogi-console]
             [logseq.common.authorization :as authorization]
             [logseq.db :as ldb]
+            [logseq.db-sync.batch :as batch]
             [logseq.db-sync.common :as common :refer [cors-headers]]
             [logseq.db-sync.index :as index]
             [logseq.db-sync.malli-schema :as db-sync-schema]
@@ -254,13 +255,8 @@
 (defn- import-snapshot-rows!
   [sql table rows]
   (when (seq rows)
-    (doseq [[addr content addresses] rows]
-      (common/sql-exec sql
-                       (str "insert into " table " (addr, content, addresses) values (?, ?, ?)"
-                            " on conflict(addr) do update set content = excluded.content, addresses = excluded.addresses")
-                       addr
-                       content
-                       addresses))))
+    (doseq [{:keys [sql args]} (batch/rows->insert-batches table rows nil)]
+      (apply common/sql-exec sql sql args))))
 
 (defn- finalize-import!
   [^js self reset?]

deps/db-sync/test/logseq/db_sync/batch_test.cljs

@@ -0,0 +1,17 @@
+(ns logseq.db-sync.batch-test
+  (:require [cljs.test :refer [deftest is testing]]
+            [logseq.db-sync.batch :as batch]))
+
+(deftest rows->insert-batches-test
+  (testing "splits rows into batches based on max params"
+    (let [rows [[1 "a" nil]
+                [2 "b" nil]
+                [3 "c" nil]]
+          batches (batch/rows->insert-batches "kvs_import" rows {:max-params 6})]
+      (is (= 2 (count batches)))
+      (is (= [1 "a" nil 2 "b" nil] (:args (first batches))))
+      (is (= [3 "c" nil] (:args (second batches))))
+      (is (string? (:sql (first batches))))
+      (is (string? (:sql (second batches))))))
+  (testing "empty rows returns no batches"
+    (is (= [] (batch/rows->insert-batches "kvs_import" [] {:max-params 6})))))

deps/db-sync/worker/wrangler.toml

@@ -26,10 +26,6 @@ database_id = "c020574a-5623-407b-be0c-cd192bab9545"
 tag = "v1"
 new_sqlite_classes = [ "SyncDO" ]
 
-[[migrations]]
-tag = "v2"
-deleted_classes = ["SyncIndexDO"]
-
 [vars]
 COGNITO_JWKS_URL = "https://cognito-idp.us-east-1.amazonaws.com/us-east-1_dtagLnju8/.well-known/jwks.json"
 COGNITO_ISSUER = "https://cognito-idp.us-east-1.amazonaws.com/us-east-1_dtagLnju8"

insert-batches-test

@@ -0,0 +1,11 @@
+yarn run v1.22.22
+$ clojure -M:test compile test
+[:test] Compiling ...
+[:test] Build completed. (831 files, 25 compiled, 0 warnings, 6.94s)
+Done in 10.38s.
+yarn run v1.22.22
+$ node static/tests.js -v logseq.db-sync.batch-test/rows-
+:run-background-task :logseq.db.common.entity-plus/reset-immutable-entities-cache!
+:run-background-task :frontend.background-tasks/sync-to-worker-network-online-status
+:run-background-task :frontend.worker.embedding/subscribe-state
+info Visit https://yarnpkg.com/en/docs/cli/run for documentation about this command.