Merge branch 'feat/db' into feat/capacitor-new

.github/workflows/build.yml

@@ -2,10 +2,6 @@ name: CI
 
 on:
   push:
-    branches: [master]
-    paths-ignore:
-      - '*.md'
-  pull_request:
     branches: [master, "feat/db"]
     paths-ignore:
       - '*.md'

.github/workflows/clj-e2e.yml

@@ -2,14 +2,6 @@ name: Clojure E2E
 
 on:
   push:
-    branches: [master]
-    paths:
-      - 'clj-e2e/**'
-      - '.github/workflows/clj-e2e.yml'
-      - src/**
-      - deps/**
-      - packages/**
-  pull_request:
     branches: [master, "feat/db"]
     paths:
       - 'clj-e2e/**'

.github/workflows/clj-rtc-e2e.yml

@@ -2,14 +2,6 @@ name: Clojure RTC E2E
 
 on:
   push:
-    branches: [master]
-    paths:
-      - 'clj-e2e/**'
-      - '.github/workflows/clj-rtc-e2e.yml'
-      - src/**
-      - deps/**
-      - packages/**
-  pull_request:
     branches: [master, "feat/db"]
     paths:
       - 'clj-e2e/**'

.github/workflows/db.yml

@@ -3,12 +3,6 @@ name: logseq/db CI
 on:
   # Path filters ensure jobs only kick off if a change is made to db
   push:
-    branches: [master]
-    paths:
-      - 'deps/db/**'
-      - '.github/workflows/db.yml'
-      - '!deps/db/**.md'
-  pull_request:
     branches: [master, "feat/db"]
     paths:
       - 'deps/db/**'

.github/workflows/graph-parser.yml

@@ -4,17 +4,10 @@ on:
   # Path filters ensure jobs only kick off if a change is made to graph-parser or
   # its local dependencies
   push:
-    branches: [master]
-    paths:
-      - 'deps/graph-parser/**'
-      # db is a local dep that could break functionality in this lib and should trigger this
-      - 'deps/db/**'
-      - '.github/workflows/graph-parser.yml'
-      - '!deps/graph-parser/**.md'
-  pull_request:
     branches: [master, "feat/db"]
     paths:
       - 'deps/graph-parser/**'
+      # db is a local dep that could break functionality in this lib and should trigger this
       - 'deps/db/**'
       - '.github/workflows/graph-parser.yml'
       - '!deps/graph-parser/**.md'

.github/workflows/logseq-common.yml

@@ -3,12 +3,6 @@ name: logseq/common CI
 on:
   # Path filters ensure jobs only kick off if a change is made to common
   push:
-    branches: [master]
-    paths:
-      - 'deps/common/**'
-      - '.github/workflows/logseq-common.yml'
-      - '!deps/common/**.md'
-  pull_request:
     branches: [master, "feat/db"]
     paths:
       - 'deps/common/**'

.github/workflows/outliner.yml

@@ -4,17 +4,10 @@ on:
   # Path filters ensure jobs only kick off if a change is made to outliner or
   # its local dependencies
   push:
-    branches: [master]
-    paths:
-      - 'deps/outliner/**'
-      # db is a local dep that could break functionality in this lib and should trigger this
-      - 'deps/db/**'
-      - '.github/workflows/outliner.yml'
-      - '!deps/outliner/**.md'
-  pull_request:
     branches: [master, "feat/db"]
     paths:
       - 'deps/outliner/**'
+      # db is a local dep that could break functionality in this lib and should trigger this
       - 'deps/db/**'
       - '.github/workflows/outliner.yml'
       - '!deps/outliner/**.md'

.github/workflows/publishing.yml

@@ -4,17 +4,10 @@ on:
   # Path filters ensure jobs only kick off if a change is made to publishing or
   # its local dependencies
   push:
-    branches: [master]
-    paths:
-      - 'deps/publishing/**'
-      # db is a local dep that could break functionality in this lib and should trigger this
-      - 'deps/db/**'
-      - '.github/workflows/publishing.yml'
-      - '!deps/publishing/**.md'
-  pull_request:
     branches: [master, "feat/db"]
     paths:
       - 'deps/publishing/**'
+      # db is a local dep that could break functionality in this lib and should trigger this
       - 'deps/db/**'
       - '.github/workflows/publishing.yml'
       - '!deps/publishing/**.md'

clj-e2e/src/logseq/e2e/rtc.clj

@@ -54,3 +54,21 @@
 (defn rtc-stop
   []
   (util/search-and-click "(Dev) RTC Stop"))
+
+(defmacro with-stop-restart-rtc
+  "- rtc stop on `stop-pw-pages` in order
+  - run `body`
+  - rtc start and exec `after-start-body` in order"
+  [stop-pw-pages start-pw-page+after-start-body & body]
+  (let [after-body
+        (cons
+         'do
+         (for [[p body] (partition 2 start-pw-page+after-start-body)]
+           `(w/with-page ~p
+              (rtc-start)
+              ~body)))]
+    `(do
+       (doseq [p# ~stop-pw-pages]
+         (w/with-page p# (rtc-stop)))
+       ~@body
+       ~after-body)))

clj-e2e/test/logseq/e2e/rtc_extra_test.clj

@@ -145,17 +145,19 @@
               (reset! *latest-remote-tx remote-tx))))]
     (testing "add some task blocks while rtc disconnected on page1"
       (let [*latest-remote-tx (atom nil)]
-        (with-stop-restart-rtc @*page1 #(insert-task-blocks-in-page2 *latest-remote-tx))
-        (w/with-page @*page1
-          (rtc/wait-tx-update-to @*latest-remote-tx))
+        (rtc/with-stop-restart-rtc
+          [@*page1]
+          [@*page1 (rtc/wait-tx-update-to @*latest-remote-tx)]
+          (insert-task-blocks-in-page2 *latest-remote-tx))
         (validate-task-blocks)
         (validate-2-graphs)))
 
     (testing "update task blocks while rtc disconnected on page1"
       (let [*latest-remote-tx (atom nil)]
-        (with-stop-restart-rtc @*page1 #(update-task-blocks-in-page2 *latest-remote-tx))
-        (w/with-page @*page1
-          (rtc/wait-tx-update-to @*latest-remote-tx))
+        (rtc/with-stop-restart-rtc
+          [@*page1]
+          [@*page1 (rtc/wait-tx-update-to @*latest-remote-tx)]
+          (update-task-blocks-in-page2 *latest-remote-tx))
         (validate-task-blocks)
         (validate-2-graphs)))
 
@@ -187,9 +189,10 @@
               (reset! *latest-remote-tx remote-tx))))]
     (testing "add different types user properties on page2 while keeping rtc connected on page1"
       (let [*latest-remote-tx (atom nil)]
-        (with-stop-restart-rtc @*page1 #(insert-new-property-blocks-in-page2 *latest-remote-tx "rtc-property-test-1"))
-        (w/with-page @*page1
-          (rtc/wait-tx-update-to @*latest-remote-tx))
+        (rtc/with-stop-restart-rtc
+          [@*page1]
+          [@*page1 (rtc/wait-tx-update-to @*latest-remote-tx)]
+          (insert-new-property-blocks-in-page2 *latest-remote-tx "rtc-property-test-1"))
         (validate-2-graphs)))
 
     (new-logseq-page)
@@ -225,9 +228,10 @@
       ;; testing while rtc off then on
       (let [*latest-remote-tx (atom nil)]
         (new-logseq-page)
-        (with-stop-restart-rtc @*page1 #(test-fn-in-page2 *latest-remote-tx))
-        (w/with-page @*page1
-          (rtc/wait-tx-update-to @*latest-remote-tx))
+        (rtc/with-stop-restart-rtc
+          [@*page1]
+          [@*page1 (rtc/wait-tx-update-to @*latest-remote-tx)]
+          (test-fn-in-page2 *latest-remote-tx))
         (validate-2-graphs)))))
 
 (deftest rtc-outliner-conflict-update-test
@@ -243,30 +247,14 @@
           (rtc/wait-tx-update-to @*latest-remote-tx))
         (validate-2-graphs)))
     (testing "disconnect on page1 and page2, do some conflict updates, reconnect and check"
-      (w/with-page @*page1 (rtc/rtc-stop))
-      (w/with-page @*page2 (rtc/rtc-stop))
-
-      ;; TODO: more updates
-      (w/with-page @*page1
-        (w/click (format ".ls-block :text('%s')" (str title-prefix "-" 1)))
-        (b/indent))
-      (w/with-page @*page2
-        (w/click (format ".ls-block :text('%s')" (str title-prefix "-" 0)))
-        (b/delete-blocks))
-      (w/with-page @*page1 (rtc/rtc-start))
-      (w/with-page @*page2 (rtc/rtc-start))
-      (w/with-page @*page1 (rtc/with-wait-tx-updated (b/new-block "xxxx")))
-      (w/with-page @*page2 (rtc/with-wait-tx-updated (b/new-block "yyyy")))
+      (rtc/with-stop-restart-rtc
+        [@*page1 @*page2]
+        [@*page1 (rtc/with-wait-tx-updated (b/new-block "xxxx"))
+         @*page2 (rtc/with-wait-tx-updated (b/new-block "yyyy"))]
+        (w/with-page @*page1
+          (w/click (format ".ls-block :text('%s')" (str title-prefix "-" 1)))
+          (b/indent))
+        (w/with-page @*page2
+          (w/click (format ".ls-block :text('%s')" (str title-prefix "-" 0)))
+          (b/delete-blocks)))
       (validate-2-graphs))))
-
-(comment
-  (do (w/with-page @*page1 (rtc/rtc-stop))
-      (w/with-page @*page2 (rtc/rtc-stop)))
-
-  (do (w/with-page @*page1 (rtc/rtc-start))
-      (w/with-page @*page2 (rtc/rtc-start)))
-
-  (let [title-prefix "xxxx"
-        property-type "Text"]
-    (w/with-page @*page1
-      (b/new-block (str title-prefix "-" property-type)))))

deps/db/src/logseq/db/sqlite/create_graph.cljs

@@ -190,10 +190,27 @@
      :logseq.property/hide? true
      :logseq.property/built-in? true})])
 
+(defn- build-initial-files [config-content]
+  [{:block/uuid (common-uuid/gen-uuid :builtin-block-uuid "logseq/config.edn")
+    :file/path (str "logseq/" "config.edn")
+    :file/content config-content
+    :file/created-at (js/Date.)
+    :file/last-modified-at (js/Date.)}
+   {:block/uuid (common-uuid/gen-uuid :builtin-block-uuid "logseq/custom.css")
+    :file/path (str "logseq/" "custom.css")
+    :file/content ""
+    :file/created-at (js/Date.)
+    :file/last-modified-at (js/Date.)}
+   {:block/uuid (common-uuid/gen-uuid :builtin-block-uuid "logseq/custom.js")
+    :file/path (str "logseq/" "custom.js")
+    :file/content ""
+    :file/created-at (js/Date.)
+    :file/last-modified-at (js/Date.)}])
+
 (defn build-db-initial-data
   "Builds tx of initial data for a new graph including key values, initial files,
    built-in properties and built-in classes"
-  [config-content & {:keys [import-type]}]
+  [config-content & {:keys [import-type graph-git-sha]}]
   (assert (string? config-content))
   (let [initial-data (cond->
                       [(sqlite-util/kv :logseq.kv/db-type "db")
@@ -204,22 +221,10 @@
                        {:db/ident :logseq.property/empty-placeholder
                         :block/uuid (common-uuid/gen-uuid :builtin-block-uuid :logseq.property/empty-placeholder)}]
                        import-type
-                       (into (sqlite-util/import-tx import-type)))
-        initial-files [{:block/uuid (common-uuid/gen-uuid :builtin-block-uuid "logseq/config.edn")
-                        :file/path (str "logseq/" "config.edn")
-                        :file/content config-content
-                        :file/created-at (js/Date.)
-                        :file/last-modified-at (js/Date.)}
-                       {:block/uuid (common-uuid/gen-uuid :builtin-block-uuid "logseq/custom.css")
-                        :file/path (str "logseq/" "custom.css")
-                        :file/content ""
-                        :file/created-at (js/Date.)
-                        :file/last-modified-at (js/Date.)}
-                       {:block/uuid (common-uuid/gen-uuid :builtin-block-uuid "logseq/custom.js")
-                        :file/path (str "logseq/" "custom.js")
-                        :file/content ""
-                        :file/created-at (js/Date.)
-                        :file/last-modified-at (js/Date.)}]
+                       (into (sqlite-util/import-tx import-type))
+                       graph-git-sha
+                       (conj (sqlite-util/kv :logseq.kv/graph-git-sha graph-git-sha)))
+        initial-files (build-initial-files config-content)
         {properties-tx :tx :keys [properties]} (build-initial-properties)
         db-ident->properties (zipmap (map :db/ident properties) properties)
         default-classes (build-initial-classes db-ident->properties)

deps/graph-parser/script/db_import.cljs

@@ -163,7 +163,8 @@
         db-name (if (= 1 (count init-conn-args)) (first init-conn-args) (second init-conn-args))
         db-dir (if (= 1 (count init-conn-args)) (node-path/dirname (first init-conn-args)) (second init-conn-args))
         file-graph' (resolve-path file-graph)
-        conn (apply outliner-cli/init-conn (conj init-conn-args {:classpath (cp/get-classpath)}))
+        conn (apply outliner-cli/init-conn (conj init-conn-args {:classpath (cp/get-classpath)
+                                                                 :import-type :cli/db-import}))
         directory? (.isDirectory (fs/statSync file-graph'))
         user-options (cond-> (merge {:all-tags false} (dissoc options :verbose :files :help :continue))
                        ;; coerce option collection into strings

deps/outliner/src/logseq/outliner/cli.cljs

@@ -1,15 +1,16 @@
 (ns ^:node-only logseq.outliner.cli
   "Primary ns for outliner CLI fns"
-  (:require [borkdude.rewrite-edn :as rewrite]
+  (:require ["child_process" :as child-process]
+            ["fs" :as fs]
+            ["path" :as node-path]
+            [borkdude.rewrite-edn :as rewrite]
             [clojure.string :as string]
             [datascript.core :as d]
-            [logseq.db.sqlite.create-graph :as sqlite-create-graph]
-            [logseq.db.sqlite.build :as sqlite-build]
+            [logseq.common.config :as common-config]
             [logseq.db.common.sqlite-cli :as sqlite-cli]
-            [logseq.outliner.db-pipeline :as db-pipeline]
-            ["fs" :as fs]
-            ["path" :as node-path]
-            [logseq.common.config :as common-config]))
+            [logseq.db.sqlite.build :as sqlite-build]
+            [logseq.db.sqlite.create-graph :as sqlite-create-graph]
+            [logseq.outliner.db-pipeline :as db-pipeline]))
 
 (defn- find-on-classpath [classpath rel-path]
   (some (fn [dir]
@@ -26,6 +27,14 @@
               m)
       str))
 
+(defn- get-git-sha
+  []
+  (let [res (child-process/spawnSync "git"
+                                     #js ["rev-parse" "--short" "HEAD"]
+                                     #js {})]
+    (when (zero? (.-status res))
+      (string/trim (str (.-stdout res))))))
+
 (defn- setup-init-data
   "Setup initial data same as frontend.handler.repo/create-db"
   [conn {:keys [additional-config classpath import-type]
@@ -37,8 +46,11 @@
           true
           (common-config/create-config-for-db-graph)
           additional-config
-          (pretty-print-merge additional-config))]
-    (d/transact! conn (sqlite-create-graph/build-db-initial-data config-content {:import-type import-type}))))
+          (pretty-print-merge additional-config))
+        git-sha (get-git-sha)]
+    (d/transact! conn (sqlite-create-graph/build-db-initial-data config-content
+                                                                 (merge {:import-type import-type}
+                                                                        (when git-sha {:graph-git-sha git-sha}))))))
 
 (defn init-conn
   "Create sqlite DB, initialize datascript connection and sync listener and then

src/main/frontend/handler/repo.cljs

@@ -131,7 +131,8 @@
 (defn combine-local-&-remote-graphs
   [local-repos remote-repos]
   (when-let [repos' (seq (concat (map (fn [{:keys [sync-meta metadata] :as repo}]
-                                        (let [graph-id (some-> (or (:kv/value metadata) (second sync-meta)) str)]
+                                        (let [graph-id (some-> (or (:kv/value metadata)
+                                                                   (second sync-meta)) str)]
                                           (if graph-id (assoc repo :GraphUUID graph-id) repo)))
                                       local-repos)
                                  (some->> remote-repos
@@ -181,7 +182,8 @@
   (->
    (p/let [config (common-config/create-config-for-db-graph config/config-default-content)
            _ (persist-db/<new full-graph-name
-                              (cond-> {:config config}
+                              (cond-> {:config config
+                                       :graph-git-sha config/revision}
                                 file-graph-import? (assoc :import-type :file-graph)))
            _ (start-repo-db-if-not-exists! full-graph-name)
            _ (state/add-repo! {:url full-graph-name :root (config/get-local-dir full-graph-name)})

src/main/frontend/worker/db_metadata.cljs

@@ -1,13 +1,17 @@
 (ns frontend.worker.db-metadata
   "Fns to read/write metadata.edn file for db-based."
-  (:require [frontend.worker.util :as worker-util]
-            [promesa.core :as p]))
+  (:require ["/frontend/idbkv" :as idb-keyval]))
+
+(defonce ^:private store (delay (idb-keyval/newStore "localforage" "keyvaluepairs" 2)))
+
+(defn- gen-key
+  [repo]
+  (str "metadata###" repo))
 
 (defn <store
   [repo metadata-str]
-  (p/let [^js root (.getDirectory js/navigator.storage)
-          dir-handle (.getDirectoryHandle root (str "." (worker-util/get-pool-name repo)))
-          file-handle (.getFileHandle dir-handle "metadata.edn" #js {:create true})
-          writable (.createWritable file-handle)
-          _ (.write writable metadata-str)]
-    (.close writable)))
+  (idb-keyval/set (gen-key repo) metadata-str @store))
+
+(defn <get
+  [repo]
+  (idb-keyval/get (gen-key repo) @store))

src/main/frontend/worker/db_worker.cljs

@@ -14,6 +14,7 @@
             [frontend.common.missionary :as c.m]
             [frontend.common.thread-api :as thread-api :refer [def-thread-api]]
             [frontend.worker.db-listener :as db-listener]
+            [frontend.worker.db-metadata :as worker-db-metadata]
             [frontend.worker.db.fix :as db-fix]
             [frontend.worker.db.migrate :as db-migrate]
             [frontend.worker.db.validate :as worker-db-validate]
@@ -326,7 +327,7 @@
   (.exec db "PRAGMA journal_mode=WAL"))
 
 (defn- create-or-open-db!
-  [repo {:keys [config import-type datoms]}]
+  [repo {:keys [config import-type datoms] :as opts}]
   (when-not (worker-state/get-sqlite-conn repo)
     (p/let [[db search-db client-ops-db :as dbs] (get-dbs repo)
             storage (new-sqlite-storage db)
@@ -363,7 +364,7 @@
         (when (and db-based? (not initial-data-exists?) (not datoms))
           (let [config (or config "")
                 initial-data (sqlite-create-graph/build-db-initial-data config
-                                                                        (when import-type {:import-type import-type}))]
+                                                                        (select-keys opts [:import-type :graph-git-sha]))]
             (d/transact! conn initial-data {:initial-db? true})))
 
         ;; TODO: remove this once we can ensure there's no bug for missing addresses
@@ -407,13 +408,14 @@
 
 (defn- <list-all-dbs
   []
-  (let [dir? #(= (.-kind %) "directory")]
+  (let [dir? #(= (.-kind %) "directory")
+        db-dir-prefix ".logseq-pool-"]
     (p/let [^js root (.getDirectory js/navigator.storage)
             values-iter (when (dir? root) (.values root))
             values (when values-iter (iter->vec values-iter))
             current-dir-dirs (filter dir? values)
             db-dirs (filter (fn [file]
-                              (string/starts-with? (.-name file) ".logseq-pool-"))
+                              (string/starts-with? (.-name file) db-dir-prefix))
                             current-dir-dirs)]
       (prn :debug
            :db-dirs (map #(.-name %) db-dirs)
@@ -424,9 +426,8 @@
                                            ;; TODO: DRY
                                            (string/replace "+3A+" ":")
                                            (string/replace "++" "/"))
-                            metadata-file-handle (.getFileHandle dir "metadata.edn" #js {:create true})
-                            metadata-file (.getFile metadata-file-handle)
-                            metadata (.text metadata-file)]
+                            repo (str sqlite-util/db-version-prefix graph-name)
+                            metadata (worker-db-metadata/<get repo)]
                       {:name graph-name
                        :metadata (edn/read-string metadata)})) db-dirs)))))
 
@@ -467,7 +468,6 @@
 ;; [graph service]
 (defonce *service (atom []))
 (defonce fns {"remoteInvoke" thread-api/remote-function})
-(declare <init-service!)
 
 (defn- start-db!
   [repo {:keys [close-other-db?]
@@ -846,14 +846,14 @@
            (js/console.error (str "DB is not found for " repo))))))))
 
 (defn- on-become-master
-  [repo config import?]
+  [repo start-opts]
   (js/Promise.
    (m/sp
-     (c.m/<? (init-sqlite-module!))
-     (when-not import?
-       (c.m/<? (start-db! repo {:config config}))
-       (assert (some? (worker-state/get-datascript-conn repo))))
-     (m/? (rtc.core/new-task--rtc-start true)))))
+    (c.m/<? (init-sqlite-module!))
+    (when-not (:import-type start-opts)
+      (c.m/<? (start-db! repo start-opts))
+      (assert (some? (worker-state/get-datascript-conn repo))))
+    (m/? (rtc.core/new-task--rtc-start true)))))
 
 (def broadcast-data-types
   (set (map
@@ -866,7 +866,7 @@
          :rtc-sync-state])))
 
 (defn- <init-service!
-  [graph config import?]
+  [graph start-opts]
   (let [[prev-graph service] @*service]
     (some-> prev-graph close-db!)
     (when graph
@@ -874,9 +874,9 @@
         service
         (p/let [service (shared-service/<create-service graph
                                                         (bean/->js fns)
-                                                        #(on-become-master graph config import?)
+                                                        #(on-become-master graph start-opts)
                                                         broadcast-data-types
-                                                        {:import? import?})]
+                                                        {:import? (:import-type? start-opts)})]
           (assert (p/promise? (get-in service [:status :ready])))
           (reset! *service [graph service])
           service)))))
@@ -897,7 +897,7 @@
                                 ;; because shared-service operates at the graph level,
                                 ;; creating a new database or switching to another one requires re-initializing the service.
                                 (let [[graph opts] (ldb/read-transit-str (last args))]
-                                  (p/let [service (<init-service! graph (:config opts) (some? (:import-type opts)))]
+                                  (p/let [service (<init-service! graph opts)]
                                     (get-in service [:status :ready])
                                     ;; wait for service ready
                                     (js-invoke (:proxy service) k args)))

src/main/frontend/worker/device.cljs

@@ -99,7 +99,7 @@
     (let [device-uuid (c.m/<? (<get-item item-key-device-id))]
       (when-not device-uuid
         (let [get-ws-create-task (new-get-ws-create-task token)
-              agent-data (js->clj (.toJSON js/navigator.userAgentData) :keywordize-keys true)
+              agent-data (js->clj (some-> js/navigator.userAgentData .toJSON) :keywordize-keys true)
               generated-device-name (string/join
                                      "-"
                                      [(:platform agent-data)