Merge branch 'master' into refactor/page-parent

.clj-kondo/config.edn

@@ -248,4 +248,4 @@
            frontend.common.thread-api/defkeyword cljs.spec.alpha/def}
  :skip-comments true
  :output {:progress true
-          :exclude-files ["src/test/docs-0.10.9/"]}}
+          :exclude-files ["src/test/docs-0.10.12/"]}}

.github/ISSUE_TEMPLATE/config.yml

@@ -1,4 +1,7 @@
 contact_links:
+  - name: Database Version Issue
+    url: https://github.com/logseq/db-test/issues
+    about: For issues related to unreleased Database Version
   - name: Feature request
     url: https://discuss.logseq.com/new-topic?category=feature-requests
     about: Suggest an idea for Logseq

.github/workflows/build-android.yml

@@ -42,7 +42,7 @@ on:
 
 env:
   CLOJURE_VERSION: '1.11.1.1413'
-  NODE_VERSION: '22'
+  NODE_VERSION: '20'
   JAVA_VERSION: '17'
 
 jobs:

.github/workflows/build-demo.yml

@@ -19,7 +19,7 @@ on:
 
 env:
   CLOJURE_VERSION: '1.11.1.1413'
-  NODE_VERSION: '22'
+  NODE_VERSION: '20'
   JAVA_VERSION: '17'
 
 jobs:

.github/workflows/build-desktop-release.yml

@@ -48,7 +48,7 @@ on:
 
 env:
   CLOJURE_VERSION: '1.11.1.1413'
-  NODE_VERSION: '22'
+  NODE_VERSION: '20'
   JAVA_VERSION: '11'
 
 jobs:

.github/workflows/build-ios-release.yml

@@ -12,7 +12,7 @@ on:
 
 env:
   CLOJURE_VERSION: '1.11.1.1413'
-  NODE_VERSION: '22'
+  NODE_VERSION: '20'
   JAVA_VERSION: '11'
 
 jobs:

.github/workflows/build-ios.yml

@@ -17,7 +17,7 @@ on:
 
 env:
   CLOJURE_VERSION: '1.11.1.1413'
-  NODE_VERSION: '22'
+  NODE_VERSION: '20'
   JAVA_VERSION: '11'
 
 jobs:

.github/workflows/build-stage.yml

@@ -19,7 +19,7 @@ on:
 
 env:
   CLOJURE_VERSION: '1.11.1.1413'
-  NODE_VERSION: '22'
+  NODE_VERSION: '20'
   JAVA_VERSION: '17'
 
 jobs:

.github/workflows/build.yml

@@ -6,7 +6,7 @@ on:
     paths-ignore:
       - '*.md'
   pull_request:
-    branches: [master, "feat/db"]
+    branches: [master]
     paths-ignore:
       - '*.md'
 
@@ -14,7 +14,7 @@ env:
   CLOJURE_VERSION: '1.11.1.1413'
   JAVA_VERSION: '11'
   # This is the latest node version we can run.
-  NODE_VERSION: '22'
+  NODE_VERSION: '20'
   BABASHKA_VERSION: '1.0.168'
 
 jobs:

.github/workflows/clj-e2e.yml

@@ -2,7 +2,7 @@ name: Clojure E2E
 
 on:
   push:
-    branches: [master, "feat/db"]
+    branches: [master]
     paths:
       - 'clj-e2e/**'
       - '.github/workflows/clj-e2e.yml'
@@ -10,7 +10,7 @@ on:
       - deps/**
       - packages/**
   pull_request:
-    branches: [master, "feat/db"]
+    branches: [master]
     paths:
       - 'clj-e2e/**'
       - '.github/workflows/clj-e2e.yml'
@@ -84,8 +84,8 @@ jobs:
 
       - name: Run e2e tests
         run: cd clj-e2e && timeout 30m bb dev
-        env:
-          DEBUG: "pw:api"
+        # env:
+        #   DEBUG: "pw:api"
 
       - name: Collect screenshots
         if: ${{ failure() }}

.github/workflows/clj-rtc-e2e.yml

@@ -2,7 +2,7 @@ name: Clojure RTC E2E
 
 on:
   push:
-    branches: [master, "feat/db"]
+    branches: [master]
     paths:
       - 'clj-e2e/**'
       - '.github/workflows/clj-rtc-e2e.yml'
@@ -10,7 +10,7 @@ on:
       - deps/**
       - packages/**
   pull_request:
-    branches: [master, "feat/db"]
+    branches: [master]
     paths:
       - 'clj-e2e/**'
       - '.github/workflows/clj-rtc-e2e.yml'
@@ -85,8 +85,8 @@ jobs:
 
       - name: Run e2e tests
         run: cd clj-e2e && timeout 30m bb run-rtc-extra-test
-        env:
-          DEBUG: "pw:api"
+        # env:
+        # DEBUG: "pw:api"
 
       - name: Collect screenshots
         if: ${{ failure() }}

.github/workflows/db.yml

@@ -9,7 +9,7 @@ on:
       - '.github/workflows/db.yml'
       - '!deps/db/**.md'
   pull_request:
-    branches: [master, "feat/db"]
+    branches: [master]
     paths:
       - 'deps/db/**'
       - '.github/workflows/db.yml'
@@ -23,7 +23,7 @@ env:
   CLOJURE_VERSION: '1.11.1.1413'
   JAVA_VERSION: '11'
   # This is the latest node version we can run.
-  NODE_VERSION: '22'
+  NODE_VERSION: '20'
   BABASHKA_VERSION: '1.0.168'
 
 jobs:

.github/workflows/e2e.yml

@@ -19,11 +19,12 @@ env:
   CLOJURE_VERSION: '1.11.1.1413'
   JAVA_VERSION: '11'
   # This is the latest node version we can run.
-  NODE_VERSION: '22'
+  NODE_VERSION: '20'
   BABASHKA_VERSION: '1.0.168'
 
 jobs:
   e2e-test-build:
+    if: false                   # disabled
     name: Build Test Artifact
     runs-on: ubuntu-22.04
     steps:

.github/workflows/graph-parser.yml

@@ -12,7 +12,7 @@ on:
       - '.github/workflows/graph-parser.yml'
       - '!deps/graph-parser/**.md'
   pull_request:
-    branches: [master, "feat/db"]
+    branches: [master]
     paths:
       - 'deps/graph-parser/**'
       - 'deps/db/**'
@@ -28,7 +28,7 @@ env:
   # This is the same as 1.8.
   JAVA_VERSION: '11'
   # This is the latest node version we can run.
-  NODE_VERSION: '22'
+  NODE_VERSION: '20'
   BABASHKA_VERSION: '1.0.168'
 
 jobs:

.github/workflows/logseq-common.yml

@@ -9,7 +9,7 @@ on:
       - '.github/workflows/logseq-common.yml'
       - '!deps/common/**.md'
   pull_request:
-    branches: [master, "feat/db"]
+    branches: [master]
     paths:
       - 'deps/common/**'
       - '.github/workflows/logseq-common.yml'
@@ -23,7 +23,7 @@ env:
   CLOJURE_VERSION: '1.11.1.1413'
   JAVA_VERSION: '11'
   # This is the latest node version we can run.
-  NODE_VERSION: '22'
+  NODE_VERSION: '20'
   BABASHKA_VERSION: '1.0.168'
 
 jobs:

.github/workflows/outliner.yml

@@ -12,7 +12,7 @@ on:
       - '.github/workflows/outliner.yml'
       - '!deps/outliner/**.md'
   pull_request:
-    branches: [master, "feat/db"]
+    branches: [master]
     paths:
       - 'deps/outliner/**'
       - 'deps/db/**'

.github/workflows/publishing.yml

@@ -12,7 +12,7 @@ on:
       - '.github/workflows/publishing.yml'
       - '!deps/publishing/**.md'
   pull_request:
-    branches: [master, "feat/db"]
+    branches: [master]
     paths:
       - 'deps/publishing/**'
       - 'deps/db/**'
@@ -28,7 +28,7 @@ env:
   # This is the same as 1.8.
   JAVA_VERSION: '11'
   # This is the latest node version we can run.
-  NODE_VERSION: '22'
+  NODE_VERSION: '20'
   BABASHKA_VERSION: '1.0.168'
 
 jobs:

.gitignore

@@ -49,6 +49,7 @@ charlie/
 docker
 android/app/src/main/assets/capacitor.plugin.json
 ios/App/App/capacitor.config.json
+ios/App/App/public
 
 startup.png
 
@@ -72,3 +73,4 @@ clj-e2e/.wally
 clj-e2e/resources
 clj-e2e/e2e-dump
 .dir-locals.el
+.projectile

.projectile

@@ -1,16 +0,0 @@
--/.git
--/.cpcache
--/.shadow-cljs/
--/node_modules/
--/.cpcache
--/.shadow-cljs/
--/resources/static/js/cljs-runtime/
--/resources/static/js/main.js
--/resources/static/js/sentry.min.js
--/resources/static/js/highlight.min.js
--/resources/static/js/katex.min.js
--/resources/static/js/mhchem.min.js
--/resources/static/js/mldoc.min.js
--/resources/static/js/sci.min.js
--/resources/static/js/excalidraw.min.js
--/resources/static/js/react-force-graph.min.js

README.md

@@ -54,6 +54,7 @@
 
 ## Table of Contents
 
+  * [<g-emoji class="g-emoji" alias="database" fallback-src="https://github.githubassets.com/images/icons/emoji/unicode/1f680.png">🚀</g-emoji> Database Version](#-database-version)
   * [<g-emoji class="g-emoji" alias="thinking" fallback-src="https://github.githubassets.com/images/icons/emoji/unicode/1f914.png">🤔</g-emoji> Why Logseq?](#-why-logseq)
   * [<g-emoji class="g-emoji" alias="eyes" fallback-src="https://github.githubassets.com/images/icons/emoji/unicode/1f440.png">👀</g-emoji> How can I use it?](#-how-can-i-use-it)
   * [<g-emoji class="g-emoji" alias="books" fallback-src="https://github.githubassets.com/images/icons/emoji/unicode/1f4da.png">📚</g-emoji> Learn more](#-learn-more)
@@ -65,6 +66,14 @@
   * [<g-emoji class="g-emoji" alias="sparkles" fallback-src="https://github.githubassets.com/images/icons/emoji/unicode/2728.png">✨</g-emoji> Inspiration](#-inspiration)
 * [<g-emoji class="g-emoji" alias="pray" fallback-src="https://github.githubassets.com/images/icons/emoji/unicode/1f64f.png">🙏</g-emoji> Thank You](#-thank-you)
 
+## 🚀 Database Version
+
+The Database version (DB version) of Logseq introduces DB graphs while maintaining support for file graphs. [See this page](https://github.com/logseq/docs/blob/master/db-version.md) to get an overview of the main features for DB graphs. If you are an existing user, [see this page](https://github.com/logseq/docs/blob/master/db-version-changes.md) to get an overview of changes with the DB version.
+
+The DB version of Logseq is alpha software. When using DB graphs, we recommend you create a dedicated test graph and choose one project/workflow that’s not crucial for you. **Data loss is possible**, which is why we recommend [automated backups](https://github.com/logseq/docs/blob/master/db-version.md#automated-backup) or making [regular SQLite DB backups](https://github.com/logseq/docs/blob/master/db-version.md#graph-export). When using file graphs, **data corruption is possible** as some file content can be duplicated. We only recommend using file graphs if you are making regular backups with git.
+
+To try the latest web version, go to https://test.logseq.com/. For DB version issues, please report them to https://github.com/logseq/db-test/issues. To try the latest desktop version, go to https://github.com/logseq/logseq/actions/workflows/build-desktop-release.yml and click on the latest release. Scroll to the bottom and under the `Artifacts` section download the artifact for your operating system.
+
 ## 🤔 Why Logseq?
 
 [Logseq](https://logseq.com) is a **knowledge management** and **collaboration** platform. It focuses on **privacy**, **longevity**, and [**user control**](https://www.gnu.org/philosophy/free-sw.en.html). Logseq offers a range of **powerful tools** for **knowledge management**, **collaboration**, **PDF annotation**, and **task management** with support for multiple file formats, including **Markdown** and **Org-mode**, and **various features** for organizing and structuring your notes.

clj-e2e/dev/user.clj

@@ -7,9 +7,11 @@
             [logseq.e2e.fixtures :as fixtures]
             [logseq.e2e.graph :as graph]
             [logseq.e2e.keyboard :as k]
+            [logseq.e2e.locator :as loc]
             [logseq.e2e.multi-tabs-basic-test]
             [logseq.e2e.outliner-basic-test]
             [logseq.e2e.plugins-basic-test]
+            [logseq.e2e.property-basic-test]
             [logseq.e2e.reference-basic-test]
             [logseq.e2e.rtc-basic-test]
             [logseq.e2e.rtc-extra-test]
@@ -21,7 +23,7 @@
 (reset! config/*port 3001)
 ;; show ui
 (reset! config/*headless false)
-(reset! config/*slow-mo 50)
+(reset! config/*slow-mo 30)
 
 (def *futures (atom {}))
 
@@ -35,6 +37,11 @@
   (->> (future (run-tests 'logseq.e2e.commands-basic-test))
        (swap! *futures assoc :commands-test)))
 
+(defn run-property-basic-test
+  []
+  (->> (future (run-tests 'logseq.e2e.property-basic-test))
+       (swap! *futures assoc :property-test)))
+
 (defn run-outliner-test
   []
   (->> (future (run-tests 'logseq.e2e.outliner-basic-test))
@@ -72,7 +79,8 @@
              'logseq.e2e.outliner-basic-test
              'logseq.e2e.rtc-basic-test
              'logseq.e2e.plugins-basic-test
-             'logseq.e2e.reference-basic-test))
+             'logseq.e2e.reference-basic-test
+             'logseq.e2e.property-basic-test))
 
 (defn start
   []

clj-e2e/src/logseq/e2e/assert.clj

@@ -40,10 +40,7 @@
 
 (defn assert-editor-mode
   []
-  (let [klass ".editor-wrapper textarea"
-        editor (w/-query klass)]
-    (w/wait-for editor)
-    editor))
+  (assert-have-count ".editor-wrapper textarea" 1))
 
 (defn assert-selected-block-text
   [text]

clj-e2e/src/logseq/e2e/block.clj

@@ -8,27 +8,38 @@
             [wally.main :as w]))
 
 (defn open-last-block
+  "Open the last existing block or pressing add button to create a new block"
   []
   (util/double-esc)
   (assert/assert-in-normal-mode?)
-  (w/click (last (w/query ".ls-page-blocks .ls-block .block-content"))))
+  (let [blocks-count (util/page-blocks-count)
+        last-block (-> (if (zero? blocks-count)
+                         (w/query ".ls-page-blocks .block-add-button")
+                         (w/query ".ls-page-blocks .page-blocks-inner .ls-block .block-content"))
+                       (last))]
+    (w/click last-block)))
 
 (defn save-block
   [text]
+  (assert/assert-have-count util/editor-q 1)
   (w/click util/editor-q)
   (w/fill util/editor-q text)
   (assert/assert-is-visible (loc/filter util/editor-q :has-text text)))
 
 (defn new-block
   [title]
-  (let [editor (util/get-editor)
-        blocks-count (util/blocks-count)]
+  (let [editor (util/get-editor)]
     (when-not editor (open-last-block))
     (assert/assert-editor-mode)
-    (k/enter)
-    (assert/assert-have-count ".ls-block" (inc blocks-count))
-    (assert/assert-editor-mode)
-    (save-block title)))
+    (let [last-id (.getAttribute (w/-query ".editor-wrapper textarea") "id")]
+      (is (some? last-id))
+      (k/enter)
+      (assert/assert-is-visible
+       (loc/filter ".editor-wrapper"
+                   :has "textarea"
+                   :has-not (str "#" last-id)))
+      (assert/assert-editor-mode)
+      (save-block title))))
 
 ;; TODO: support tree
 (defn new-blocks
@@ -37,7 +48,7 @@
     (when-not editor? (open-last-block))
     (assert/assert-editor-mode)
     (let [value (util/get-edit-content)]
-      (if (string/blank? value)           ; empty block
+      (if (string/blank? value)         ; empty block
         (save-block (first titles))
         (new-block (first titles))))
     (doseq [title (rest titles)]
@@ -62,6 +73,7 @@
 
 (defn wait-editor-text
   [text]
+  (assert/assert-have-count util/editor-q 1)
   (w/wait-for (format ".editor-wrapper textarea:text('%s')" text)))
 
 (def copy #(k/press "ControlOrMeta+c" {:delay 100}))

clj-e2e/src/logseq/e2e/keyboard.clj

@@ -1,6 +1,8 @@
 (ns logseq.e2e.keyboard
   (:require [wally.main :as w]))
 
+(def ^:private mac? (= "Mac OS X" (System/getProperty "os.name")))
+
 (def press w/keyboard-press)
 
 (def enter #(press "Enter"))
@@ -9,5 +11,11 @@
 (def tab #(press "Tab"))
 (def shift+tab #(press "Shift+Tab"))
 (def shift+enter #(press "Shift+Enter"))
+(def shift+arrow-up #(press "Shift+ArrowUp"))
+(def shift+arrow-down #(press "Shift+ArrowDown"))
+
 (def arrow-up #(press "ArrowUp"))
 (def arrow-down #(press "ArrowDown"))
+
+(def meta+shift+arrow-up #(press (str (if mac? "Meta" "Alt") "+Shift+ArrowUp")))
+(def meta+shift+arrow-down #(press (str (if mac? "Meta" "Alt") "+Shift+ArrowDown")))

clj-e2e/src/logseq/e2e/page.clj

@@ -1,13 +1,19 @@
 (ns logseq.e2e.page
-  (:require [logseq.e2e.assert :as assert]
+  (:require [logseq.e2e.keyboard :as k]
             [logseq.e2e.util :as util]
             [wally.main :as w]
-            [wally.selectors :as ws]))
+            [wally.selectors :as ws])
+  (:import (com.microsoft.playwright TimeoutError)))
 
 (defn goto-page
   [page-name]
   (assert (string? page-name) page-name)
-  (util/search-and-click page-name))
+  (try
+    (util/search-and-click page-name)
+    (catch TimeoutError _e
+      ;; try one more time
+      (k/esc)
+      (util/search-and-click page-name))))
 
 (defn new-page
   [title]

clj-e2e/src/logseq/e2e/rtc.clj

@@ -20,6 +20,8 @@
      ~@body
      (loop [i# 5]
        (when (zero? i#) (throw (ex-info "wait-tx-updated failed" {:old m# :new (get-rtc-tx)})))
+       (util/wait-timeout 500)
+       (w/wait-for "button.cloud.on.idle")
        (util/wait-timeout 1000)
        (let [new-m# (get-rtc-tx)
              new-local-tx# (or (:local-tx new-m#) 0)
@@ -52,3 +54,21 @@
 (defn rtc-stop
   []
   (util/search-and-click "(Dev) RTC Stop"))
+
+(defmacro with-stop-restart-rtc
+  "- rtc stop on `stop-pw-pages` in order
+  - run `body`
+  - rtc start and exec `after-start-body` in order"
+  [stop-pw-pages start-pw-page+after-start-body & body]
+  (let [after-body
+        (cons
+         'do
+         (for [[p body] (partition 2 start-pw-page+after-start-body)]
+           `(w/with-page ~p
+              (rtc-start)
+              ~body)))]
+    `(do
+       (doseq [p# ~stop-pw-pages]
+         (w/with-page p# (rtc-stop)))
+       ~@body
+       ~after-body)))

clj-e2e/src/logseq/e2e/util.clj

@@ -48,7 +48,8 @@
 
 (defn get-edit-block-container
   []
-  (first (w/query ".ls-block" {:has (w/-query ".editor-wrapper textarea")})))
+  (assert/assert-have-count editor-q 1)
+  (first (w/query ".ls-block" {:has (w/-query editor-q)})))
 
 (defn input
   "Notice this will replace the existing input value with `text`"
@@ -97,11 +98,11 @@
 (defn blocks-count
   "Blocks count including page title"
   []
-  (count-elements ".ls-block"))
+  (count-elements ".ls-block:not(.block-add-button)"))
 
 (defn page-blocks-count
   []
-  (count-elements ".ls-page-blocks .ls-block"))
+  (count-elements ".ls-page-blocks .page-blocks-inner .ls-block"))
 
 (defn exit-edit
   []
@@ -131,7 +132,7 @@
 
 (defn get-page-blocks-contents
   []
-  (w/all-text-contents ".ls-page-blocks .ls-block .block-title-wrap"))
+  (w/all-text-contents ".ls-page-blocks .ls-block:not(.block-add-button) .block-title-wrap"))
 
 (def mac? (= "Mac OS X" (System/getProperty "os.name")))
 
@@ -183,7 +184,7 @@
   (w/click (first (w/query (format "a.menu-link:has-text(\"%s\")" tag))))
   ;; wait tag added on ui
   (assert/assert-is-visible
-   (-> ".ls-block"
+   (-> ".ls-block:not(.block-add-button)"
        (loc/filter :has ".editor-wrapper textarea")
        (loc/filter :has (format ".block-tag :text('%s')" tag)))))
 

clj-e2e/test/logseq/e2e/commands_basic_test.clj

@@ -1,7 +1,5 @@
 (ns logseq.e2e.commands-basic-test
   (:require
-   [clj-time.core :as t]
-   [clj-time.local :as tl]
    [clojure.string :as string]
    [clojure.test :refer [deftest testing is use-fixtures]]
    [logseq.e2e.assert :as assert]
@@ -297,12 +295,3 @@
     (util/exit-edit)
     (w/click "a.cloze")
     (w/wait-for "a.cloze-revealed")))
-
-(deftest new-property-test
-  (testing "new property"
-    (b/new-block "")
-    (util/input-command "add new property")
-    (util/input "p1")
-    (w/click "a:has-text(\"+ New option: p1\")")
-    (k/enter)
-    (is (= "p1" (util/get-text "a.property-k")))))

clj-e2e/test/logseq/e2e/custom_report.clj

@@ -13,6 +13,8 @@
 
 (def ^:dynamic *pw-page->console-logs* nil)
 
+(def ^:dynamic *preserve-graph* nil)
+
 (defn screenshot
   [page test-name]
   (println :screenshot test-name)
@@ -22,6 +24,22 @@
        (.setPath (java.nio.file.Paths/get "e2e-dump/"
                                           (into-array [(format "./screenshot-%s-%s.png" test-name (System/currentTimeMillis))]))))))
 
+(defn- collect-info-when-error-or-failed
+  []
+  ;; screenshot for all pw pages when :error
+  (when-let [all-contexts (seq *pw-contexts*)]
+    (doseq [page (mapcat pw-page/get-pages all-contexts)]
+      (screenshot page (string/join "-" (map (comp str :name meta) t/*testing-vars*)))))
+
+  ;; dump console logs
+  (when-let [pw-page->console-logs (some-> *pw-page->console-logs* deref)]
+    (doseq [[_pw-page logs] pw-page->console-logs]
+      (spit (format "e2e-dump/console-logs-%s.txt" (System/currentTimeMillis))
+            (with-out-str (pp/pprint logs)))))
+
+  (when (some? *preserve-graph*)
+    (set! *preserve-graph* true)))
+
 (defmethod t/report :error
   [m]
   ;; copy from default impl
@@ -37,16 +55,7 @@
         (stack/print-cause-trace actual t/*stack-trace-depth*)
         (prn actual))))
 
-  ;; screenshot for all pw pages when :error
-  (when-let [all-contexts (seq *pw-contexts*)]
-    (doseq [page (mapcat pw-page/get-pages all-contexts)]
-      (screenshot page (string/join "-" (map (comp str :name meta) t/*testing-vars*)))))
-
-  ;; dump console logs
-  (when-let [pw-page->console-logs (some-> *pw-page->console-logs* deref)]
-    (doseq [[_pw-page logs] pw-page->console-logs]
-      (spit (format "e2e-dump/console-logs-%s.txt" (System/currentTimeMillis))
-            (with-out-str (pp/pprint logs))))))
+  (collect-info-when-error-or-failed))
 
 (defmethod t/report :fail
   [m]
@@ -58,13 +67,4 @@
     (println "expected:" (pr-str (:expected m)))
     (println "  actual:" (pr-str (:actual m))))
 
-  ;; screenshot for all pw pages when :fail
-  (when-let [all-contexts (seq *pw-contexts*)]
-    (doseq [page (mapcat pw-page/get-pages all-contexts)]
-      (screenshot page (string/join "-" (map (comp str :name meta) t/*testing-vars*)))))
-
-;; dump console logs
-  (when-let [pw-page->console-logs (some-> *pw-page->console-logs* deref)]
-    (doseq [[_pw-page logs] pw-page->console-logs]
-      (spit (format "e2e-dump/console-logs-%s.txt" (System/currentTimeMillis))
-            (with-out-str (pp/pprint logs))))))
+  (collect-info-when-error-or-failed))

clj-e2e/test/logseq/e2e/property_basic_test.clj

@@ -0,0 +1,52 @@
+(ns logseq.e2e.property-basic-test
+  (:require [clojure.test :refer [deftest testing is use-fixtures run-test run-tests]]
+            [logseq.e2e.assert :as assert]
+            [logseq.e2e.block :as b]
+            [logseq.e2e.fixtures :as fixtures]
+            [logseq.e2e.keyboard :as k]
+            [logseq.e2e.locator :as loc]
+            [logseq.e2e.util :as util]
+            [wally.main :as w]))
+
+(use-fixtures :once fixtures/open-page)
+
+(use-fixtures :each
+  fixtures/new-logseq-page
+  fixtures/validate-graph)
+
+(def ^:private property-types ["Text" "Number" "Date" "DateTime" "Checkbox" "Url" "Node"])
+
+(defn add-new-properties
+  [title-prefix]
+  (b/new-blocks (map #(str title-prefix "-" %) property-types))
+  (doseq [property-type property-types]
+    (let [property-name (str "p-" title-prefix "-" property-type)]
+      (w/click (util/get-by-text (str title-prefix "-" property-type) true))
+      (k/press "Control+e")
+      (util/input-command "Add new property")
+      (w/click "input[placeholder]")
+      (util/input property-name)
+      (w/click (util/get-by-text "New option:" false))
+      (assert/assert-is-visible (w/get-by-text "Select a property type"))
+      (w/click (loc/and "span" (util/get-by-text property-type true)))
+      (case property-type
+        "Text" (do
+                 (w/click (format ".property-pair:has-text('%s') > .ls-block" property-name))
+                 (util/input "Text"))
+        "Number" (do (assert/assert-is-visible (format "input[placeholder='%s']" (str "Set " property-name)))
+                     (util/input "111")
+                     (w/click (w/get-by-text "New option:")))
+        ("DateTime" "Date") (do
+                              (assert/assert-is-visible ".ls-property-dialog")
+                              (k/enter)
+                              (k/esc))
+        "Checkbox" nil
+        "Url" nil
+        "Node" (do
+                 (w/click (w/get-by-text "Skip choosing tag"))
+                 (util/input (str title-prefix "-Node-value"))
+                 (w/click (w/get-by-text "New option:")))))))
+
+(deftest new-property-test
+  (let [title-prefix "new-property-test"]
+    (add-new-properties title-prefix)))

clj-e2e/test/logseq/e2e/rtc_extra_test.clj

@@ -1,20 +1,21 @@
 (ns logseq.e2e.rtc-extra-test
   (:require
-   [clojure.test :refer [deftest testing is use-fixtures run-test]]
+   [clojure.test :refer [deftest testing is use-fixtures run-test run-tests]]
    [com.climate.claypoole :as cp]
    [logseq.e2e.assert :as assert]
    [logseq.e2e.block :as b]
+   [logseq.e2e.custom-report :as custom-report]
    [logseq.e2e.fixtures :as fixtures :refer [*page1 *page2]]
    [logseq.e2e.graph :as graph]
    [logseq.e2e.keyboard :as k]
    [logseq.e2e.locator :as loc]
    [logseq.e2e.outliner-basic-test :as outliner-basic-test]
    [logseq.e2e.page :as page]
+   [logseq.e2e.property-basic-test :as property-basic-test]
    [logseq.e2e.rtc :as rtc]
    [logseq.e2e.settings :as settings]
    [logseq.e2e.util :as util]
-   [wally.main :as w]
-   [wally.repl :as repl]))
+   [wally.main :as w]))
 
 (defn- prepare-rtc-graph-fixture
   "open 2 app instances, add a rtc graph, check this graph available on other instance"
@@ -33,11 +34,13 @@
       (graph/wait-for-remote-graph graph-name)
       (graph/switch-graph graph-name true))
 
-    (f)
-
-    ;; cleanup
-    (w/with-page @*page2
-      (graph/remove-remote-graph graph-name))))
+    (binding [custom-report/*preserve-graph* false]
+      (f)
+      ;; cleanup
+      (if custom-report/*preserve-graph*
+        (println "Don't remove graph: " graph-name)
+        (w/with-page @*page2
+          (graph/remove-remote-graph graph-name))))))
 
 (defn- new-logseq-page
   "new logseq page and switch to this page on both page1 and page2"
@@ -81,9 +84,19 @@
          [@*page1 @*page2])]
     (assert/assert-graph-summary-equal p1-summary p2-summary)))
 
+(def status->icon-name
+  {"Backlog" "Backlog"
+   "Todo" "Todo"
+   "Doing" "InProgress50"
+   "In review" "InReview"
+   "Done" "Done"
+   "Canceled" "Cancelled"})
+
+(def priorities ["No priority" "Low" "Medium" "High" "Urgent"])
+
 (defn- validate-task-blocks
   []
-  (let [icon-names ["Backlog" "Todo" "InProgress50" "InReview" "Done" "Cancelled"]
+  (let [icon-names (vals status->icon-name)
         icon-name->count
         (w/with-page @*page2
           (into
@@ -99,14 +112,20 @@
 
 (defn- insert-task-blocks
   [title-prefix]
-  (doseq [status ["Backlog" "Todo" "Doing" "In review" "Done" "Canceled"]
-          priority ["No priority" "Low" "Medium" "High" "Urgent"]]
+  (doseq [status (keys status->icon-name)
+          priority priorities]
     (b/new-block (str title-prefix "-" status "-" priority))
     (util/input-command status)
     (util/input-command priority)))
 
 (defn- update-task-blocks
-  [])
+  []
+  (let [qs-partitions (partition-all 5 (seq (.all (loc/filter ".ls-block" :has ".ui__icon"))))]
+    (doseq [q-seq qs-partitions]
+      (doseq [q q-seq]
+        (w/click q)
+        (util/input-command (rand-nth (keys status->icon-name)))
+        (util/input-command (rand-nth priorities))))))
 
 (deftest rtc-task-blocks-test
   (let [insert-task-blocks-in-page2
@@ -116,13 +135,29 @@
                   (rtc/with-wait-tx-updated
                     (insert-task-blocks "t1"))]
               (reset! *latest-remote-tx remote-tx))
-            ;; TODO: more operations
-            (util/exit-edit)))]
-    (testing "rtc-stop app1, add some task blocks, then rtc-start on app1"
+            (util/exit-edit)))
+        update-task-blocks-in-page2
+        (fn [*latest-remote-tx]
+          (w/with-page @*page2
+            (let [{:keys [_local-tx remote-tx]}
+                  (rtc/with-wait-tx-updated
+                    (update-task-blocks))]
+              (reset! *latest-remote-tx remote-tx))))]
+    (testing "add some task blocks while rtc disconnected on page1"
       (let [*latest-remote-tx (atom nil)]
-        (with-stop-restart-rtc @*page1 #(insert-task-blocks-in-page2 *latest-remote-tx))
-        (w/with-page @*page1
-          (rtc/wait-tx-update-to @*latest-remote-tx))
+        (rtc/with-stop-restart-rtc
+          [@*page1]
+          [@*page1 (rtc/wait-tx-update-to @*latest-remote-tx)]
+          (insert-task-blocks-in-page2 *latest-remote-tx))
+        (validate-task-blocks)
+        (validate-2-graphs)))
+
+    (testing "update task blocks while rtc disconnected on page1"
+      (let [*latest-remote-tx (atom nil)]
+        (rtc/with-stop-restart-rtc
+          [@*page1]
+          [@*page1 (rtc/wait-tx-update-to @*latest-remote-tx)]
+          (update-task-blocks-in-page2 *latest-remote-tx))
         (validate-task-blocks)
         (validate-2-graphs)))
 
@@ -134,35 +169,15 @@
         (w/with-page @*page1
           (rtc/wait-tx-update-to @*latest-remote-tx))
         (validate-task-blocks)
-        (validate-2-graphs)))))
+        (validate-2-graphs)))
 
-(defn- add-new-properties
-  [title-prefix]
-  (b/new-blocks (map #(str title-prefix "-" %) ["Text" "Number" "Date" "DateTime" "Checkbox" "Url" "Node"]))
-  (doseq [property-type ["Text" "Number" "Date" "DateTime" "Checkbox" "Url" "Node"]]
-    (let [property-name (str "p-" title-prefix "-" property-type)]
-      (w/click (util/get-by-text (str title-prefix "-" property-type) true))
-      (k/press "Control+e")
-      (util/input-command "Add new property")
-      (util/input property-name)
-      (w/click (w/get-by-text "New option:"))
-      (assert/assert-is-visible (w/get-by-text "Select a property type"))
-      (w/click (loc/and "span" (util/get-by-text property-type true)))
-      (case property-type
-        "Text" (util/input "Text")
-        "Number" (do (assert/assert-is-visible (format "input[placeholder='%s']" (str "Set " property-name)))
-                     (util/input "111")
-                     (w/click (w/get-by-text "New option:")))
-        ("DateTime" "Date") (do
-                              (assert/assert-is-visible ".ls-property-dialog")
-                              (k/enter)
-                              (k/esc))
-        "Checkbox" nil
-        "Url" nil
-        "Node" (do
-                 (w/click (w/get-by-text "Skip choosing tag"))
-                 (util/input (str title-prefix "-Node-value"))
-                 (w/click (w/get-by-text "New option:")))))))
+    (testing "update task blocks while rtc connected on page1"
+      (let [*latest-remote-tx (atom nil)]
+        (update-task-blocks-in-page2 *latest-remote-tx)
+        (w/with-page @*page1
+          (rtc/wait-tx-update-to @*latest-remote-tx))
+        (validate-task-blocks)
+        (validate-2-graphs)))))
 
 (deftest rtc-property-test
   (let [insert-new-property-blocks-in-page2
@@ -170,15 +185,14 @@
           (w/with-page @*page2
             (let [{:keys [_local-tx remote-tx]}
                   (rtc/with-wait-tx-updated
-                    (add-new-properties title-prefix))]
+                    (property-basic-test/add-new-properties title-prefix))]
               (reset! *latest-remote-tx remote-tx))))]
-    (testing "page1: rtc-stop
-page2: create some user properties with different type
-page1: rtc-start"
+    (testing "add different types user properties on page2 while keeping rtc connected on page1"
       (let [*latest-remote-tx (atom nil)]
-        (with-stop-restart-rtc @*page1 #(insert-new-property-blocks-in-page2 *latest-remote-tx "rtc-property-test-1"))
-        (w/with-page @*page1
-          (rtc/wait-tx-update-to @*latest-remote-tx))
+        (rtc/with-stop-restart-rtc
+          [@*page1]
+          [@*page1 (rtc/wait-tx-update-to @*latest-remote-tx)]
+          (insert-new-property-blocks-in-page2 *latest-remote-tx "rtc-property-test-1"))
         (validate-2-graphs)))
 
     (new-logseq-page)
@@ -201,26 +215,74 @@ page1: rtc-start"
                                (let [{:keys [_local-tx remote-tx]}
                                      (rtc/with-wait-tx-updated
                                        (test-fn))]
-                                 (reset! *latest-remote-tx remote-tx))))]
+                                 (reset! *latest-remote-tx remote-tx))))
+          *latest-remote-tx (atom nil)]
+      (new-logseq-page)
+      (rtc/with-stop-restart-rtc
+        [@*page1]
+        [@*page1 (rtc/wait-tx-update-to @*latest-remote-tx)]
+        (test-fn-in-page2 *latest-remote-tx))
+      (validate-2-graphs))))
 
-      ;; testing while rtc connected
+(deftest rtc-outliner-conflict-update-test
+  (let [title-prefix "rtc-outliner-conflict-update-test"]
+    (testing "add some blocks, ensure them synced"
       (let [*latest-remote-tx (atom nil)]
-        (new-logseq-page)
-        (test-fn-in-page2 *latest-remote-tx)
         (w/with-page @*page1
+          (let [{:keys [_local-tx remote-tx]}
+                (rtc/with-wait-tx-updated
+                  (b/new-blocks (map #(str title-prefix "-" %) (range 10))))]
+            (reset! *latest-remote-tx remote-tx)))
+        (w/with-page @*page2
           (rtc/wait-tx-update-to @*latest-remote-tx))
-        (validate-2-graphs))
-
-      ;; testing while rtc off then on
-      (let [*latest-remote-tx (atom nil)]
-        (new-logseq-page)
-        (with-stop-restart-rtc @*page1 #(test-fn-in-page2 *latest-remote-tx))
+        (validate-2-graphs)))
+    (testing "page1: indent block1 as child of block0, page2: delete block0"
+      (rtc/with-stop-restart-rtc
+        [@*page1 @*page2]
+        [@*page1 (rtc/with-wait-tx-updated
+                   (k/esc)
+                   (assert/assert-in-normal-mode?)
+                   (b/new-block "page1-done-1"))
+         @*page2 (rtc/with-wait-tx-updated
+                   (k/esc)
+                   (assert/assert-in-normal-mode?)
+                   (b/new-block "page2-done-1"))]
         (w/with-page @*page1
-          (rtc/wait-tx-update-to @*latest-remote-tx))
-        (validate-2-graphs)))))
-
-(comment
-  (let [title-prefix "xxxx"
-        property-type "Text"]
-    (w/with-page @*page1
-      (b/new-block (str title-prefix "-" property-type)))))
+          (w/click (format ".ls-block :text('%s')" (str title-prefix "-" 1)))
+          (b/indent))
+        (w/with-page @*page2
+          (w/click (format ".ls-block :text('%s')" (str title-prefix "-" 0)))
+          (b/delete-blocks)))
+      (validate-2-graphs))
+    (testing "
+origin:
+- block2
+- block3
+- block4
+page1:
+- block2
+  - block3
+    - block4
+page2:
+;; block2 deleted
+- block4
+  - block3"
+      (rtc/with-stop-restart-rtc
+        [@*page1 @*page2]
+        [@*page1 (rtc/with-wait-tx-updated (b/new-block "page1-done-2"))
+         @*page2 (rtc/with-wait-tx-updated (b/new-block "page2-done-2"))]
+        (w/with-page @*page1
+          (w/click (format ".ls-block :text('%s')" (str title-prefix "-" 3)))
+          (b/indent)
+          (k/arrow-down)
+          (b/indent)
+          (b/indent))
+        (w/with-page @*page2
+          (w/click (format ".ls-block :text('%s')" (str title-prefix "-" 2)))
+          (b/delete-blocks)
+          (w/click (format ".ls-block :text('%s')" (str title-prefix "-" 3)))
+          (k/shift+arrow-down)
+          (k/meta+shift+arrow-down)
+          (k/enter)
+          (b/indent)))
+      (validate-2-graphs))))

deps.edn

@@ -5,7 +5,7 @@
                                          :sha     "5d672bf84ed944414b9f61eeb83808ead7be9127"}
 
   datascript/datascript                 {:git/url "https://github.com/logseq/datascript" ;; fork
-                                         :sha     "4b1f15f05a6b4a718a62c247956206480e361ea6"}
+                                         :sha     "b28f6574b9447bba9ccaa5d2b0cfd79308acf0e3"}
 
   datascript-transit/datascript-transit {:mvn/version "0.3.0"}
   borkdude/rewrite-edn                  {:mvn/version "0.4.9"}
@@ -25,7 +25,7 @@
   hiccups/hiccups                       {:mvn/version "0.3.0"}
   tongue/tongue                         {:mvn/version "0.4.4"}
   org.clojure/core.async                {:mvn/version "1.6.673"}
-  thheller/shadow-cljs                  {:mvn/version "2.19.0"}
+  thheller/shadow-cljs                  {:mvn/version "2.28.23"}
   expound/expound                       {:mvn/version "0.8.6"}
   com.lambdaisland/glogi                {:mvn/version "1.1.144"}
   binaryage/devtools                    {:mvn/version "1.0.5"}

deps/common/nbb.edn

@@ -1,4 +1,4 @@
 {:paths ["src" "resources"]
  :deps
  {io.github.nextjournal/nbb-test-runner
-  {:git/sha "012017a8a8983d05f905f38f631f5222f25b9ed9"}}}
+  {:git/sha "b379325cfa5a3306180649da5de3bf5166414e71"}}}

deps/common/package.json

@@ -3,7 +3,7 @@
   "version": "1.0.0",
   "private": true,
   "devDependencies": {
-    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v23"
+    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v24"
   },
   "scripts": {
     "test": "yarn nbb-logseq -cp test -m nextjournal.test-runner"

deps/common/resources/templates/config.edn

@@ -327,10 +327,6 @@
  ;; Default value: 2
  :ref/default-open-blocks-level 2
 
- ;; Configure the threshold for linked references before collapsing.
- ;; Default value: 100
- :ref/linked-references-collapsed-threshold 50
-
  ;; Graph view configuration.
  ;; Example usage:
  ;; :graph/settings
@@ -408,4 +404,4 @@
  ;;  :page-ref?        true        ;; Default value: true
  ;;  :properties?      true        ;; Default value: true
  ;;  :list?            false}      ;; Default value: false
- }
+ }

deps/common/src/logseq/common/util.cljs

@@ -292,13 +292,13 @@
       (:block/name page)))
 
 (defn string-join-path
-  #_:clj-kondo/ignore
   "Replace all `strings/join` used to construct paths with this function to reduce lint output.
   https://github.com/logseq/logseq/pull/8679"
   [parts]
+  #_:clj-kondo/ignore
   (string/join "/" parts))
 
-(def escape-chars "[]{}().+*?|$")
+(def ^:private escape-chars "\\[]{}().+*?|$^")
 
 (defn escape-regex-chars
   "Escapes characters in string `old-value"

deps/common/src/logseq/common/uuid.cljs

@@ -40,7 +40,8 @@ the remaining chars for data of this type"
   - :journal-page-uuid, 00000001
   - :db-ident-block-uuid, 00000002
   - :migrate-new-block-uuid, 00000003
-  - :builtin-block-uuid, 00000004"
+  - :builtin-block-uuid, 00000004
+  - :view-block-uuid, 00000006"
   ([] (d/squuid))
   ([type' v]
    (assert (some? v))
@@ -48,7 +49,8 @@ the remaining chars for data of this type"
      :journal-page-uuid (gen-journal-page-uuid v)
      :db-ident-block-uuid (gen-db-ident-block-uuid v)
      :migrate-new-block-uuid (gen-block-uuid v "00000003")
-     :builtin-block-uuid (gen-block-uuid v "00000004"))))
+     :builtin-block-uuid (gen-block-uuid v "00000004")
+     :view-block-uuid (gen-block-uuid v "00000006"))))
 
 (defn gen-journal-template-block
   "Persistent uuid for journal template block"

deps/common/test/logseq/common/util_test.cljs

@@ -1,39 +1,47 @@
 (ns logseq.common.util-test
-  (:require [clojure.test :refer [deftest are]]
+  (:require [clojure.test :refer [deftest are testing]]
             [logseq.common.util :as common-util]))
 
 (deftest valid-edn-keyword?
   (are [x y]
        (= (common-util/valid-edn-keyword? x) y)
 
-       ":foo-bar"  true
-       ":foo!"     true
-       ":foo,bar"  false
-       "4"         false
-       "foo bar"   false
-       "`property" false))
+    ":foo-bar"  true
+    ":foo!"     true
+    ":foo,bar"  false
+    "4"         false
+    "foo bar"   false
+    "`property" false))
 
 (deftest extract-file-extension?
   (are [x y]
        (= (common-util/path->file-ext x) y)
-       "foo.bar" "bar"
-       "foo"     nil
-       "foo.bar.baz" "baz"
-       "../assets/audio.mp3" "mp3"
+    "foo.bar" "bar"
+    "foo"     nil
+    "foo.bar.baz" "baz"
+    "../assets/audio.mp3" "mp3"
        ;; From https://www.w3.org/TR/media-frags/
-       "../assets/audio.mp3?t=10,20" "mp3"
-       "../assets/audio.mp3?t=10,20#t=10" "mp3"
-       "/root/Documents/audio.mp3" "mp3"
-       "C:\\Users\\foo\\Documents\\audio.mp3" "mp3"
-       "/root/Documents/audio" nil
-       "/root/Documents/audio." nil
-       "special/characters/aäääöüß.7z" "7z"
-       "asldk lakls .lsad" "lsad"
-       "中文asldk lakls .lsad" "lsad"))
+    "../assets/audio.mp3?t=10,20" "mp3"
+    "../assets/audio.mp3?t=10,20#t=10" "mp3"
+    "/root/Documents/audio.mp3" "mp3"
+    "C:\\Users\\foo\\Documents\\audio.mp3" "mp3"
+    "/root/Documents/audio" nil
+    "/root/Documents/audio." nil
+    "special/characters/aäääöüß.7z" "7z"
+    "asldk lakls .lsad" "lsad"
+    "中文asldk lakls .lsad" "lsad"))
 
 (deftest url?
   (are [x y]
        (= (common-util/url? x) y)
-       "http://logseq.com" true
-       "prop:: value" false
-       "a:" false))
+    "http://logseq.com" true
+    "prop:: value" false
+    "a:" false))
+
+(deftest escape-regex-chars
+  (testing "ensure the result is a valid regex string"
+    (are [x]
+         (some? (re-find (re-pattern  (common-util/escape-regex-chars x)) x))
+      "[[page-name]]"
+      "end-with-backslash\\"
+      "\\[]{}().+*?|$^")))

deps/common/yarn.lock

@@ -2,9 +2,9 @@
 # yarn lockfile v1
 
 
-"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v23":
-  version "1.2.173-feat-db-v23"
-  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/810782c4ddba6346c4ab8ae6740b60438c07cd01"
+"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v24":
+  version "1.2.173-feat-db-v24"
+  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/3d5b78b0382c7253bf9874c1f38586dd338434f4"
   dependencies:
     import-meta-resolve "^4.1.0"
 

deps/db/.carve/ignore

@@ -33,4 +33,14 @@ logseq.db.common.initial-data/with-parent
 ;; API
 logseq.db.common.initial-data/get-block-and-children
 ;; API
-logseq.db.common.initial-data/get-initial-data
+logseq.db.common.initial-data/get-initial-data
+;; API
+logseq.db.sqlite.debug/find-missing-addresses
+;; API
+logseq.db.sqlite.debug/find-missing-addresses-node-version
+;; API
+logseq.db.sqlite.gc/gc-kvs-table!
+;; API
+logseq.db.sqlite.gc/gc-kvs-table-node-version!
+;; API
+logseq.db.sqlite.gc/ensure-no-garbage

deps/db/deps.edn

@@ -1,7 +1,7 @@
 {:deps
  ;; These nbb-logseq deps are kept in sync with https://github.com/logseq/nbb-logseq/blob/main/bb.edn
  {datascript/datascript {:git/url "https://github.com/logseq/datascript" ;; fork
-                         :sha     "4b1f15f05a6b4a718a62c247956206480e361ea6"}
+                         :sha     "b28f6574b9447bba9ccaa5d2b0cfd79308acf0e3"}
   datascript-transit/datascript-transit {:mvn/version "0.3.0"
                                          :exclusions [datascript/datascript]}
   cljs-bean/cljs-bean         {:mvn/version "1.5.0"}
@@ -11,7 +11,7 @@
   ;; Any other deps should be added here and to nbb.edn
   logseq/common                {:local/root "../common"}
   logseq/clj-fractional-indexing        {:git/url "https://github.com/logseq/clj-fractional-indexing"
-                                         :sha     "7182b7878410f78536dc2b6df35ed32ef9cd6b61"}
+                                         :sha     "1087f0fb18aa8e25ee3bbbb0db983b7a29bce270"}
   borkdude/rewrite-edn {:mvn/version "0.4.9"}
   metosin/malli {:mvn/version "0.16.1"}
   medley/medley {:mvn/version "1.4.0"}}

deps/db/nbb.edn

@@ -7,7 +7,7 @@
   ;; Used by db scripts with outliner.cli
   borkdude/rewrite-edn {:mvn/version "0.4.9"}
   logseq/clj-fractional-indexing        {:git/url "https://github.com/logseq/clj-fractional-indexing"
-                                         :sha     "7182b7878410f78536dc2b6df35ed32ef9cd6b61"}
+                                         :sha     "1087f0fb18aa8e25ee3bbbb0db983b7a29bce270"}
   io.github.nextjournal/nbb-test-runner
-  {:git/sha "012017a8a8983d05f905f38f631f5222f25b9ed9"}
+  {:git/sha "b379325cfa5a3306180649da5de3bf5166414e71"}
   io.github.pez/baldr {:mvn/version "1.0.9"}}}

deps/db/package.json

@@ -3,7 +3,7 @@
   "version": "1.0.0",
   "private": true,
   "devDependencies": {
-    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v23"
+    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v24"
   },
   "dependencies": {
     "better-sqlite3": "11.10.0"

deps/db/script/create_graph.cljs

@@ -2,12 +2,11 @@
   "A script that creates or updates a DB graph given a sqlite.build EDN file.
    If the given graph already exists, the EDN file updates the graph."
   (:require ["fs" :as fs]
-            ["os" :as os]
             ["path" :as node-path]
             [babashka.cli :as cli]
             [clojure.edn :as edn]
-            [clojure.string :as string]
             [datascript.core :as d]
+            [logseq.db.common.sqlite-cli :as sqlite-cli]
             [logseq.db.sqlite.export :as sqlite-export]
             [logseq.outliner.cli :as outliner-cli]
             [nbb.classpath :as cp]
@@ -21,17 +20,6 @@
     path
     (node-path/join (or js/process.env.ORIGINAL_PWD ".") path)))
 
-(defn- get-dir-and-db-name
-  "Gets dir and db name for use with open-db! Works for relative and absolute paths and
-   defaults to ~/logseq/graphs/ when no '/' present in name"
-  [graph-dir]
-  (if (string/includes? graph-dir "/")
-    (let [resolve-path' #(if (node-path/isAbsolute %) %
-                             ;; $ORIGINAL_PWD used by bb tasks to correct current dir
-                             (node-path/join (or js/process.env.ORIGINAL_PWD ".") %))]
-      ((juxt node-path/dirname node-path/basename) (resolve-path' graph-dir)))
-    [(node-path/join (os/homedir) "logseq" "graphs") graph-dir]))
-
 (def spec
   "Options spec"
   {:help {:alias :h
@@ -48,11 +36,13 @@
             (println (str "Usage: $0 GRAPH-NAME EDN-PATH [OPTIONS]\nOptions:\n"
                           (cli/format-opts {:spec spec})))
             (js/process.exit 1))
-        [dir db-name] (get-dir-and-db-name graph-dir)
+        init-conn-args (conj (sqlite-cli/->open-db-args graph-dir))
         sqlite-build-edn (merge (if (:import options) {} {:auto-create-ontology? true})
                                 (-> (resolve-path edn-path) fs/readFileSync str edn/read-string))
-        graph-exists? (fs/existsSync (node-path/join dir db-name))
-        conn (outliner-cli/init-conn dir db-name {:classpath (cp/get-classpath) :import-type :cli/create-graph})
+        graph-exists? (fs/existsSync (apply node-path/join init-conn-args))
+        db-name (if (= 1 (count init-conn-args)) (first init-conn-args) (second init-conn-args))
+        conn (apply outliner-cli/init-conn
+                    (conj init-conn-args {:classpath (cp/get-classpath) :import-type :cli/create-graph}))
         {:keys [init-tx block-props-tx misc-tx] :as _txs}
         (if (:import options)
           (sqlite-export/build-import sqlite-build-edn @conn {})

deps/db/script/diff_graphs.cljs

@@ -1,27 +1,13 @@
 (ns diff-graphs
   "A script that diffs two DB graphs through their sqlite.build EDN"
-  (:require ["os" :as os]
-            ["path" :as node-path]
-            [babashka.cli :as cli]
+  (:require [babashka.cli :as cli]
             [clojure.data :as data]
             [clojure.pprint :as pprint]
-            [clojure.string :as string]
             [logseq.common.config :as common-config]
             [logseq.db.common.sqlite-cli :as sqlite-cli]
             [logseq.db.sqlite.export :as sqlite-export]
             [nbb.core :as nbb]))
 
-(defn- get-dir-and-db-name
-  "Gets dir and db name for use with open-db! Works for relative and absolute paths and
-   defaults to ~/logseq/graphs/ when no '/' present in name"
-  [graph-dir]
-  (if (string/includes? graph-dir "/")
-    (let [resolve-path' #(if (node-path/isAbsolute %) %
-                             ;; $ORIGINAL_PWD used by bb tasks to correct current dir
-                             (node-path/join (or js/process.env.ORIGINAL_PWD ".") %))]
-      ((juxt node-path/dirname node-path/basename) (resolve-path' graph-dir)))
-    [(node-path/join (os/homedir) "logseq" "graphs") graph-dir]))
-
 (def spec
   "Options spec"
   {:help {:alias :h
@@ -43,8 +29,8 @@
             (println (str "Usage: $0 GRAPH-NAME GRAPH-NAME2 [& ARGS] [OPTIONS]\nOptions:\n"
                           (cli/format-opts {:spec spec})))
             (js/process.exit 1))
-        conn (apply sqlite-cli/open-db! (get-dir-and-db-name graph-dir))
-        conn2 (apply sqlite-cli/open-db! (get-dir-and-db-name graph-dir2))
+        conn (apply sqlite-cli/open-db! (sqlite-cli/->open-db-args graph-dir))
+        conn2 (apply sqlite-cli/open-db! (sqlite-cli/->open-db-args graph-dir2))
         export-options (select-keys options [:include-timestamps? :exclude-namespaces :exclude-built-in-pages?])
         export-map (sqlite-export/build-export @conn {:export-type :graph :graph-options export-options})
         export-map2 (sqlite-export/build-export @conn2 {:export-type :graph :graph-options export-options})

deps/db/script/dump_datoms.cljs

@@ -3,28 +3,21 @@
 
      $ yarn -s nbb-logseq script/dump_datoms.cljs db-name datoms.edn"
     (:require ["fs" :as fs]
-              ["os" :as os]
-              ["path" :as path]
+              ["path" :as node-path]
               [clojure.pprint :as pprint]
               [datascript.core :as d]
               [logseq.db.common.sqlite-cli :as sqlite-cli]
               [nbb.core :as nbb]))
 
-(defn read-graph
-  "The db graph bare version of gp-cli/parse-graph"
-  [graph-name]
-  (let [graphs-dir (path/join (os/homedir) "logseq/graphs")]
-    (sqlite-cli/open-db! graphs-dir graph-name)))
-
 (defn -main [args]
   (when (< (count args) 2)
     (println "Usage: $0 GRAPH FILE")
     (js/process.exit 1))
   (let [[graph-name file*] args
-        conn (read-graph graph-name)
+        conn (apply sqlite-cli/open-db! (sqlite-cli/->open-db-args graph-name))
         datoms (mapv #(vec %) (d/datoms @conn :eavt))
         parent-dir (or js/process.env.ORIGINAL_PWD ".")
-        file (path/join parent-dir file*)]
+        file (node-path/join parent-dir file*)]
     (println "Writing" (count datoms) "datoms to" file)
     (fs/writeFileSync file (with-out-str (pprint/pprint datoms)))))
 

deps/db/script/export_graph.cljs

@@ -1,12 +1,9 @@
 (ns export-graph
   "A script that exports a graph to a sqlite.build EDN file"
   (:require ["fs" :as fs]
-            ["os" :as os]
             ["path" :as node-path]
             [babashka.cli :as cli]
-            [clojure.edn :as edn]
             [clojure.pprint :as pprint]
-            [clojure.string :as string]
             [logseq.db.common.sqlite-cli :as sqlite-cli]
             [logseq.db.sqlite.export :as sqlite-export]
             [nbb.core :as nbb]))
@@ -18,17 +15,6 @@
     path
     (node-path/join (or js/process.env.ORIGINAL_PWD ".") path)))
 
-(defn- get-dir-and-db-name
-  "Gets dir and db name for use with open-db! Works for relative and absolute paths and
-   defaults to ~/logseq/graphs/ when no '/' present in name"
-  [graph-dir]
-  (if (string/includes? graph-dir "/")
-    (let [resolve-path' #(if (node-path/isAbsolute %) %
-                             ;; $ORIGINAL_PWD used by bb tasks to correct current dir
-                             (node-path/join (or js/process.env.ORIGINAL_PWD ".") %))]
-      ((juxt node-path/dirname node-path/basename) (resolve-path' graph-dir)))
-    [(node-path/join (os/homedir) "logseq" "graphs") graph-dir]))
-
 (def spec
   "Options spec"
   {:help {:alias :h
@@ -54,8 +40,7 @@
             (println (str "Usage: $0 GRAPH-NAME [& ARGS] [OPTIONS]\nOptions:\n"
                           (cli/format-opts {:spec spec})))
             (js/process.exit 1))
-        [dir db-name] (get-dir-and-db-name graph-dir)
-        conn (sqlite-cli/open-db! dir db-name)
+        conn (apply sqlite-cli/open-db! (sqlite-cli/->open-db-args graph-dir))
         export-options (dissoc options :file)
         export-map (sqlite-export/build-export @conn {:export-type :graph :graph-options export-options})]
     (if (:file options)

deps/db/script/query.cljs

@@ -3,15 +3,12 @@
 
   $ yarn -s nbb-logseq script/query.cljs db-name '[:find (pull ?b [:block/name :block/title]) :where [?b :block/created-at]]'"
   (:require ["child_process" :as child-process]
-            ["os" :as os]
-            ["path" :as node-path]
             [babashka.cli :as cli]
             [clojure.edn :as edn]
             [clojure.pprint :as pprint]
-            [clojure.string :as string]
             [datascript.core :as d]
-            [logseq.db.frontend.rules :as rules]
             [logseq.db.common.sqlite-cli :as sqlite-cli]
+            [logseq.db.frontend.rules :as rules]
             [nbb.core :as nbb]))
 
 (defn- sh
@@ -22,17 +19,6 @@
                            (clj->js (rest cmd))
                            (clj->js (merge {:stdio "inherit"} opts))))
 
-(defn- get-dir-and-db-name
-  "Gets dir and db name for use with open-db! Works for relative and absolute paths and
-   defaults to ~/logseq/graphs/ when no '/' present in name"
-  [graph-dir]
-  (if (string/includes? graph-dir "/")
-    (let [resolve-path' #(if (node-path/isAbsolute %) %
-                             ;; $ORIGINAL_PWD used by bb tasks to correct current dir
-                             (node-path/join (or js/process.env.ORIGINAL_PWD ".") %))]
-      ((juxt node-path/dirname node-path/basename) (resolve-path' graph-dir)))
-    [(node-path/join (os/homedir) "logseq" "graphs") graph-dir]))
-
 (def spec
   "Options spec"
   {:help {:alias :h
@@ -48,9 +34,8 @@
             :coerce []
             :desc "Lookup entities instead of query"}})
 
-(defn query-graph [graph-dir args'' options]
-  (let [[dir db-name] (get-dir-and-db-name graph-dir)
-        conn (sqlite-cli/open-db! dir db-name)
+(defn query-graph [graph args'' options]
+  (let [conn (apply sqlite-cli/open-db! (sqlite-cli/->open-db-args graph))
         results (if (:entity options)
                   (map #(when-let [ent (d/entity @conn
                                                  (if (string? %) (edn/read-string %) %))]

deps/db/script/validate_db.cljs

@@ -1,15 +1,12 @@
 (ns validate-db
   "Script that validates the datascript db of a DB graph
    NOTE: This script is also used in CI to confirm our db's schema is up to date"
-  (:require ["os" :as os]
-            ["path" :as node-path]
-            [babashka.cli :as cli]
+  (:require [babashka.cli :as cli]
             [cljs.pprint :as pprint]
-            [clojure.string :as string]
             [datascript.core :as d]
+            [logseq.db.common.sqlite-cli :as sqlite-cli]
             [logseq.db.frontend.malli-schema :as db-malli-schema]
             [logseq.db.frontend.validate :as db-validate]
-            [logseq.db.common.sqlite-cli :as sqlite-cli]
             [malli.core :as m]
             [malli.error :as me]
             [nbb.core :as nbb]))
@@ -50,17 +47,6 @@
         (js/process.exit 1))
       (println "Valid!"))))
 
-(defn- get-dir-and-db-name
-  "Gets dir and db name for use with open-db! Works for relative and absolute paths and
-   defaults to ~/logseq/graphs/ when no '/' present in name"
-  [graph-dir]
-  (if (string/includes? graph-dir "/")
-    (let [resolve-path' #(if (node-path/isAbsolute %) %
-                             ;; $ORIGINAL_PWD used by bb tasks to correct current dir
-                             (node-path/join (or js/process.env.ORIGINAL_PWD ".") %))]
-      ((juxt node-path/dirname node-path/basename) (resolve-path' graph-dir)))
-    [(node-path/join (os/homedir) "logseq" "graphs") graph-dir]))
-
 (def spec
   "Options spec"
   {:help {:alias :h
@@ -86,8 +72,9 @@
     (validate-db* db ent-maps options)))
 
 (defn- validate-graph [graph-dir options]
-  (let [[dir db-name] (get-dir-and-db-name graph-dir)
-        conn (try (sqlite-cli/open-db! dir db-name)
+  (let [open-db-args (sqlite-cli/->open-db-args graph-dir)
+        db-name (if (= 1 (count open-db-args)) (first open-db-args) (second open-db-args))
+        conn (try (apply sqlite-cli/open-db! open-db-args)
                   (catch :default e
                     (println "Error: For graph" (str (pr-str graph-dir) ":") (str e))
                     (js/process.exit 1)))]

deps/db/src/logseq/db.cljs

@@ -18,7 +18,6 @@
             [logseq.db.frontend.db :as db-db]
             [logseq.db.frontend.entity-util :as entity-util]
             [logseq.db.frontend.property :as db-property]
-            [logseq.db.frontend.rules :as rules]
             [logseq.db.frontend.schema :as db-schema]
             [logseq.db.sqlite.util :as sqlite-util])
   (:refer-clojure :exclude [object?]))
@@ -428,18 +427,7 @@
       ;; only return the first result for idiot-proof
     (first (:block/_alias (d/entity db alias-id)))))
 
-(defn get-block-alias
-  [db eid]
-  (->>
-   (d/q
-    '[:find [?e ...]
-      :in $ ?eid %
-      :where
-      (alias ?eid ?e)]
-    db
-    eid
-    (:alias rules/rules))
-   distinct))
+(def get-block-alias common-initial-data/get-block-alias)
 
 (defn page-alias-set
   [db page-id]

deps/db/src/logseq/db/common/entity_plus.cljc

@@ -22,7 +22,7 @@
     :block/pre-block? :block/scheduled :block/deadline :block/type :block/name :block/marker
 
     :block.temp/ast-title
-    :block.temp/fully-loaded? :block.temp/ast-body
+    :block.temp/fully-loaded? :block.temp/has-children? :block.temp/ast-body
 
     :db/valueType :db/cardinality :db/ident :db/index
 
@@ -63,7 +63,7 @@
 
 (defn entity-memoized
   [db eid]
-  (if (qualified-keyword? eid)
+  (if (and (qualified-keyword? eid) (not (exists? js/process))) ; don't memoize on node
     (when-not (contains? nil-db-ident-entities eid) ;fast return nil
       (if (and @*reset-cache-background-task-running?
                (contains? immutable-db-ident-entities eid)) ;return cache entity if possible which isn't nil

deps/db/src/logseq/db/common/initial_data.cljs

@@ -9,7 +9,9 @@
             [logseq.db.common.entity-plus :as entity-plus]
             [logseq.db.common.entity-util :as common-entity-util]
             [logseq.db.common.order :as db-order]
-            [logseq.db.frontend.entity-util :as entity-util]))
+            [logseq.db.frontend.class :as db-class]
+            [logseq.db.frontend.entity-util :as entity-util]
+            [logseq.db.frontend.rules :as rules]))
 
 (defn- get-pages-by-name
   [db page-name]
@@ -33,6 +35,19 @@
        sort
        first))
 
+(defn get-block-alias
+  [db eid]
+  (->>
+   (d/q
+    '[:find [?e ...]
+      :in $ ?eid %
+      :where
+      (alias ?eid ?e)]
+    db
+    eid
+    (:alias rules/rules))
+   distinct))
+
 (comment
   (defn- get-built-in-files
     [db]
@@ -139,8 +154,7 @@
   [db block-uuid]
   (let [ids (get-block-children-ids db block-uuid)]
     (when (seq ids)
-      (let [ids' (map (fn [id] [:block/uuid id]) ids)]
-        (d/pull-many db '[*] ids')))))
+      (map (fn [id] (d/entity db [:block/uuid id])) ids))))
 
 (defn- with-raw-title
   [m entity]
@@ -163,9 +177,13 @@
         (or
          (= (:db/id ref-block) id)
          (= id (:db/id (:block/page ref-block)))
+         (= id (:db/id (:logseq.property/view-for ref-block)))
          (entity-util/hidden? (:block/page ref-block))
          (entity-util/hidden? ref-block)
-         (contains? (set (map :db/id (:block/tags ref-block))) (:db/id entity))
+         (and (entity-util/class? entity)
+              (let [children (db-class/get-structured-children db id)
+                    class-ids (set (conj children id))]
+                (some class-ids (map :db/id (:block/tags ref-block)))))
          (some? (get ref-block (:db/ident entity)))))
       (or
        (= (:db/id ref-block) id)
@@ -174,17 +192,30 @@
 (defn get-block-refs-count
   [db id]
   (or
-   (some->> (d/entity db id)
-            :block/_refs
-            (remove (fn [ref-block] (hidden-ref? db ref-block id)))
-            count)
+   (let [with-alias (->> (get-block-alias db id)
+                         (cons id)
+                         distinct)]
+     (some->> with-alias
+              (map #(d/entity db %))
+              (mapcat :block/_refs)
+              (remove (fn [ref-block] (hidden-ref? db ref-block id)))
+              count))
    0))
 
 (defn ^:large-vars/cleanup-todo get-block-and-children
-  [db id {:keys [children? children-only? nested-children? properties children-props]}]
-  (let [block (d/entity db (if (uuid? id)
-                             [:block/uuid id]
-                             id))
+  [db id-or-page-name {:keys [children? children-only? nested-children? properties children-props]}]
+  (let [block (let [eid (cond (uuid? id-or-page-name)
+                              [:block/uuid id-or-page-name]
+                              (integer? id-or-page-name)
+                              id-or-page-name
+                              :else nil)]
+                (cond
+                  eid
+                  (d/entity db eid)
+                  (string? id-or-page-name)
+                  (d/entity db (get-first-page-by-name db (name id-or-page-name)))
+                  :else
+                  nil))
         block-refs-count? (some #{:block.temp/refs-count} properties)
         whiteboard? (common-entity-util/whiteboard? block)]
     (when block
@@ -215,7 +246,8 @@
                             (if (= children-props '[*])
                               (entity->map block)
                               (-> (select-keys block children-props)
-                                  (with-raw-title block))))
+                                  (with-raw-title block)
+                                  (assoc :block.temp/has-children? (some? (:block/_parent block))))))
                           children)))]
         (if children-only?
           {:children children}

deps/db/src/logseq/db/common/reference.cljs

@@ -0,0 +1,146 @@
+(ns logseq.db.common.reference
+  "References"
+  (:require [cljs.reader :as reader]
+            [clojure.string :as string]
+            [datascript.core :as d]
+            [logseq.common.log :as log]
+            [logseq.db :as ldb]
+            [logseq.db.common.entity-plus :as entity-plus]
+            [logseq.db.common.initial-data :as common-initial-data]
+            [logseq.db.frontend.class :as db-class]
+            [logseq.db.frontend.rules :as rules]))
+
+(defn get-filters
+  [db page]
+  (let [db-based? (entity-plus/db-based-graph? db)]
+    (if db-based?
+      (let [included-pages (:logseq.property.linked-references/includes page)
+            excluded-pages (:logseq.property.linked-references/excludes page)]
+        (when (or (seq included-pages) (seq excluded-pages))
+          {:included included-pages
+           :excluded excluded-pages}))
+      (let [k :filters
+            properties (:block/properties page)
+            properties-str (or (get properties k) "{}")]
+        (try (let [result (reader/read-string properties-str)]
+               (when (seq result)
+                 (let [excluded-pages (->> (filter #(false? (second %)) result)
+                                           (keep first)
+                                           (keep #(ldb/get-page db %)))
+                       included-pages (->> (filter #(true? (second %)) result)
+                                           (keep first)
+                                           (keep #(ldb/get-page db %)))]
+                   {:included included-pages
+                    :excluded excluded-pages})))
+             (catch :default e
+               (log/error :syntax/filters e)))))))
+
+(defn- build-include-exclude-query
+  [variable includes excludes]
+  (concat
+   (for [include includes]
+     [variable :block/path-refs include])
+   (for [exclude excludes]
+     (list 'not [variable :block/path-refs exclude]))))
+
+(defn- filter-refs-children-query
+  [includes excludes class-ids]
+  (let [clauses (concat
+                 [;; find refs from refed block's children only
+                  '(block-parent ?b ?c)
+                  ;; find all levels of parents
+                  '(block-parent ?p ?c)]
+                 (build-include-exclude-query '?c includes excludes)
+                 (when class-ids
+                   (mapcat
+                    (fn [class-id]
+                      (map
+                       (fn [variable]
+                         (list 'not [variable :block/tags class-id]))
+                       ['?b '?p '?c]))
+                    class-ids)))]
+    (into [:find '?b '?p '?c
+           :in '$ '% '[?id ...]
+           :where
+           ['?b :block/refs '?id]]
+          clauses)))
+
+(defn- filter-refs-query
+  [includes excludes class-ids]
+  (let [clauses (concat
+                 (build-include-exclude-query '?b includes excludes)
+                 (for [class-id class-ids]
+                   (list 'not ['?b :block/tags class-id])))]
+    (into [:find '[?b ...]
+           :in '$ '% '[?id ...]
+           :where
+           ['?b :block/refs '?id]]
+          clauses)))
+
+(defn- remove-hidden-ref
+  [db page-id refs]
+  (remove (fn [block] (common-initial-data/hidden-ref? db block page-id)) refs))
+
+(defn- get-ref-pages-count
+  [db id ref-blocks children-ids]
+  (when (seq ref-blocks)
+    (let [children (->> children-ids
+                        (map (fn [id] (d/entity db id)))
+                        (remove-hidden-ref db id))]
+      (->> (concat (mapcat :block/path-refs ref-blocks)
+                   (mapcat :block/refs children))
+           frequencies
+           (keep (fn [[ref size]]
+                   (when (and (ldb/page? ref)
+                              (not= (:db/id ref) id)
+                              (not= :block/tags (:db/ident ref))
+                              (not (common-initial-data/hidden-ref? db ref id)))
+                     [(:block/title ref) size])))
+           (sort-by second #(> %1 %2))))))
+
+(defn get-linked-references
+  [db id]
+  (let [entity (d/entity db id)
+        ids (set (cons id (ldb/get-block-alias db id)))
+        page-filters (get-filters db entity)
+        excludes (map :db/id (:excluded page-filters))
+        includes (map :db/id (:included page-filters))
+        filter-exists? (or (seq excludes) (seq includes))
+        class-ids (when (ldb/class? entity)
+                    (let [class-children (db-class/get-structured-children db id)]
+                      (set (conj class-children id))))
+        rules (rules/extract-rules rules/db-query-dsl-rules [:block-parent] {})
+        ref-blocks-query-result (d/q (filter-refs-query includes excludes class-ids) db rules ids)
+        children-query-result (d/q (filter-refs-children-query includes excludes class-ids) db rules ids)
+        ref-blocks (->> (map first children-query-result)
+                        (concat ref-blocks-query-result)
+                        distinct
+                        (map (fn [id] (d/entity db id)))
+                        (remove-hidden-ref db id))
+        children-ids (->>
+                      (distinct (concat
+                                 (map second children-query-result)
+                                 (map last children-query-result)))
+                      (remove (set (map :db/id ref-blocks))))
+        ref-pages-count (get-ref-pages-count db id ref-blocks children-ids)]
+    {:ref-pages-count ref-pages-count
+     :ref-blocks ref-blocks
+     :ref-matched-children-ids (when filter-exists? (set children-ids))}))
+
+(defn get-unlinked-references
+  [db id]
+  (let [entity (d/entity db id)
+        title (string/lower-case (:block/title entity))]
+    (when-not (string/blank? title)
+      (let [ids (->> (d/datoms db :avet :block/title)
+                     (keep (fn [d]
+                             (when (and (not= id (:e d)) (string/includes? (string/lower-case (:v d)) title))
+                               (:e d)))))]
+        (keep
+         (fn [eid]
+           (let [e (d/entity db eid)]
+             (when-not (or (some #(= id %) (map :db/id (:block/refs e)))
+                           (:block/link e)
+                           (ldb/built-in? e))
+               e)))
+         ids)))))

deps/db/src/logseq/db/common/sqlite_cli.cljs

@@ -2,8 +2,10 @@
   "Primary ns to interact with DB files for DB and file graphs with node.js based CLIs"
   (:require ["better-sqlite3" :as sqlite3]
             ["fs" :as fs]
+            ["os" :as os]
             ["path" :as node-path]
             [cljs-bean.core :as bean]
+            [clojure.string :as string]
             ;; FIXME: datascript.core has to come before datascript.storage or else nbb fails
             [datascript.core]
             [datascript.storage :refer [IStorage]]
@@ -30,16 +32,12 @@
 
 (defn- upsert-addr-content!
   "Upsert addr+data-seq. Should be functionally equivalent to db-worker/upsert-addr-content!"
-  [db data delete-addrs]
+  [db data]
   (let [insert (.prepare db "INSERT INTO kvs (addr, content, addresses) values ($addr, $content, $addresses) on conflict(addr) do update set content = $content, addresses = $addresses")
-        delete (.prepare db "Delete from kvs WHERE addr = ? AND NOT EXISTS (SELECT 1 FROM json_each(addresses) WHERE value = ?);")
         insert-many (.transaction ^object db
                                   (fn [data]
                                     (doseq [item data]
-                                      (.run ^object insert item))
-                                    (doseq [addr delete-addrs]
-                                      (when addr
-                                        (.run ^object delete addr)))))]
+                                      (.run ^object insert item))))]
     (insert-many data)))
 
 (defn- restore-data-from-addr
@@ -59,16 +57,9 @@
   "Creates a datascript storage for sqlite. Should be functionally equivalent to db-worker/new-sqlite-storage"
   [db]
   (reify IStorage
-    (-store [_ addr+data-seq delete-addrs]
-            ;; Only difference from db-worker impl is that js data maps don't start with '$' e.g. :$addr -> :addr
-      (let [used-addrs (set (mapcat
-                             (fn [[addr data]]
-                               (cons addr
-                                     (when (map? data)
-                                       (:addresses data))))
-                             addr+data-seq))
-            delete-addrs (remove used-addrs delete-addrs)
-            data (map
+    (-store [_ addr+data-seq _delete-addrs]
+      ;; Only difference from db-worker impl is that js data maps don't start with '$' e.g. :$addr -> :addr
+      (let [data (map
                   (fn [[addr data]]
                     (let [data' (if (map? data) (dissoc data :addresses) data)
                           addresses (when (map? data)
@@ -78,22 +69,49 @@
                            :content (sqlite-util/transit-write data')
                            :addresses addresses}))
                   addr+data-seq)]
-        (upsert-addr-content! db data delete-addrs)))
+        (upsert-addr-content! db data)))
     (-restore [_ addr]
       (restore-data-from-addr db addr))))
 
+(defn open-sqlite-datascript!
+  "Returns a map including `conn` for datascript connection and `sqlite` for sqlite connection"
+  ([db-full-path]
+   (open-sqlite-datascript! nil db-full-path))
+  ([graphs-dir db-name]
+   (let [[base-name db-full-path]
+         (if (nil? graphs-dir)
+           [(node-path/basename db-name) db-name]
+           [db-name (second (common-sqlite/get-db-full-path graphs-dir db-name))])
+         db (new sqlite db-full-path nil)
+        ;; For both desktop and CLI, only file graphs have db-name that indicate their db type
+         schema (if (common-sqlite/local-file-based-graph? base-name)
+                  file-schema/schema
+                  db-schema/schema)]
+     (common-sqlite/create-kvs-table! db)
+     (let [storage (new-sqlite-storage db)
+           conn (common-sqlite/get-storage-conn storage schema)]
+       {:sqlite db
+        :conn conn}))))
+
 (defn open-db!
   "For a given database name, opens a sqlite db connection for it, creates
   needed sqlite tables if not created and returns a datascript connection that's
   connected to the sqlite db"
-  [graphs-dir db-name]
-  (let [[_db-sanitized-name db-full-path] (common-sqlite/get-db-full-path graphs-dir db-name)
-        db (new sqlite db-full-path nil)
-        ;; For both desktop and CLI, only file graphs have db-name that indicate their db type
-        schema (if (common-sqlite/local-file-based-graph? db-name)
-                 file-schema/schema
-                 db-schema/schema)]
-    (common-sqlite/create-kvs-table! db)
-    (let [storage (new-sqlite-storage db)
-          conn (common-sqlite/get-storage-conn storage schema)]
-      conn)))
+  ([db-full-path]
+   (open-db! nil db-full-path))
+  ([graphs-dir db-name]
+   (:conn (open-sqlite-datascript! graphs-dir db-name))))
+
+(defn ->open-db-args
+  "Creates args for open-db from a graph arg. Works for relative and absolute paths and
+   defaults to ~/logseq/graphs/ when no '/' present in name"
+  [graph-dir-or-path]
+  ;; Pass full path directly to allow for paths that don't have standard graph naming convention
+  (if (node-path/isAbsolute graph-dir-or-path)
+    [graph-dir-or-path]
+    (if (string/includes? graph-dir-or-path "/")
+      (let [resolve-path' #(if (node-path/isAbsolute %) %
+                             ;; $ORIGINAL_PWD used by bb tasks to correct current dir
+                               (node-path/join (or js/process.env.ORIGINAL_PWD ".") %))]
+        ((juxt node-path/dirname node-path/basename) (resolve-path' graph-dir-or-path)))
+      [(node-path/join (os/homedir) "logseq" "graphs") graph-dir-or-path])))

deps/db/src/logseq/db/common/view.cljs

@@ -1,15 +1,13 @@
 (ns logseq.db.common.view
   "Main namespace for view fns."
-  (:require [cljs.reader :as reader]
-            [clojure.set :as set]
+  (:require [clojure.set :as set]
             [clojure.string :as string]
             [datascript.core :as d]
             [datascript.impl.entity :as de]
-            [logseq.common.log :as log]
             [logseq.common.util :as common-util]
             [logseq.db :as ldb]
             [logseq.db.common.entity-plus :as entity-plus]
-            [logseq.db.common.initial-data :as common-initial-data]
+            [logseq.db.common.reference :as db-reference]
             [logseq.db.frontend.class :as db-class]
             [logseq.db.frontend.entity-util :as entity-util]
             [logseq.db.frontend.property :as db-property]
@@ -50,7 +48,9 @@
                                          (into {})))
         get-property-value-fn (fn [entity]
                                 (if (de/entity? property)
-                                  (get-property-value-for-search entity property)
+                                  (if (= :date (:logseq.property/type property))
+                                    (:block/journal-day (get entity db-ident))
+                                    (get-property-value-for-search entity property))
                                   (get entity db-ident)))]
     (fn [entity]
       (cond
@@ -86,28 +86,29 @@
                              (not asc?)
                              reverse)
 
-                           (not (ldb/db-based-graph? db)) ; file graph properties don't support index
-                           (sort (common-util/by-sorting
-                                  [{:get-value get-value-fn
-                                    :asc? asc?}]) entities)
-
                            :else
-                           (let [ref-type? (= :db.type/ref (:db/valueType property))]
-                             (if ref-type?
-                               (sort-ref-entities-by-single-property entities sorting get-value-fn)
-                               (let [datoms (cond->
-                                             (->> (d/datoms db :avet id)
-                                                  (common-util/distinct-by :e)
-                                                  vec)
-                                              (not asc?)
-                                              rseq)
-                                     row-ids (set (map :db/id entities))
-                                     id->row (zipmap (map :db/id entities) entities)]
-                                 (keep
-                                  (fn [d]
-                                    (when (row-ids (:e d))
-                                      (id->row (:e d))))
-                                  datoms)))))
+                           (sort-ref-entities-by-single-property entities sorting get-value-fn)
+
+                           ;; (let [ref-type? (= :db.type/ref (:db/valueType property))]
+                           ;;   (if ref-type?
+                           ;;     (sort-ref-entities-by-single-property entities sorting get-value-fn)
+
+                           ;;     ;; FIXME: entity may not have the value
+                           ;;     ;; (let [datoms (cond->
+                           ;;     ;;               (->> (d/datoms db :avet id)
+                           ;;     ;;                    (common-util/distinct-by :e)
+                           ;;     ;;                    vec)
+                           ;;     ;;                (not asc?)
+                           ;;     ;;                rseq)
+                           ;;     ;;       row-ids (set (map :db/id entities))
+                           ;;     ;;       id->row (zipmap (map :db/id entities) entities)]
+                           ;;     ;;   (keep
+                           ;;     ;;    (fn [d]
+                           ;;     ;;      (when (row-ids (:e d))
+                           ;;     ;;        (id->row (:e d))))
+                           ;;     ;;    datoms))
+                           ;;     ))
+                           )
                          distinct)]
     (if partition?
       (partition-by get-value-fn sorted-entities)
@@ -261,94 +262,6 @@
             result)))
       (:filters filters)))))
 
-(defn filter-blocks
-  [filters ref-blocks]
-  (let [exclude-ids (set (map :db/id (:excluded filters)))
-        include-ids (set (map :db/id (:included filters)))
-        get-ids (fn [block]
-                  (set (map :db/id (:block/path-refs block))))]
-    (cond->> ref-blocks
-      (seq exclude-ids)
-      (remove (fn [block]
-                (let [ids (get-ids block)]
-                  (seq (set/intersection exclude-ids ids)))))
-
-      (seq include-ids)
-      (filter (fn [block]
-                (let [ids (get-ids block)]
-                  (set/subset? include-ids ids)))))))
-
-(defn get-filters
-  [db page]
-  (let [db-based? (entity-plus/db-based-graph? db)]
-    (if db-based?
-      (let [included-pages (:logseq.property.linked-references/includes page)
-            excluded-pages (:logseq.property.linked-references/excludes page)]
-        (when (or (seq included-pages) (seq excluded-pages))
-          {:included included-pages
-           :excluded excluded-pages}))
-      (let [k :filters
-            properties (:block/properties page)
-            properties-str (or (get properties k) "{}")]
-        (try (let [result (reader/read-string properties-str)]
-               (when (seq result)
-                 (let [excluded-pages (->> (filter #(false? (second %)) result)
-                                           (keep first)
-                                           (keep #(ldb/get-page db %)))
-                       included-pages (->> (filter #(true? (second %)) result)
-                                           (keep first)
-                                           (keep #(ldb/get-page db %)))]
-                   {:included included-pages
-                    :excluded excluded-pages})))
-             (catch :default e
-               (log/error :syntax/filters e)))))))
-
-(defn- get-linked-references
-  [db id]
-  (let [entity (d/entity db id)
-        ids (set (cons id (ldb/get-block-alias db id)))
-        refs (mapcat (fn [id] (:block/_refs (d/entity db id))) ids)
-        page-filters (get-filters db entity)
-        full-ref-blocks (->> refs
-                             (remove (fn [block] (common-initial-data/hidden-ref? db block id)))
-                             (common-util/distinct-by :db/id))
-        ref-blocks (cond->> full-ref-blocks
-                     (seq page-filters)
-                     (filter-blocks page-filters))
-        ref-pages-count (->> (mapcat (fn [block]
-                                       (->>
-                                        (cons
-                                         (:block/title (:block/page block))
-                                         (map (fn [b]
-                                                (when (and (ldb/page? b) (not= (:db/id b) id))
-                                                  (:block/title b)))
-                                              (:block/refs block)))
-                                        distinct))
-                                     full-ref-blocks)
-                             (remove nil?)
-                             (frequencies)
-                             (sort-by second #(> %1 %2)))]
-    {:ref-pages-count ref-pages-count
-     :ref-blocks ref-blocks}))
-
-(defn- get-unlinked-references
-  [db id]
-  (let [entity (d/entity db id)
-        title (string/lower-case (:block/title entity))]
-    (when-not (string/blank? title)
-      (let [ids (->> (d/datoms db :avet :block/title)
-                     (keep (fn [d]
-                             (when (and (not= id (:e d)) (string/includes? (string/lower-case (:v d)) title))
-                               (:e d)))))]
-        (keep
-         (fn [eid]
-           (let [e (d/entity db eid)]
-             (when-not (or (some #(= id %) (map :db/id (:block/refs e)))
-                           (:block/link e)
-                           (ldb/built-in? e))
-               e)))
-         ids)))))
-
 (defn- get-exclude-page-ids
   [db]
   (->>
@@ -407,10 +320,10 @@
        (keep (fn [id] (non-hidden-e id))))
 
       :linked-references
-      (get-linked-references db view-for-id)
+      (db-reference/get-linked-references db view-for-id)
 
       :unlinked-references
-      (get-unlinked-references db view-for-id)
+      (db-reference/get-unlinked-references db view-for-id)
 
       :query-result
       nil
@@ -476,7 +389,10 @@
                                    [label value] (cond ref-type?
                                                        [(db-property/property-value-content e)
                                                         (select-keys e [:db/id :block/uuid])]
-                                                       (= :datetime (:logseq.property/type property))
+                                                       ;; FIXME: Move query concerns out of :label as UI labels are usually strings
+                                                       ;; All non-string values need to be passed to the query builder since non-ref prop values use the actual value
+                                                       ;; This check is less fragile than listing all the property types to support e.g. :datetime, :checkbox, :keyword, :any
+                                                       (not (string? v))
                                                        [v v]
                                                        :else
                                                        [(str v) v])]
@@ -490,6 +406,10 @@
        values)
      (common-util/distinct-by :label))))
 
+(defn- get-query-properties
+  [entities]
+  (distinct (mapcat keys entities)))
+
 (defn ^:api ^:large-vars/cleanup-todo get-view-data
   [db view-id {:keys [journals? _view-for-id view-feature-type group-by-property-ident input query-entity-ids filters sorting]
                :as opts}]
@@ -576,4 +496,6 @@
        {:count (count filtered-entities)
         :data (distinct data')}
         (= feat-type :linked-references)
-        (assoc :ref-pages-count (:ref-pages-count entities-result))))))
+        (merge (select-keys entities-result [:ref-pages-count :ref-matched-children-ids]))
+        query?
+        (assoc :properties (get-query-properties entities-result))))))

deps/db/src/logseq/db/file_based/rules.cljc

@@ -14,7 +14,13 @@
   "Rules used by frontend.db.query-dsl for file graphs. The symbols ?b and ?p
   respectively refer to block and page. Do not alter them as they are
   programmatically built by the query-dsl ns"
-  {:page-property
+  {:block-parent
+   '[[(block-parent ?p ?c)
+      [?c :block/parent ?p]]
+     [(block-parent ?p ?c)
+      [?t :block/parent ?p]
+      (block-parent ?t ?c)]]
+   :page-property
    '[(page-property ?p ?key ?val)
      [?p :block/name]
      [?p :block/properties ?prop]

deps/db/src/logseq/db/frontend/kv_entity.cljs

@@ -25,4 +25,7 @@ RTC won't start when major-schema-versions don't match"
   :logseq.kv/graph-backup-folder          {:doc "Backup folder for automated backup feature"
                                            :rtc {:rtc/ignore-entity-when-init-upload true
                                                  :rtc/ignore-entity-when-init-download true}}
-  :logseq.kv/graph-initial-schema-version {:doc "Graph's schema version when created"})
+  :logseq.kv/graph-initial-schema-version {:doc "Graph's schema version when created"}
+  :logseq.kv/graph-last-gc-at             {:doc "Last time graph gc at"
+                                           :rtc {:rtc/ignore-entity-when-init-upload true
+                                                 :rtc/ignore-entity-when-init-download true}})

deps/db/src/logseq/db/frontend/validate.cljs

@@ -97,7 +97,7 @@
         ent-maps* (db-malli-schema/datoms->entities datoms)
         ent-maps (mapv
                   ;; Remove some UI interactions adding this e.g. import
-                  #(dissoc % :block.temp/fully-loaded?)
+                  #(dissoc % :block.temp/fully-loaded? :block.temp/has-children?)
                   (db-malli-schema/update-properties-in-ents db ent-maps*))
         errors (binding [db-malli-schema/*db-for-validate-fns* db]
                  (-> (map (fn [e]

deps/db/src/logseq/db/sqlite/build.cljs

@@ -645,7 +645,7 @@
     @uuids))
 
 (defn- build-blocks-tx*
-  [{:keys [pages-and-blocks properties auto-create-ontology?]
+  [{:keys [pages-and-blocks properties auto-create-ontology? build-existing-tx?]
     :as options}]
   (let [pages-and-blocks' (pre-build-pages-and-blocks pages-and-blocks properties (dissoc options :pages-and-blocks :properties))
         page-uuids (create-page-uuids pages-and-blocks')
@@ -662,6 +662,9 @@
                                            (mapv #(if (db-class/logseq-class? (:db/ident %))
                                                     %
                                                     (or (some->> (:db/ident %) class-ident->id (hash-map :db/id))
+                                                        ;; Allow existing user classes to be specified as idents
+                                                        (when (and build-existing-tx? (some->> (:db/ident %) (get classes)))
+                                                          (:db/ident %))
                                                         (throw (ex-info (str "No :db/id found for :db/ident " (pr-str %)) {}))))
                                                  cs)))
                                  m))

deps/db/src/logseq/db/sqlite/create_graph.cljs

@@ -190,10 +190,27 @@
      :logseq.property/hide? true
      :logseq.property/built-in? true})])
 
+(defn- build-initial-files [config-content]
+  [{:block/uuid (common-uuid/gen-uuid :builtin-block-uuid "logseq/config.edn")
+    :file/path (str "logseq/" "config.edn")
+    :file/content config-content
+    :file/created-at (js/Date.)
+    :file/last-modified-at (js/Date.)}
+   {:block/uuid (common-uuid/gen-uuid :builtin-block-uuid "logseq/custom.css")
+    :file/path (str "logseq/" "custom.css")
+    :file/content ""
+    :file/created-at (js/Date.)
+    :file/last-modified-at (js/Date.)}
+   {:block/uuid (common-uuid/gen-uuid :builtin-block-uuid "logseq/custom.js")
+    :file/path (str "logseq/" "custom.js")
+    :file/content ""
+    :file/created-at (js/Date.)
+    :file/last-modified-at (js/Date.)}])
+
 (defn build-db-initial-data
   "Builds tx of initial data for a new graph including key values, initial files,
    built-in properties and built-in classes"
-  [config-content & {:keys [import-type]}]
+  [config-content & {:keys [import-type graph-git-sha]}]
   (assert (string? config-content))
   (let [initial-data (cond->
                       [(sqlite-util/kv :logseq.kv/db-type "db")
@@ -204,22 +221,10 @@
                        {:db/ident :logseq.property/empty-placeholder
                         :block/uuid (common-uuid/gen-uuid :builtin-block-uuid :logseq.property/empty-placeholder)}]
                        import-type
-                       (into (sqlite-util/import-tx import-type)))
-        initial-files [{:block/uuid (common-uuid/gen-uuid :builtin-block-uuid "logseq/config.edn")
-                        :file/path (str "logseq/" "config.edn")
-                        :file/content config-content
-                        :file/created-at (js/Date.)
-                        :file/last-modified-at (js/Date.)}
-                       {:block/uuid (common-uuid/gen-uuid :builtin-block-uuid "logseq/custom.css")
-                        :file/path (str "logseq/" "custom.css")
-                        :file/content ""
-                        :file/created-at (js/Date.)
-                        :file/last-modified-at (js/Date.)}
-                       {:block/uuid (common-uuid/gen-uuid :builtin-block-uuid "logseq/custom.js")
-                        :file/path (str "logseq/" "custom.js")
-                        :file/content ""
-                        :file/created-at (js/Date.)
-                        :file/last-modified-at (js/Date.)}]
+                       (into (sqlite-util/import-tx import-type))
+                       graph-git-sha
+                       (conj (sqlite-util/kv :logseq.kv/graph-git-sha graph-git-sha)))
+        initial-files (build-initial-files config-content)
         {properties-tx :tx :keys [properties]} (build-initial-properties)
         db-ident->properties (zipmap (map :db/ident properties) properties)
         default-classes (build-initial-classes db-ident->properties)

deps/db/src/logseq/db/sqlite/debug.cljs

@@ -0,0 +1,37 @@
+(ns logseq.db.sqlite.debug
+  "SQLite debug fns"
+  (:require [cljs-bean.core :as bean]
+            [clojure.set]
+            [logseq.db.sqlite.util :as sqlite-util]))
+
+(defn find-missing-addresses
+  "WASM version to find missing addresses from the kvs table"
+  [^Object db]
+  (let [schema (some->> (.exec db #js {:sql "select content from kvs where addr = 0"
+                                       :rowMode "array"})
+                        bean/->clj
+                        ffirst
+                        sqlite-util/transit-read)
+        result (->> (.exec db #js {:sql "select addr, addresses from kvs"
+                                   :rowMode "array"})
+                    bean/->clj
+                    (keep (fn [[addr addresses]]
+                            [addr (bean/->clj (js/JSON.parse addresses))])))
+        used-addresses (set (concat (mapcat second result)
+                                    [0 1 (:eavt schema) (:avet schema) (:aevt schema)]))]
+    (clojure.set/difference used-addresses (set (map first result)))))
+
+(defn find-missing-addresses-node-version
+  "Node version to find missing addresses from the kvs table"
+  [^Object db]
+  (let [schema (let [stmt (.prepare db "select content from kvs where addr = ?")
+                     content (.-content (.get stmt 0))]
+                 (sqlite-util/transit-read content))
+        stmt (.prepare db "select addr, addresses from kvs")
+        result (->> (.all ^Object stmt)
+                    bean/->clj
+                    (keep (fn [{:keys [addr addresses]}]
+                            [addr (bean/->clj (js/JSON.parse addresses))])))
+        used-addresses (set (concat (mapcat second result)
+                                    [0 1 (:eavt schema) (:avet schema) (:aevt schema)]))]
+    (clojure.set/difference used-addresses  (set (map first result)))))

deps/db/src/logseq/db/sqlite/export.cljs

@@ -527,8 +527,9 @@
 
 (defn- build-view-nodes-export
   "Exports given nodes from a view. Nodes are a random mix of blocks and pages"
-  [db eids]
-  (let [nodes (map #(d/entity db %) eids)
+  [db rows {:keys [group-by?]}]
+  (let [eids (if group-by? (mapcat second rows) rows)
+        nodes (map #(d/entity db %) eids)
         property-value-ents (mapcat #(->> (apply dissoc (db-property/properties %) db-property/public-db-attribute-properties)
                                           vals
                                           (filter de/entity?))
@@ -859,7 +860,7 @@
           :page
           (build-page-export db (:page-id options))
           :view-nodes
-          (build-view-nodes-export db (:node-ids options))
+          (build-view-nodes-export db (:rows options) (select-keys options [:group-by?]))
           :selected-nodes
           (build-selected-nodes-export db (:node-ids options))
           :graph-ontology

deps/db/src/logseq/db/sqlite/gc.cljs

@@ -0,0 +1,120 @@
+(ns logseq.db.sqlite.gc
+  "GC unused addresses from `kvs` table"
+  (:require [cljs-bean.core :as bean]
+            [clojure.set :as set]
+            [logseq.db.sqlite.util :as sqlite-util]))
+
+(defn- walk-addresses
+  "Given a map of parent address to children addresses and a root address,
+   returns a set of all used addresses including the root and its descendants."
+  [root addr->children]
+  (println :debug :walk-addresses :root root)
+  (time
+   (letfn [(collect-addresses [addr]
+             (let [children (addr->children addr)]
+               (into #{addr} (mapcat collect-addresses children))))]
+     (collect-addresses root))))
+
+(defonce get-non-refed-addrs-sql
+  "WITH all_referenced AS (
+     SELECT CAST(value AS INTEGER) AS addr
+     FROM kvs, json_each(kvs.addresses)
+  )
+  SELECT kvs.addr
+  FROM kvs
+  WHERE kvs.addr NOT IN (SELECT addr FROM all_referenced)")
+
+(defn- get-unused-addresses
+  [db]
+  (let [schema (some->> (.exec db #js {:sql "select content from kvs where addr = 0"
+                                       :rowMode "array"})
+                        bean/->clj
+                        ffirst
+                        sqlite-util/transit-read)
+          ;; 0: Datascript sets 0 as the address to store the db's meta, including addresses for :eavt, :avet, and aevt index.
+          ;; 1: Datascript sets 1 for tail, to improve the performance
+        internal-addrs (set [0 1 (:eavt schema) (:avet schema) (:aevt schema)])
+        non-refed-addrs (->> (.exec db #js {:sql get-non-refed-addrs-sql
+                                            :rowMode "array"})
+                             (map first)
+                             set)]
+    (set/difference non-refed-addrs internal-addrs)))
+
+(defn gc-kvs-table!
+  "WASM version to GC kvs table to remove unused addresses"
+  [^Object db {:keys [full-gc?] :as opts}]
+  (when db
+    (let [unused-addresses (get-unused-addresses db)]
+      (if (seq unused-addresses)
+        (do
+          (println :debug :db-gc :unused-addresses unused-addresses)
+          (.transaction db (fn [tx]
+                             (doseq [addr unused-addresses]
+                               (.exec tx #js {:sql "Delete from kvs where addr = ?"
+                                              :bind #js [addr]}))))
+          (when full-gc?
+            (gc-kvs-table! db opts)))
+        (println :debug :db-gc "There's no garbage data that's need to be collected.")))))
+
+(defn- get-unused-addresses-node-version
+  [db]
+  (let [schema (let [stmt (.prepare db "select content from kvs where addr = ?")
+                     content (.-content (.get stmt 0))]
+                 (sqlite-util/transit-read content))
+        internal-addrs (set [0 1 (:eavt schema) (:avet schema) (:aevt schema)])
+        non-refed-addrs (let [stmt (.prepare db get-non-refed-addrs-sql)]
+                          (->> (.all ^object stmt)
+                               bean/->clj
+                               (map :addr)
+                               (set)))]
+    (set/difference non-refed-addrs internal-addrs)))
+
+(defn- get-unused-addresses-node-walk-version
+  [db]
+  (let [schema (let [stmt (.prepare db "select content from kvs where addr = ?")
+                     content (.-content (.get stmt 0))]
+                 (sqlite-util/transit-read content))
+        set-addresses #{(:eavt schema) (:avet schema) (:aevt schema)}
+        internal-addresses (conj set-addresses 0 1)
+        parent->children (let [stmt (.prepare db "select addr, addresses from kvs")]
+                           (->> (.all ^object stmt)
+                                bean/->clj
+                                (map (fn [{:keys [addr addresses]}]
+                                       [addr (bean/->clj (js/JSON.parse addresses))]))
+                                (into {})))
+        used-addresses (->> (mapcat (fn [set-root-addr]
+                                      (walk-addresses set-root-addr parent->children)) set-addresses)
+                            set
+                            (set/union internal-addresses))]
+    (set/difference (set (keys parent->children)) used-addresses)))
+
+(defn gc-kvs-table-node-version!
+  "Node version to GC kvs table to remove unused addresses
+  `walk?` - `true`: walk all used addresses, `false`: gc recursively"
+  [^Object db walk?]
+  (let [unused-addresses (if walk?
+                           (get-unused-addresses-node-walk-version db)
+                           (get-unused-addresses-node-version db))
+        addrs-count (let [stmt (.prepare db "select count(*) as c from kvs")]
+                      (.-c (.get stmt)))]
+    (println :debug "addrs total count: " addrs-count)
+    (if (seq unused-addresses)
+      (do
+        (println :debug :db-gc :unused-addresses-count (count unused-addresses))
+        (let [stmt (.prepare db "Delete from kvs where addr = ?")
+              delete (.transaction
+                      db
+                      (fn [addrs]
+                        (doseq [addr addrs]
+                          (.run stmt addr))))]
+          (delete (bean/->js unused-addresses))
+          (when-not walk?
+            (gc-kvs-table-node-version! db false))))
+      (println :debug :db-gc "There's no garbage data that's need to be collected."))))
+
+(defn ensure-no-garbage
+  [^Object db]
+  (let [unused-addresses (get-unused-addresses-node-version db)]
+    ;; (println :debug :db-gc :unused-addresses-count (count unused-addresses))
+    ;; (println :debug :unused-addresses unused-addresses)
+    (empty? unused-addresses)))

deps/db/test/logseq/db/sqlite/export_test.cljs

@@ -524,14 +524,14 @@
                             (mapv #(vector :block/uuid (:block/uuid %)))))
         conn2 (db-test/create-conn)
         {:keys [init-tx block-props-tx] :as _txs}
-        (-> (sqlite-export/build-export @conn {:export-type :view-nodes :node-ids (get-node-ids @conn)})
+        (-> (sqlite-export/build-export @conn {:export-type :view-nodes :rows (get-node-ids @conn)})
             (sqlite-export/build-import @conn2 {}))
         ;; _ (cljs.pprint/pprint _txs)
         _ (d/transact! conn2 init-tx)
         _ (d/transact! conn2 block-props-tx)
         _ (validate-db @conn2)
         imported-nodes (sqlite-export/build-export @conn2 {:export-type :view-nodes
-                                                           :node-ids (get-node-ids @conn2)})]
+                                                           :rows (get-node-ids @conn2)})]
 
     (is (= (sort-pages-and-blocks (:pages-and-blocks original-data)) (:pages-and-blocks imported-nodes)))
     (is (= (expand-properties (:properties original-data)) (:properties imported-nodes)))

deps/db/test/logseq/db/sqlite/gc_test.cljs

@@ -0,0 +1,49 @@
+(ns logseq.db.sqlite.gc-test
+  (:require ["fs" :as fs]
+            ["path" :as node-path]
+            [cljs.test :refer [deftest async use-fixtures is testing]]
+            [datascript.core :as d]
+            [logseq.db.common.sqlite-cli :as sqlite-cli]
+            [logseq.db.sqlite.debug :as sqlite-debug]
+            [logseq.db.sqlite.gc :as sqlite-gc]))
+
+(use-fixtures
+  :each
+ ;; Cleaning tmp/ before leaves last tmp/ after a test run for dev and debugging
+  {:before
+   #(async done
+           (if (fs/existsSync "tmp")
+             (fs/rm "tmp" #js {:recursive true} (fn [err]
+                                                  (when err (js/console.log err))
+                                                  (done)))
+             (done)))})
+
+(defn- create-graph-dir
+  [dir db-name]
+  (fs/mkdirSync (node-path/join dir db-name) #js {:recursive true}))
+
+(deftest ^:long gc-kvs-table-test
+  (testing "Create a datascript db, gc it and ensure there's no missing addrs and garbage addrs"
+    (create-graph-dir "tmp/graphs" "test-db")
+
+    (let [{:keys [conn sqlite]} (sqlite-cli/open-sqlite-datascript! "tmp/graphs" "test-db")
+          tx-data (map (fn [i] {:block/uuid (random-uuid)
+                                :block/title (str "title " i)})
+                       (range 0 500000))]
+      (println "DB start transacting")
+      (d/transact! conn tx-data)
+      (println "DB transacted")
+      (let [non-ordered-tx (->> (shuffle tx-data)
+                                (take 100000)
+                                (map (fn [block] [:db/retractEntity [:block/uuid (:block/uuid block)]])))]
+        (d/transact! conn non-ordered-tx))
+      (println "gc time")
+      ;; `true` to walk addresses and `false` to recursively run gc
+      (time (sqlite-gc/gc-kvs-table-node-version! sqlite false))
+
+      ;; ensure there's no missing address (broken db)
+      (is (empty? (sqlite-debug/find-missing-addresses-node-version sqlite))
+          "Found missing addresses!")
+
+      (is (true? (sqlite-gc/ensure-no-garbage sqlite))
+          "Found garbage addresses!"))))

deps/db/yarn.lock

@@ -2,9 +2,9 @@
 # yarn lockfile v1
 
 
-"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v23":
-  version "1.2.173-feat-db-v23"
-  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/810782c4ddba6346c4ab8ae6740b60438c07cd01"
+"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v24":
+  version "1.2.173-feat-db-v24"
+  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/3d5b78b0382c7253bf9874c1f38586dd338434f4"
   dependencies:
     import-meta-resolve "^4.1.0"
 

deps/graph-parser/nbb.edn

@@ -5,6 +5,6 @@
   logseq/db
   {:local/root "../db"}
   io.github.nextjournal/nbb-test-runner
-  {:git/sha "012017a8a8983d05f905f38f631f5222f25b9ed9"}
+  {:git/sha "b379325cfa5a3306180649da5de3bf5166414e71"}
   borkdude/rewrite-edn {:mvn/version "0.4.9"}
   io.github.pez/baldr {:mvn/version "1.0.9"}}}

deps/graph-parser/package.json

@@ -3,7 +3,7 @@
   "version": "1.0.0",
   "private": true,
   "devDependencies": {
-    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v23",
+    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v24",
     "better-sqlite3": "11.10.0"
   },
   "dependencies": {

deps/graph-parser/script/db_import.cljs

@@ -4,15 +4,14 @@
    the import process"
   (:require ["fs" :as fs]
             ["fs/promises" :as fsp]
-            ["os" :as os]
             ["path" :as node-path]
-            #_:clj-kondo/ignore
             [babashka.cli :as cli]
             [cljs.pprint :as pprint]
             [clojure.set :as set]
             [clojure.string :as string]
             [datascript.core :as d]
             [logseq.common.graph :as common-graph]
+            [logseq.db.common.sqlite-cli :as sqlite-cli]
             [logseq.graph-parser.exporter :as gp-exporter]
             [logseq.outliner.cli :as outliner-cli]
             [logseq.outliner.pipeline :as outliner-pipeline]
@@ -125,17 +124,6 @@
       (p/let [_ (gp-exporter/export-doc-files conn files' <read-file doc-options)]
         {:import-state (:import-state doc-options)}))))
 
-(defn- get-dir-and-db-name
-  "Gets dir and db name for use with open-db! Works for relative and absolute paths and
-   defaults to ~/logseq/graphs/ when no '/' present in name"
-  [graph-dir]
-  (if (string/includes? graph-dir "/")
-    (let [resolve-path' #(if (node-path/isAbsolute %) %
-                             ;; $ORIGINAL_PWD used by bb tasks to correct current dir
-                             (node-path/join (or js/process.env.ORIGINAL_PWD ".") %))]
-      ((juxt node-path/dirname node-path/basename) (resolve-path' graph-dir)))
-    [(node-path/join (os/homedir) "logseq" "graphs") graph-dir]))
-
 (def spec
   "Options spec"
   {:help {:alias :h
@@ -171,9 +159,14 @@
             (println (str "Usage: $0 FILE-GRAPH DB-GRAPH [OPTIONS]\nOptions:\n"
                           (cli/format-opts {:spec spec})))
             (js/process.exit 1))
-        [dir db-name] (get-dir-and-db-name db-graph-dir)
+        init-conn-args (sqlite-cli/->open-db-args db-graph-dir)
+        db-name (if (= 1 (count init-conn-args)) (first init-conn-args) (second init-conn-args))
+        db-full-dir (if (= 1 (count init-conn-args))
+                      (node-path/dirname (first init-conn-args))
+                      (apply node-path/join init-conn-args))
         file-graph' (resolve-path file-graph)
-        conn (outliner-cli/init-conn dir db-name {:classpath (cp/get-classpath)})
+        conn (apply outliner-cli/init-conn (conj init-conn-args {:classpath (cp/get-classpath)
+                                                                 :import-type :cli/db-import}))
         directory? (.isDirectory (fs/statSync file-graph'))
         user-options (cond-> (merge {:all-tags false} (dissoc options :verbose :files :help :continue))
                        ;; coerce option collection into strings
@@ -182,12 +175,11 @@
                        true
                        (set/rename-keys {:all-tags :convert-all-tags? :remove-inline-tags :remove-inline-tags?}))
         _ (when (:verbose options) (prn :options user-options))
-        options' (merge {:user-options user-options
-                         :graph-name db-name}
+        options' (merge {:user-options user-options}
                         (select-keys options [:files :verbose :continue :debug]))]
     (p/let [{:keys [import-state]}
             (if directory?
-              (import-file-graph-to-db file-graph' (node-path/join dir db-name) conn options')
+              (import-file-graph-to-db file-graph' db-full-dir conn options')
               (import-files-to-db file-graph' conn options'))]
 
       (when-let [ignored-props (seq @(:ignored-properties import-state))]

deps/graph-parser/src/logseq/graph_parser.cljs

@@ -51,7 +51,7 @@
   (let [existing-file-page (get-file-page db file-path)
         pages-to-clear (distinct (filter some? [existing-file-page (:db/id file-page)]))
         blocks (mapcat (fn [page-id]
-                         (ldb/get-page-blocks db page-id {:pull-keys [:db/id :block/uuid]}))
+                         (:block/_page (d/entity db page-id)))
                        pages-to-clear)
         retain-uuids (set (keep :block/uuid retain-uuid-blocks))]
     (retract-blocks-tx (distinct blocks) retain-uuids)))
@@ -63,12 +63,10 @@ Options available:
   * :delete-blocks-fn - Optional fn which is called with the new page, file and existing block uuids
   which may be referenced elsewhere. Used to delete the existing blocks before saving the new ones.
    Implemented in file-common-handler/validate-and-get-blocks-to-delete for IoC
-* :skip-db-transact? - Boolean which skips transacting in order to batch transactions. Default is false
-* :extract-options - Options map to pass to extract/extract"
+  * :extract-options - Options map to pass to extract/extract"
   ([conn file-path content] (parse-file conn file-path content {}))
-  ([conn file-path content {:keys [delete-blocks-fn extract-options skip-db-transact? ctime mtime]
-                            :or {delete-blocks-fn (constantly [])
-                                 skip-db-transact? false}
+  ([conn file-path content {:keys [delete-blocks-fn extract-options ctime mtime]
+                            :or {delete-blocks-fn (constantly [])}
                             :as options}]
    (let [format (common-util/get-format file-path)
          file-content [{:file/path file-path}]
@@ -91,7 +89,7 @@ Options available:
 
                      :else nil)
                block-ids (map (fn [block] {:block/uuid (:block/uuid block)}) blocks)
-               delete-blocks (delete-blocks-fn @conn (first pages) file-path block-ids)
+               delete-blocks (delete-blocks-fn (first pages) file-path block-ids)
                block-refs-ids (->> (mapcat :block/refs blocks)
                                    (filter (fn [ref] (and (vector? ref)
                                                           (= :block/uuid (first ref)))))
@@ -110,13 +108,9 @@ Options available:
                           (or ctime (nil? file-entity))
                           (assoc :file/created-at (or ctime (js/Date.)))
                           mtime
-                          (assoc :file/last-modified-at mtime))])
-         result (if skip-db-transact?
-                  tx
-                  (do
-                    (ldb/transact! conn tx (select-keys options [:new-graph? :from-disk?]))
-                    nil))]
-     {:tx result
+                          (assoc :file/last-modified-at mtime))])]
+     (ldb/transact! conn tx (select-keys options [:new-graph? :from-disk?]))
+     {:tx tx
       :ast ast})))
 
 (defn filter-files

deps/graph-parser/src/logseq/graph_parser/block.cljs

@@ -698,23 +698,15 @@
                                                    (str (gp-property/colons-org "id") " " (:block/uuid block)))))]
                                (string/replace-first c replace-str ""))))))
 
-(defn block-exists-in-another-page?
-  "For sanity check only.
-   For renaming file externally, the file is actually deleted and transacted before-hand."
-  [db block-uuid current-page-name]
-  (when (and db current-page-name)
-    (when-let [block-page-name (:block/name (:block/page (d/entity db [:block/uuid block-uuid])))]
-      (not= current-page-name block-page-name))))
-
 (defn fix-block-id-if-duplicated!
-  "If the block exists in another page, we need to fix it
-   If the block exists in the current extraction process, we also need to fix it"
-  [db page-name *block-exists-in-extraction block]
-  (let [block (if (or (@*block-exists-in-extraction (:block/uuid block))
-                      (block-exists-in-another-page? db (:block/uuid block) page-name))
+  "If the block exists in another page or the current page, we need to fix it"
+  [db page-name *extracted-block-ids block]
+  (let [block-page-name (:block/name (:block/page (d/entity db [:block/uuid (:block/uuid block)])))
+        block (if (or (and block-page-name (not= block-page-name page-name))
+                      (contains? @*extracted-block-ids (:block/uuid block)))
                 (fix-duplicate-id block)
                 block)]
-    (swap! *block-exists-in-extraction conj (:block/uuid block))
+    (swap! *extracted-block-ids conj (:block/uuid block))
     block))
 
 (defn extract-blocks

deps/graph-parser/src/logseq/graph_parser/cli.cljs

@@ -3,10 +3,10 @@
   (:require ["fs" :as fs]
             ["path" :as path]
             [clojure.edn :as edn]
-            [logseq.common.graph :as common-graph]
             [logseq.common.config :as common-config]
-            [logseq.graph-parser :as graph-parser]
+            [logseq.common.graph :as common-graph]
             [logseq.common.util :as common-util]
+            [logseq.graph-parser :as graph-parser]
             [logseq.graph-parser.db :as gp-db]))
 
 (defn- slurp
@@ -28,10 +28,10 @@
   [dir* config]
   (let [dir (path/resolve dir*)]
     (->> (common-graph/get-files dir)
-        (map #(hash-map :file/path %))
-        graph-parser/filter-files
-        (remove-hidden-files dir config)
-        (mapv #(assoc % :file/content (slurp (:file/path %)))))))
+         (map #(hash-map :file/path %))
+         graph-parser/filter-files
+         (remove-hidden-files dir config)
+         (mapv #(assoc % :file/content (slurp (:file/path %)))))))
 
 (defn- read-config
   "Reads repo-specific config from logseq/config.edn"
@@ -45,8 +45,7 @@
   [conn files {:keys [config] :as options}]
   (let [extract-options (merge {:date-formatter (common-config/get-date-formatter config)
                                 :user-config config
-                                :filename-format (or (:file/name-format config) :legacy)
-                                :extracted-block-ids (atom #{})}
+                                :filename-format (or (:file/name-format config) :legacy)}
                                (select-keys options [:verbose]))]
     (mapv
      (fn [{:file/keys [path content]}]

deps/graph-parser/src/logseq/graph_parser/extract.cljc

@@ -220,15 +220,13 @@
 (defn- extract-pages-and-blocks
   "uri-encoded? - if is true, apply URL decode on the file path
    options -
-     :extracted-block-ids - An atom that contains all block ids that have been extracted in the current page (not yet saved to db)
      :resolve-uuid-fn - Optional fn which is called to resolve uuids of each block. Enables diff-merge
        (2 ways diff) based uuid resolution upon external editing.
        returns a list of the uuids, given the receiving ast, or nil if not able to resolve.
        Implemented in reset-file-handler/diff-merge-uuids-2ways for IoC
        Called in gp-extract/extract as AST is being parsed and properties are extracted there"
-  [format ast properties file content {:keys [date-formatter db filename-format extracted-block-ids resolve-uuid-fn]
-                                       :or {extracted-block-ids (atom #{})
-                                            resolve-uuid-fn (constantly nil)}
+  [format ast properties file content {:keys [date-formatter db filename-format resolve-uuid-fn]
+                                       :or {resolve-uuid-fn (constantly nil)}
                                        :as options}]
   (assert db "Datascript DB is required")
   (try
@@ -237,9 +235,10 @@
           options' (assoc options :page-name page-name)
           ;; In case of diff-merge (2way) triggered, use the uuids to override the ones extracted from the AST
           override-uuids (resolve-uuid-fn format ast content options')
+          *extracted-block-ids (atom #{})
           blocks (->> (gp-block/extract-blocks ast content format options')
                       (attach-block-ids-if-match override-uuids)
-                      (mapv #(gp-block/fix-block-id-if-duplicated! db page-name extracted-block-ids %))
+                      (mapv #(gp-block/fix-block-id-if-duplicated! db page-name *extracted-block-ids %))
                       ;; FIXME: use page uuid
                       (gp-block/with-parent-and-order {:block/name page-name})
                       (vec))

deps/graph-parser/src/logseq/graph_parser/test/docs_graph_helper.cljs

@@ -1,11 +1,11 @@
 (ns logseq.graph-parser.test.docs-graph-helper
   "Helper fns for setting up and running tests against docs graph"
-  (:require ["fs" :as fs]
-            ["child_process" :as child-process]
+  (:require ["child_process" :as child-process]
+            ["fs" :as fs]
             [cljs.test :refer [is testing]]
             [clojure.string :as string]
-            [logseq.common.config :as common-config]
-            [datascript.core :as d]))
+            [datascript.core :as d]
+            [logseq.common.config :as common-config]))
 
 ;; Helper fns for test setup
 ;; =========================
@@ -105,16 +105,15 @@
                 (into {})))
         "Task marker counts")
 
-    (is (= {:markdown 7322 :org 500} (get-block-format-counts db))
+    (is (= {:markdown 7372 :org 500} (get-block-format-counts db))
         "Block format counts")
 
-    (is (= {:rangeincludes 13, :description 137, :updated-at 46, :tags 5,
-            :logseq.order-list-type 16, :query-table 8, :logseq.macro-arguments 105,
-            :parent 14, :logseq.tldraw.shape 79, :card-last-score 5, :card-repeats 5,
-            :name 16, :card-next-schedule 5, :ls-type 79, :card-last-interval 5, :type 166,
-            :template 5, :domainincludes 7, :title 114, :alias 62, :supports 6, :id 145,
-            :url 30, :card-ease-factor 5, :logseq.macro-name 105, :created-at 46,
-            :card-last-reviewed 5, :platforms 79, :initial-version 16, :heading 315}
+    (is (= {:rangeincludes 13, :description 137, :updated-at 46, :tags 5, :logseq.order-list-type 16, :query-table 8,
+            :logseq.macro-arguments 105, :parent 14, :logseq.tldraw.shape 79, :card-last-score 5, :card-repeats 5,
+            :name 16, :card-next-schedule 5, :ls-type 79, :card-last-interval 5, :type
+            166, :template 5, :domainincludes 7, :title 114, :alias 62, :supports 6, :id
+            146, :url 30, :card-ease-factor 5, :logseq.macro-name 105, :created-at 46,
+            :card-last-reviewed 5, :platforms 79, :initial-version 16, :heading 332}
            (get-top-block-properties db))
         "Counts for top block properties")
 
@@ -159,7 +158,7 @@
   ;; Counts assertions help check for no major regressions. These counts should
   ;; only increase over time as the docs graph rarely has deletions
   (testing "Counts"
-    (is (= 340 (count files)) "Correct file count")
+    (is (= 339 (count files)) "Correct file count")
     (is (= 33
            (ffirst
             (d/q '[:find (count ?b)

deps/graph-parser/test/logseq/graph_parser/cli_test.cljs

@@ -1,20 +1,20 @@
 (ns ^:node-only logseq.graph-parser.cli-test
   (:require [cljs.test :refer [deftest is testing]]
-            [logseq.graph-parser.cli :as gp-cli]
-            [logseq.graph-parser.test.docs-graph-helper :as docs-graph-helper]
             [clojure.string :as string]
-            [datascript.core :as d]))
+            [datascript.core :as d]
+            [logseq.graph-parser.cli :as gp-cli]
+            [logseq.graph-parser.test.docs-graph-helper :as docs-graph-helper]))
 
 ;; Integration test that test parsing a large graph like docs
 (deftest ^:integration parse-graph
-  (let [graph-dir "test/resources/docs-0.10.9"
-        _ (docs-graph-helper/clone-docs-repo-if-not-exists graph-dir "v0.10.9")
+  (let [graph-dir "test/resources/docs-0.10.12"
+        _ (docs-graph-helper/clone-docs-repo-if-not-exists graph-dir "v0.10.12")
         {:keys [conn files asts]} (gp-cli/parse-graph graph-dir {:verbose false})]
 
     (docs-graph-helper/docs-graph-assertions @conn graph-dir files)
 
     (testing "Additional counts"
-      (is (= 57814 (count (d/datoms @conn :eavt))) "Correct datoms count"))
+      (is (= 58149 (count (d/datoms @conn :eavt))) "Correct datoms count"))
 
     (testing "Asts"
       (is (seq asts) "Asts returned are non-zero")

deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs

@@ -129,9 +129,9 @@
 ;; =====
 
 (deftest-async ^:integration export-docs-graph-with-convert-all-tags
-  (p/let [file-graph-dir "test/resources/docs-0.10.9"
+  (p/let [file-graph-dir "test/resources/docs-0.10.12"
           start-time (cljs.core/system-time)
-          _ (docs-graph-helper/clone-docs-repo-if-not-exists file-graph-dir "v0.10.9")
+          _ (docs-graph-helper/clone-docs-repo-if-not-exists file-graph-dir "v0.10.12")
           conn (db-test/create-conn)
           _ (db-pipeline/add-listener conn)
           {:keys [import-state]}
@@ -139,7 +139,7 @@
           end-time (cljs.core/system-time)]
 
     ;; Add multiplicative factor for CI as it runs about twice as slow
-    (let [max-time (-> 20 (* (if js/process.env.CI 2 1)))]
+    (let [max-time (-> 25 (* (if js/process.env.CI 2 1)))]
       (is (< (-> end-time (- start-time) (/ 1000)) max-time)
           (str "Importing large graph takes less than " max-time "s")))
 

deps/graph-parser/test/logseq/graph_parser/mldoc_test.cljs

@@ -1,10 +1,10 @@
 (ns logseq.graph-parser.mldoc-test
-  (:require [logseq.graph-parser.mldoc :as gp-mldoc]
-            [logseq.graph-parser.text :as text]
+  (:require [cljs.test :refer [testing deftest are is]]
             [clojure.string :as string]
-            [logseq.graph-parser.test.docs-graph-helper :as docs-graph-helper]
             [logseq.graph-parser.cli :as gp-cli]
-            [cljs.test :refer [testing deftest are is]]))
+            [logseq.graph-parser.mldoc :as gp-mldoc]
+            [logseq.graph-parser.test.docs-graph-helper :as docs-graph-helper]
+            [logseq.graph-parser.text :as text]))
 
 (deftest test-link
   (testing "non-link"
@@ -127,7 +127,7 @@ body"
       line 2
  line 3
 line 4"]
-              (gp-mldoc/remove-indentation-spaces s 2 false)))) 
+              (gp-mldoc/remove-indentation-spaces s 2 false))))
     (is (=  "\t- block 1.1\n  line 1\n    line 2\nline 3\nline 4"
             (let [s "\t- block 1.1
 \t    line 1
@@ -135,11 +135,10 @@ line 4"]
 \t line 3
 \tline 4"]
               (gp-mldoc/remove-indentation-spaces s 3 false))))))
-    
 
 (deftest ^:integration test->edn
-  (let [graph-dir "test/resources/docs-0.10.9"
-        _ (docs-graph-helper/clone-docs-repo-if-not-exists graph-dir "v0.10.9")
+  (let [graph-dir "test/resources/docs-0.10.12"
+        _ (docs-graph-helper/clone-docs-repo-if-not-exists graph-dir "v0.10.12")
         files (#'gp-cli/build-graph-files graph-dir {})
         asts-by-file (->> files
                           (map (fn [{:file/keys [path content]}]
@@ -154,12 +153,12 @@ line 4"]
             "Drawer" 1,
             "Example" 22,
             "Footnote_Definition" 2,
-            "Heading" 6716,
+            "Heading" 6764,
             "Hiccup" 9,
             "List" 25,
-            "Paragraph" 626,
+            "Paragraph" 629,
             "Properties" 85,
-            "Property_Drawer" 509,
+            "Property_Drawer" 510,
             "Quote" 28,
             "Raw_Html" 18,
             "Src" 82,

deps/graph-parser/test/logseq/graph_parser_test.cljs

@@ -1,12 +1,12 @@
 (ns logseq.graph-parser-test
   (:require [cljs.test :refer [deftest testing is are]]
             [clojure.string :as string]
+            [datascript.core :as d]
+            [logseq.db :as ldb]
             [logseq.graph-parser :as graph-parser]
-            [logseq.graph-parser.db :as gp-db]
             [logseq.graph-parser.block :as gp-block]
-            [logseq.graph-parser.property :as gp-property]
-            [datascript.core :as d]
-            [logseq.db :as ldb]))
+            [logseq.graph-parser.db :as gp-db]
+            [logseq.graph-parser.property :as gp-property]))
 
 (def foo-edn
   "Example exported whiteboard page as an edn exportable."
@@ -78,7 +78,7 @@
                                                         (throw (js/Error "Testing unexpected failure")))]
         (try
           (parse-file conn "foo.md" "- id:: 628953c1-8d75-49fe-a648-f4c612109098"
-                      {:delete-blocks-fn (fn [_db page _file _uuids]
+                      {:delete-blocks-fn (fn [page _file _uuids]
                                            (reset! deleted-page page))})
           (catch :default _)))
       (is (= nil @deleted-page)

deps/graph-parser/test/resources/exporter-test-graph/logseq/config.edn

@@ -271,10 +271,6 @@
  ;; Default value: 2
  :ref/default-open-blocks-level 2
 
- ;; Configure the threshold for linked references before collapsing.
- ;; Default value: 100
- :ref/linked-references-collapsed-threshold 50
-
  ;; Graph view configuration.
  ;; Example usage:
  ;; :graph/settings

deps/graph-parser/yarn.lock

@@ -2,9 +2,9 @@
 # yarn lockfile v1
 
 
-"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v23":
-  version "1.2.173-feat-db-v23"
-  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/810782c4ddba6346c4ab8ae6740b60438c07cd01"
+"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v24":
+  version "1.2.173-feat-db-v24"
+  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/3d5b78b0382c7253bf9874c1f38586dd338434f4"
   dependencies:
     import-meta-resolve "^4.1.0"
 

deps/outliner/deps.edn

@@ -1,7 +1,7 @@
 {:deps
  ;; These nbb-logseq deps are kept in sync with https://github.com/logseq/nbb-logseq/blob/main/bb.edn
  {datascript/datascript {:git/url "https://github.com/logseq/datascript" ;; fork
-                         :sha     "4b1f15f05a6b4a718a62c247956206480e361ea6"}
+                         :sha     "b28f6574b9447bba9ccaa5d2b0cfd79308acf0e3"}
   com.cognitect/transit-cljs {:mvn/version "0.8.280"}
 
   ;; Any other deps should be added here and to nbb.edn

deps/outliner/nbb.edn

@@ -7,4 +7,4 @@
   metosin/malli
   {:mvn/version "0.16.1"}
   io.github.nextjournal/nbb-test-runner
-  {:git/sha "012017a8a8983d05f905f38f631f5222f25b9ed9"}}}
+  {:git/sha "b379325cfa5a3306180649da5de3bf5166414e71"}}}

deps/outliner/package.json

@@ -3,7 +3,7 @@
   "version": "1.0.0",
   "private": true,
   "devDependencies": {
-    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v23"
+    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v24"
   },
   "dependencies": {
     "better-sqlite3": "11.10.0",

deps/outliner/script/transact.cljs

@@ -1,34 +1,21 @@
 (ns transact
   "This script generically runs transactions against the queried blocks"
-  (:require ["os" :as os]
-            ["path" :as node-path]
-            [clojure.edn :as edn]
-            [clojure.string :as string]
+  (:require [clojure.edn :as edn]
             [datascript.core :as d]
-            [logseq.db.frontend.rules :as rules]
             [logseq.db.common.sqlite-cli :as sqlite-cli]
+            [logseq.db.frontend.rules :as rules]
             [logseq.outliner.db-pipeline :as db-pipeline]
             [nbb.core :as nbb]))
 
-(defn- get-dir-and-db-name
-  "Gets dir and db name for use with open-db! Works for relative and absolute paths and
-   defaults to ~/logseq/graphs/ when no '/' present in name"
-  [graph-dir]
-  (if (string/includes? graph-dir "/")
-    (let [resolve-path' #(if (node-path/isAbsolute %) %
-                             ;; $ORIGINAL_PWD used by bb tasks to correct current dir
-                             (node-path/join (or js/process.env.ORIGINAL_PWD ".") %))]
-      ((juxt node-path/dirname node-path/basename) (resolve-path' graph-dir)))
-    [(node-path/join (os/homedir) "logseq" "graphs") graph-dir]))
-
 (defn -main [args]
   (when (< (count args) 3)
     (println "Usage: $0 GRAPH-DIR QUERY TRANSACT-FN")
     (js/process.exit 1))
   (let [[graph-dir query* transact-fn*] args
         dry-run? (contains? (set args) "-n")
-        [dir db-name] (get-dir-and-db-name graph-dir)
-        conn (sqlite-cli/open-db! dir db-name)
+        open-db-args (sqlite-cli/->open-db-args graph-dir)
+        db-name (if (= 1 (count open-db-args)) (first open-db-args) (second open-db-args))
+        conn (apply sqlite-cli/open-db! open-db-args)
         ;; find blocks to update
         query (into (edn/read-string query*) [:in '$ '%]) ;; assumes no :in are in queries
         transact-fn (edn/read-string transact-fn*)

deps/outliner/src/logseq/outliner/cli.cljs

@@ -1,6 +1,7 @@
 (ns ^:node-only logseq.outliner.cli
   "Primary ns for outliner CLI fns"
-  (:require ["fs" :as fs]
+  (:require ["child_process" :as child-process]
+            ["fs" :as fs]
             ["path" :as node-path]
             [borkdude.rewrite-edn :as rewrite]
             [clojure.string :as string]
@@ -26,6 +27,14 @@
               m)
       str))
 
+(defn- get-git-sha
+  []
+  (let [res (child-process/spawnSync "git"
+                                     #js ["rev-parse" "--short" "HEAD"]
+                                     #js {})]
+    (when (zero? (.-status res))
+      (string/trim (str (.-stdout res))))))
+
 (defn- setup-init-data
   "Setup initial data same as frontend.handler.repo/create-db"
   [conn {:keys [additional-config classpath import-type]
@@ -37,8 +46,11 @@
           true
           (common-config/create-config-for-db-graph)
           additional-config
-          (pretty-print-merge additional-config))]
-    (d/transact! conn (sqlite-create-graph/build-db-initial-data config-content {:import-type import-type}))))
+          (pretty-print-merge additional-config))
+        git-sha (get-git-sha)]
+    (d/transact! conn (sqlite-create-graph/build-db-initial-data config-content
+                                                                 (merge {:import-type import-type}
+                                                                        (when git-sha {:graph-git-sha git-sha}))))))
 
 (defn init-conn
   "Create sqlite DB, initialize datascript connection and sync listener and then
@@ -46,14 +58,19 @@
    * :additional-config - Additional config map to merge into repo config.edn
    * :classpath - A java classpath string i.e. paths delimited by ':'. Used to find default config.edn
      that comes with Logseq"
-  [dir db-name & [opts]]
-  (fs/mkdirSync (node-path/join dir db-name) #js {:recursive true})
-  ;; Same order as frontend.db.conn/start!
-  (let [conn (sqlite-cli/open-db! dir db-name)]
+  [& args*]
+  (let [[args opts] (if (map? (last args*))
+                      [(butlast args*) (last args*)]
+                      [args* {}])
+        ;; Only mkdir when a dir and db-name are passed
+        _ (when (= 2 (count args))
+            (fs/mkdirSync (apply node-path/join args) #js {:recursive true}))
+        ;; Same order as frontend.db.conn/start!
+        conn (apply sqlite-cli/open-db! args)]
     (db-pipeline/add-listener conn)
     (setup-init-data conn opts)
     conn))
 
 (def build-blocks-tx
   "An alias for build-blocks-tx to specify default options for this ns"
-  sqlite-build/build-blocks-tx)
+  sqlite-build/build-blocks-tx)

deps/outliner/src/logseq/outliner/core.cljs

@@ -240,7 +240,8 @@
           m* (cond->
               (-> data'
                   (dissoc :block/children :block/meta :block/unordered
-                          :block.temp/ast-title :block.temp/ast-body :block/level :block.temp/fully-loaded?)
+                          :block.temp/ast-title :block.temp/ast-body :block/level :block.temp/fully-loaded?
+                          :block.temp/has-children?)
                   common-util/remove-nils
 
                   (fix-tag-ids db {:db-graph? db-based?}))

deps/outliner/src/logseq/outliner/property.cljs

@@ -40,8 +40,11 @@
                             true
                             (assoc property-id value)
                             (and (contains? #{:logseq.property/status :logseq.property/scheduled :logseq.property/deadline} property-id)
-                                 (or (empty? (:block/tags block)) (ldb/internal-page? block)))
-                            (assoc :block/tags :logseq.class/Task))]
+                                 (or (empty? (:block/tags block)) (ldb/internal-page? block))
+                                 (not (get (d/pull @conn [property-id] (:db/id block)) property-id)))
+                            (assoc :block/tags :logseq.class/Task)
+                            (= :logseq.property/template-applied-to property-id)
+                            (assoc :block/tags :logseq.class/Template))]
       (cond-> []
         multiple-values-empty?
         (conj [:db/retract (:db/id update-block-tx) property-id :logseq.property/empty-placeholder])
@@ -108,7 +111,8 @@
   (when (and (some? property-name) (not= property-name (:block/title property)))
     (outliner-validate/validate-page-title property-name {:node property})
     (outliner-validate/validate-page-title-characters property-name {:node property})
-    (outliner-validate/validate-block-title @conn property-name property))
+    (outliner-validate/validate-block-title @conn property-name property)
+    (outliner-validate/validate-property-title property-name))
 
   (let [changed-property-attrs
         ;; Only update property if something has changed as we are updating a timestamp
@@ -407,7 +411,7 @@
   "Updates property if property-id is given. Otherwise creates a property
    with the given property-id or :property-name option. When a property is created
    it is ensured to have a unique :db/ident"
-  [conn property-id schema {:keys [property-name] :as opts}]
+  [conn property-id schema {:keys [property-name properties] :as opts}]
   (let [db @conn
         db-ident (or property-id
                      (try (db-property/create-user-property-ident-from-name property-name)
@@ -426,9 +430,18 @@
                 (prn "property-id: " property-id ", property-name: " property-name))
         (outliner-validate/validate-page-title k-name {:node {:db/ident db-ident'}})
         (outliner-validate/validate-page-title-characters k-name {:node {:db/ident db-ident'}})
-        (ldb/transact! conn
-                       [(sqlite-util/build-new-property db-ident' schema {:title k-name})]
-                       {:outliner-op :new-property})
+        (let [db-id (:db/id properties)
+              opts (cond-> {:title k-name
+                            :properties properties}
+                     (integer? db-id)
+                     (assoc :block-uuid (:block/uuid (d/entity db db-id))))]
+          (ldb/transact! conn
+                         (concat
+                          [(sqlite-util/build-new-property db-ident' schema opts)]
+                          ;; Convert page to property
+                          (when db-id
+                            [[:db/retract db-id :block/tags :logseq.class/Page]]))
+                         {:outliner-op :upsert-property}))
         (d/entity @conn db-ident')))))
 
 (defn delete-property-value!

deps/outliner/src/logseq/outliner/validate.cljs

@@ -8,7 +8,8 @@
             [logseq.common.util.namespace :as ns-util]
             [logseq.db :as ldb]
             [logseq.db.frontend.class :as db-class]
-            [logseq.db.frontend.entity-util :as entity-util]))
+            [logseq.db.frontend.entity-util :as entity-util]
+            [logseq.db.frontend.property :as db-property]))
 
 (defn ^:api validate-page-title-characters
   "Validates characters that must not be in a page title"
@@ -138,12 +139,21 @@
                                :type :warning}}))))
 
 (defn validate-block-title
-  "Validates a block title when it has changed"
+  "Validates a block title when it has changed for a entity-util/page? or tagged node"
   [db new-title existing-block-entity]
   (validate-built-in-pages existing-block-entity)
   (validate-unique-by-name-tag-and-block-type db new-title existing-block-entity)
   (validate-disallow-page-with-journal-name new-title existing-block-entity))
 
+(defn validate-property-title
+  "Validates a property's title when it has changed"
+  [new-title]
+  (when-not (db-property/valid-property-name? new-title)
+    (throw (ex-info "Property name is invalid"
+                    {:type :notification
+                     :payload {:message "This is an invalid property name. A property name cannot start with page reference characters '#' or '[['."
+                               :type :error}}))))
+
 (defn- validate-extends-property-have-correct-type
   "Validates whether given parent and children are classes"
   [parent-ent child-ents]

deps/outliner/yarn.lock

@@ -2,9 +2,9 @@
 # yarn lockfile v1
 
 
-"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v23":
-  version "1.2.173-feat-db-v23"
-  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/810782c4ddba6346c4ab8ae6740b60438c07cd01"
+"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v24":
+  version "1.2.173-feat-db-v24"
+  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/3d5b78b0382c7253bf9874c1f38586dd338434f4"
   dependencies:
     import-meta-resolve "^4.1.0"
 

deps/publishing/nbb.edn

@@ -3,4 +3,4 @@
  {logseq/db
   {:local/root "../db"}
   io.github.nextjournal/nbb-test-runner
-  {:git/sha "012017a8a8983d05f905f38f631f5222f25b9ed9"}}}
+  {:git/sha "b379325cfa5a3306180649da5de3bf5166414e71"}}}

deps/publishing/package.json

@@ -3,7 +3,7 @@
   "version": "1.0.0",
   "private": true,
   "devDependencies": {
-    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v23",
+    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v24",
     "mldoc": "^1.5.9"
   },
   "dependencies": {

deps/publishing/src/logseq/publishing/html.cljs

@@ -2,10 +2,10 @@
   "This frontend only ns builds the publishing html including doing all the
 necessary db filtering"
   (:require [clojure.string :as string]
+            [datascript.core :as d]
+            [datascript.transit :as dt]
             [goog.string :as gstring]
             [goog.string.format]
-            [datascript.transit :as dt]
-            [datascript.core :as d]
             [logseq.publishing.db :as db]))
 
 ;; Copied from hiccup but tweaked for publish usage
@@ -126,12 +126,15 @@ necessary db filtering"
             [:script {:src "static/js/ui.js"}]
             [:script {:src "static/js/shared.js"}]
             [:script {:src "static/js/main.js"}]
-            [:script {:src "static/js/interact.min.js"}]
-            [:script {:src "static/js/highlight.min.js"}]
-            [:script {:src "static/js/katex.min.js"}]
-            [:script {:src "static/js/html2canvas.min.js"}]
-            [:script {:src "static/js/code-editor.js"}]
-            [:script {:src "static/js/custom.js"}]])))))
+            ;; Deferring scripts above results in errors
+            [:script {:defer true :src "static/js/interact.min.js"}]
+            [:script {:defer true :src "static/js/highlight.min.js"}]
+            [:script {:defer true :src "static/js/katex.min.js"}]
+            [:script {:defer true :type "module" :src "static/js/pdfjs/pdf.mjs"}]
+            [:script {:defer true :type "module" :src "static/js/pdf_viewer3.mjs"}]
+            [:script {:defer true :src "static/js/html2canvas.min.js"}]
+            [:script {:defer true :src "static/js/code-editor.js"}]
+            [:script {:defer true :src "static/js/custom.js"}]])))))
 
 (defn build-html
   "Given the graph's db, filters the db using the given options and returns the

deps/publishing/yarn.lock

@@ -2,9 +2,9 @@
 # yarn lockfile v1
 
 
-"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v23":
-  version "1.2.173-feat-db-v23"
-  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/810782c4ddba6346c4ab8ae6740b60438c07cd01"
+"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v24":
+  version "1.2.173-feat-db-v24"
+  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/3d5b78b0382c7253bf9874c1f38586dd338434f4"
   dependencies:
     import-meta-resolve "^4.1.0"
 

docs/develop-logseq.md

@@ -40,7 +40,6 @@ Open a dev environment (Browser dev app on ``localhost:3000`` or Desktop dev app
 ``cmd + shift + p`` -> ``Calva: Load/Evaluate Current File and its Requires/Dependencies``
 
 #### Connect to the web-worker context
-Notice: this works only for the `feat/db` branch for now.
 
 ##### Emacs + Cider
 When connecting to a CLJ nrepl (NOTE: if you are already in a CLJS nrepl, use `:cljs/quit` to go back to CLJ nrepl),

gulpfile.js

@@ -85,9 +85,9 @@ const common = {
         'node_modules/@glidejs/glide/dist/css/glide.theme.min.css',
       ]).pipe(gulp.dest(path.join(outputPath, 'js', 'glide'))),
       () => gulp.src([
-        'node_modules/pdfjs-dist/build/pdf.js',
-        'node_modules/pdfjs-dist/build/pdf.worker.js',
-        'node_modules/pdfjs-dist/web/pdf_viewer.js',
+        'node_modules/pdfjs-dist/legacy/build/pdf.mjs',
+        'node_modules/pdfjs-dist/legacy/build/pdf.worker.mjs',
+        'node_modules/pdfjs-dist/legacy/web/pdf_viewer.mjs',
       ]).pipe(gulp.dest(path.join(outputPath, 'js', 'pdfjs'))),
       () => gulp.src([
         'node_modules/pdfjs-dist/cmaps/*.*',
@@ -170,17 +170,21 @@ const common = {
     cb()
   },
 
-  switchReactDevelopmentMode (cb) {
-    const reactFrom = path.join(outputPath, 'js', 'react.development.js')
-    const reactTo = path.join(outputPath, 'js', 'react.production.min.js')
-    cp.execSync(`mv ${reactFrom} ${reactTo}`, { stdio: 'inherit' })
+  switchReactDevelopmentMode(cb) {
+    try {
+      const reactFrom = path.join(outputPath, 'js', 'react.development.js');
+      const reactTo = path.join(outputPath, 'js', 'react.production.min.js');
+      fs.renameSync(reactFrom, reactTo);
 
-    const reactDomFrom = path.join(outputPath, 'js', 'react-dom.development.js')
-    const reactDomTo = path.join(outputPath, 'js',
-      'react-dom.production.min.js')
-    cp.execSync(`mv ${reactDomFrom} ${reactDomTo}`, { stdio: 'inherit' })
+      const reactDomFrom = path.join(outputPath, 'js', 'react-dom.development.js');
+      const reactDomTo = path.join(outputPath, 'js', 'react-dom.production.min.js');
+      fs.renameSync(reactDomFrom, reactDomTo);
 
-    cb()
+      cb();
+    } catch (err) {
+      console.error("Error during switchReactDevelopmentMode:", err);
+      cb(err);
+    }
   },
 }
 

package.json

@@ -4,7 +4,7 @@
     "private": true,
     "main": "static/electron.js",
     "engines" : {
-      "node" : ">=22.15.0"
+      "node" : ">=20.19.1"
     },
     "devDependencies": {
         "@axe-core/playwright": "=4.4.4",
@@ -133,7 +133,7 @@
         "d3-force": "3.0.0",
         "diff": "5.0.0",
         "dompurify": "2.4.0",
-        "electron": "36.2.0",
+        "electron": "34.5.6",
         "electron-dl": "^4.0.0",
         "emoji-mart": "^5.5.2",
         "fs": "0.0.1-security",
@@ -151,7 +151,7 @@
         "mldoc": "^1.5.9",
         "path": "0.12.7",
         "path-complete-extname": "1.0.0",
-        "pdfjs-dist": "^3.9.179",
+        "pdfjs-dist": "4.2.67",
         "photoswipe": "^5.3.7",
         "pixi-graph-fork": "0.2.0",
         "pixi.js": "6.2.0",

packages/tldraw/apps/tldraw-logseq/src/lib/logseq-context.ts

@@ -55,7 +55,6 @@ export interface LogseqContextValue {
     queryBlockByUUID: (uuid: string) => any
     getBlockPageName: (uuid: string) => string
     getRedirectPageName: (uuidOrPageName: string) => string
-    insertFirstPageBlock: (pageName: string) => string
     isWhiteboardPage: (pageName: string) => boolean
     isMobile: () => boolean
     saveAsset: (file: File) => Promise<string>

public/index.html

@@ -50,6 +50,8 @@
 <script defer src="/static/js/highlight.min.js"></script>
 <script defer src="/static/js/interact.min.js"></script>
 <script defer src="/static/js/marked.min.js"></script>
+<script defer type="module" src="/static/js/pdfjs/pdf.mjs"></script>
+<script defer type="module" src="/static/js/pdf_viewer3.mjs"></script>
 <script defer src="/static/js/eventemitter3.umd.min.js"></script>
 <script defer src="/static/js/html2canvas.min.js"></script>
 <script defer src="/static/js/lsplugin.core.js"></script>

resources/css/shui.css

@@ -246,7 +246,7 @@ div[data-radix-popper-content-wrapper] {
 }
 
 .ui__dialog-content {
-  @apply outline-none;
+  @apply outline-none m-4;
 
   &[data-auto-width] {
     @apply max-w-[90vw] w-max sm:max-w-[960px];

resources/forge.config.js

@@ -5,7 +5,8 @@ module.exports = {
   packagerConfig: {
     name: 'Logseq',
     icon: './icons/logseq_big_sur.icns',
-    buildVersion: '87',
+    buildVersion: "87",
+    appBundleId: "com.logseq.logseq",
     protocols: [
       {
         "protocol": "logseq",