Merge branch 'master' into feat/worker-sync

.carve/ignore

@@ -13,8 +13,6 @@ frontend.debug/print
 frontend.extensions.code/editor
 ;; Referenced in commented TODO
 frontend.extensions.pdf.utils/get-page-bounding
-;; For repl
-logseq.graph-parser.mldoc/ast-export-markdown
 ;; Protocol fn wrapper that could be used
 frontend.fs/readdir
 ;; Referenced in TODO
@@ -46,13 +44,9 @@ frontend.components.repo/add-repo
 frontend.util/trace!
 ;; Repl fn
 frontend.util.pool/terminate-pool!
-;; Repl fn
-frontend.handler.file-based.property.util/add-page-properties
 ;; Test runners used by shadow
 frontend.test.node-test-runner/main
 frontend.test.frontend-node-test-runner/main
-;; Test runner for nbb
-logseq.graph-parser.nbb-test-runner/run-tests
 ;; For debugging
 frontend.fs.sync/debug-print-sync-events-loop
 frontend.fs.sync/stop-debug-print-sync-events-loop
@@ -76,4 +70,4 @@ frontend.common.missionary/<!
 frontend.common.missionary/background-task-running?
 ;; defonce
 mobile.components.popup/native-sheet-listener
-mobile.bottom-tabs/add-tab-listeners!
+mobile.bottom-tabs/add-tab-listeners!

.clj-kondo/config.edn

@@ -70,9 +70,7 @@
              electron.ipc ipc
              electron.utils utils
              frontend.commands commands
-             frontend.common.file-based.db common-file-db
              frontend.common.date common-date
-             frontend.common.file.util wfu
              frontend.common.missionary-util c.m
              frontend.common.schema-register sr
              frontend.common.search-fuzzy fuzzy
@@ -85,7 +83,6 @@
              frontend.config config
              frontend.date date
              frontend.db db
-             frontend.db.file-based.model file-model
              frontend.db-mixins db-mixins
              frontend.db.query-custom query-custom
              frontend.db.query-dsl query-dsl
@@ -112,15 +109,6 @@
              frontend.handler.editor.property editor-property
              frontend.handler.events events
              frontend.handler.extract extract
-             frontend.handler.file-based.file file-handler
-             frontend.handler.file-based.native-fs nfs-handler
-             frontend.handler.file-based.page file-page-handler
-             frontend.handler.file-based.page-property file-page-property
-             frontend.handler.file-based.property file-property-handler
-             frontend.handler.file-based.property.util property-util
-             frontend.handler.file-based.recent file-recent-handler
-             frontend.handler.file-based.reset-file reset-file-handler
-             frontend.handler.file-based.repo file-repo-handler
              frontend.handler.global-config global-config-handler
              frontend.handler.notification notification
              frontend.handler.page page-handler
@@ -149,8 +137,6 @@
              frontend.test.helper test-helper
              frontend.ui ui
              frontend.util util
-             frontend.util.file-based.clock clock
-             frontend.util.file-based.drawer drawer
              frontend.util.page page-util
              frontend.util.persist-var persist-var
              frontend.util.property property
@@ -183,15 +169,10 @@
              logseq.common.util.page-ref page-ref
              logseq.db ldb
              logseq.db.common.entity-plus entity-plus
-             logseq.db.common.entity-util common-entity-util
              logseq.db.common.initial-data common-initial-data
              logseq.db.common.order db-order
-             logseq.db.common.property-util db-property-util
              logseq.db.common.sqlite common-sqlite
              logseq.db.common.view db-view
-             logseq.db.file-based.rules file-rules
-             logseq.db.file-based.schema file-schema
-             logseq.db.file-based.entity-util file-entity-util
              logseq.db.frontend.class db-class
              logseq.db.frontend.content db-content
              logseq.db.frontend.db db-db
@@ -210,9 +191,7 @@
              logseq.db.sqlite.export sqlite-export
              logseq.db.sqlite.util sqlite-util
              logseq.db.test.helper db-test
-             logseq.graph-parser graph-parser
              logseq.graph-parser.block gp-block
-             logseq.graph-parser.db gp-db
              logseq.graph-parser.mldoc gp-mldoc
              logseq.graph-parser.property gp-property
              logseq.graph-parser.text text

.github/workflows/build.yml

@@ -78,11 +78,8 @@ jobs:
       - name: Build test asset
         run: clojure -M:test compile test
 
-      - name: Run some ClojureScript tests against DB version
-        run: DB_GRAPH=1 node static/tests.js -r frontend.db.query-dsl-test
-
-      - name: Run ClojureScript query tests against DB version with basic query type
-        run: DB_GRAPH=1 DB_QUERY_TYPE=basic node static/tests.js -r frontend.db.query-dsl-test
+      - name: Run ClojureScript query tests against basic query type
+        run: DB_QUERY_TYPE=basic node static/tests.js -r frontend.db.query-dsl-test
 
       - name: Run ClojureScript tests
         run: node static/tests.js -e fix-me
@@ -124,9 +121,6 @@ jobs:
       - name: Lint to keep worker independent of frontend
         run: bb lint:worker-and-frontend-separate
 
-      - name: Lint to keep db and file graph code separate
-        run: bb lint:db-and-file-graphs-separate
-
   db-graph-test:
     strategy:
       matrix:

.github/workflows/deploy-and-branch.yml

@@ -62,25 +62,11 @@ jobs:
             --exclude mobile \
             ./static/ ./public/
 
-          mkdir -p r2
-          mv ./public/js/main.js.map ./r2/${{ inputs.branch }}.main.js.map
+          rm ./public/js/*.map
 
-          sed -i "s|=main.js.map|=https://assets.logseq.io/${{ inputs.branch }}.main.js.map|g" \
-            ./public/js/main.js
         env:
-          LOGSEQ_SENTRY_DSN: ${{ secrets.LOGSEQ_SENTRY_DSN }}
           LOGSEQ_POSTHOG_TOKEN: ${{ secrets.LOGSEQ_POSTHOG_TOKEN }}
 
-      - name: Upload sourcemaps to R2
-        uses: ryand56/r2-upload-action@latest
-        with:
-          r2-account-id: 2553ea8236c11ea0f88de28fce1cbfee
-          r2-access-key-id: ${{ secrets.R2_ACCESS_KEY_ID }}
-          r2-secret-access-key: ${{ secrets.R2_SECRET_ACCESS_KEY }}
-          r2-bucket: ${{ secrets.R2_ASSETS_BUCKET }}
-          source-dir: r2
-          destination-dir: ./
-
       - name: Publish to Cloudflare Pages
         uses: cloudflare/pages-action@1
         with:

.github/workflows/cli.yml → .github/workflows/deps-cli.yml

@@ -7,7 +7,7 @@ on:
     branches: [master]
     paths:
       - 'deps/cli/**'
-      - '.github/workflows/cli.yml'
+      - '.github/workflows/deps-cli.yml'
       - '!deps/cli/**.md'
       # Deps that logseq/cli depends on should trigger this workflow
       - 'deps/outliner/**'
@@ -18,7 +18,7 @@ on:
     branches: [master]
     paths:
       - 'deps/cli/**'
-      - '.github/workflows/cli.yml'
+      - '.github/workflows/deps-cli.yml'
       - '!deps/cli/**.md'
       # Deps that logseq/cli depends on should trigger this workflow
       - 'deps/outliner/**'

.github/workflows/logseq-common.yml → .github/workflows/deps-common.yml

@@ -6,13 +6,13 @@ on:
     branches: [master]
     paths:
       - 'deps/common/**'
-      - '.github/workflows/logseq-common.yml'
+      - '.github/workflows/deps-common.yml'
       - '!deps/common/**.md'
   pull_request:
     branches: [master]
     paths:
       - 'deps/common/**'
-      - '.github/workflows/logseq-common.yml'
+      - '.github/workflows/deps-common.yml'
       - '!deps/common/**.md'
 
 defaults:

.github/workflows/db.yml → .github/workflows/deps-db.yml

@@ -6,7 +6,7 @@ on:
     branches: [master]
     paths:
       - 'deps/db/**'
-      - '.github/workflows/db.yml'
+      - '.github/workflows/deps-db.yml'
       - '!deps/db/**.md'
       # Deps that logseq/db depends on should trigger this workflow
       - 'deps/common/**'
@@ -14,7 +14,7 @@ on:
     branches: [master]
     paths:
       - 'deps/db/**'
-      - '.github/workflows/db.yml'
+      - '.github/workflows/deps-db.yml'
       - '!deps/db/**.md'
       # Deps that logseq/db depends on should trigger this workflow
       - 'deps/common/**'

.github/workflows/graph-parser.yml → .github/workflows/deps-graph-parser.yml

@@ -7,7 +7,7 @@ on:
     branches: [master]
     paths:
       - 'deps/graph-parser/**'
-      - '.github/workflows/graph-parser.yml'
+      - '.github/workflows/deps-graph-parser.yml'
       - '!deps/graph-parser/**.md'
       # Deps that logseq/graph-parser depends on should trigger this workflow
       - 'deps/db/**'
@@ -16,7 +16,7 @@ on:
     branches: [master]
     paths:
       - 'deps/graph-parser/**'
-      - '.github/workflows/graph-parser.yml'
+      - '.github/workflows/deps-graph-parser.yml'
       - '!deps/graph-parser/**.md'
       # Deps that logseq/graph-parser depends on should trigger this workflow
       - 'deps/db/**'
@@ -79,7 +79,7 @@ jobs:
         run: yarn install --frozen-lockfile
 
       - name: Run ClojureScript tests
-        run: REPEATABLE_IDENTS=true clojure -M:test
+        run: clojure -M:test
 
       - name: Run nbb-logseq tests
         run: yarn test

.github/workflows/outliner.yml → .github/workflows/deps-outliner.yml

@@ -7,7 +7,7 @@ on:
     branches: [master]
     paths:
       - 'deps/outliner/**'
-      - '.github/workflows/outliner.yml'
+      - '.github/workflows/deps-outliner.yml'
       - '!deps/outliner/**.md'
       # Deps that logseq/outliner depends on should trigger this workflow
       - 'deps/graph-parser/**'
@@ -17,7 +17,7 @@ on:
     branches: [master]
     paths:
       - 'deps/outliner/**'
-      - '.github/workflows/outliner.yml'
+      - '.github/workflows/deps-outliner.yml'
       - '!deps/outliner/**.md'
       # Deps that logseq/outliner depends on should trigger this workflow
       - 'deps/graph-parser/**'

.github/workflows/deps-publish.yml

@@ -0,0 +1,101 @@
+name: logseq/publish CI
+
+on:
+  # Path filters ensure jobs only kick off if a change is made to publish or
+  # its local dependencies
+  push:
+    branches: [master]
+    paths:
+      - 'deps/publish/**'
+      - '.github/workflows/deps-publish.yml'
+      - '!deps/publish/**.md'
+      # Deps that logseq/publish depends on should trigger this workflow
+      - 'deps/graph-parser/**'
+      - 'deps/db/**'
+      - 'deps/common/**'
+  pull_request:
+    branches: [master]
+    paths:
+      - 'deps/publish/**'
+      - '.github/workflows/deps-publish.yml'
+      - '!deps/publish/**.md'
+      # Deps that logseq/publish depends on should trigger this workflow
+      - 'deps/graph-parser/**'
+      - 'deps/db/**'
+      - 'deps/common/**'
+
+defaults:
+  run:
+    working-directory: deps/publish
+
+env:
+  CLOJURE_VERSION: '1.11.1.1413'
+  JAVA_VERSION: '21'
+  # This is the latest node version we can run.
+  NODE_VERSION: '22'
+  BABASHKA_VERSION: '1.0.168'
+
+jobs:
+  test-release:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+          cache: 'yarn'
+          cache-dependency-path: deps/publish/yarn.lock
+
+      - name: Set up Java
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'zulu'
+          java-version: ${{ env.JAVA_VERSION }}
+
+      # Clojure needed for bb step
+      - name: Set up Clojure
+        uses: DeLaGuardo/[email protected]
+        with:
+          cli: ${{ env.CLOJURE_VERSION }}
+          bb: ${{ env.BABASHKA_VERSION }}
+
+      - name: Fetch yarn deps
+        run: yarn install --frozen-lockfile
+
+      - name: Build release asset
+        run: yarn release
+
+  lint:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Java
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'zulu'
+          java-version: ${{ env.JAVA_VERSION }}
+
+      - name: Set up Clojure
+        uses: DeLaGuardo/[email protected]
+        with:
+          cli: ${{ env.CLOJURE_VERSION }}
+          bb: ${{ env.BABASHKA_VERSION }}
+
+      - name: Run clj-kondo lint
+        run: clojure -M:clj-kondo --lint src
+
+      - name: Carve lint for unused vars
+        run: bb lint:carve
+
+      - name: Lint for vars that are too large
+        run: bb lint:large-vars
+
+      - name: Lint for namespaces that aren't documented
+        run: bb lint:ns-docstrings

.github/workflows/publishing.yml → .github/workflows/deps-publishing.yml

@@ -7,7 +7,7 @@ on:
     branches: [master]
     paths:
       - 'deps/publishing/**'
-      - '.github/workflows/publishing.yml'
+      - '.github/workflows/deps-publishing.yml'
       - '!deps/publishing/**.md'
       # Deps that logseq/publishing depends on should trigger this workflow
       - 'deps/db/**'
@@ -16,7 +16,7 @@ on:
     branches: [master]
     paths:
       - 'deps/publishing/**'
-      - '.github/workflows/publishing.yml'
+      - '.github/workflows/deps-publishing.yml'
       - '!deps/publishing/**.md'
       # Deps that logseq/publishing depends on should trigger this workflow
       - 'deps/db/**'

.gitignore

@@ -52,7 +52,6 @@ ios/App/App/public
 
 startup.png
 
-/src/main/frontend/tldraw-logseq.js
 /src/test/docs*
 ~*~
 

.projectile

@@ -12,7 +12,6 @@
 -/resources/static/js/mhchem.min.js
 -/resources/static/js/mldoc.min.js
 -/resources/static/js/sci.min.js
--/resources/static/js/excalidraw.min.js
 -/resources/static/js/react-force-graph.min.js
 -/resources/js/lsplugin.user.js
 -/resources/js/pdf_viewer2.js

CODEBASE_OVERVIEW.md

@@ -60,7 +60,6 @@ This is overview of this repository's most important directories and files.
 
 - `packages/` contains JavaScript dependencies used by the frontend
   - `packages/ui/` - The frontend's component system based on shadcn
-  - `packags/tldraw/` - Custom fork of tldraw which powers whiteboards
 - `scripts` - Dev scripts
 - `clj-e2e/` - end to end clj frontend tests
 - `android/` -  Android app

README.md

@@ -83,8 +83,6 @@ To get started with the DB version:
 
 [Logseq](https://logseq.com) is a **knowledge management** and **collaboration** platform. It focuses on **privacy**, **longevity**, and [**user control**](https://www.gnu.org/philosophy/free-sw.en.html). Logseq offers a range of **powerful tools** for **knowledge management**, **collaboration**, **PDF annotation**, and **task management** with support for multiple file formats, including **Markdown** and **Org-mode**, and **various features** for organizing and structuring your notes.
 
-Logseq's **Whiteboard** feature lets you organize your knowledge and ideas using a spatial **canvas** with **shapes**, **drawings**, **website embeds**, and **connectors**. You can **visually group** and **link** your **notes** and external media (such as **videos** and **images**), enabling visual thinkers to compose, remix, **annotate**, and connect content from their knowledge base and emerging thoughts in a new way.
-
 In addition to its core features, Logseq has a growing ecosystem of **plugins** and **themes** that enable a wide range of workflows and **customization** options. **Mobile apps** are also available, providing access to most of the features of the desktop application. Whether you're a student, a professional, or anyone who values a clear and organized approach to managing your ideas and notes, Logseq is an excellent choice for anyone looking to improve their productivity and streamline their workflow.
 
 ![logseq-demo](https://user-images.githubusercontent.com/25513724/221387376-4dc419c2-0d0a-460c-a920-2d211e78b456.gif)

bb.edn

@@ -71,41 +71,41 @@
   {:doc "Run CLI with current deps/db code. Commands with JS deps are not usable e.g. mcp-server"
    :task (apply shell {:dir "deps/db"}
                 "yarn nbb-logseq -cp src:../cli/src:../graph-parser/src:../outliner/src -m logseq.cli" *command-line-args*)}
-  dev:db-query
+  dev:query
   {:doc "Query a DB graph's datascript db"
    :requires ([babashka.fs :as fs])
    :task (apply shell {:dir "deps/db" :extra-env {"ORIGINAL_PWD" (fs/cwd)}}
                 "yarn -s nbb-logseq script/query.cljs" *command-line-args*)}
 
-  dev:db-transact
+  dev:transact
   {:doc "Transact against a DB graph's datascript db"
    :requires ([babashka.fs :as fs])
    :task (apply shell {:dir "deps/outliner" :extra-env {"ORIGINAL_PWD" (fs/cwd)}}
                 "yarn -s nbb-logseq script/transact.cljs" *command-line-args*)}
 
-  dev:db-create
+  dev:create
   {:doc "Create a DB graph given a sqlite.build EDN file"
    :requires ([babashka.fs :as fs])
    :task (apply shell {:dir "deps/db" :extra-env {"ORIGINAL_PWD" (fs/cwd)}}
                 "yarn -s nbb-logseq -cp src:../outliner/src:script script/create_graph.cljs" *command-line-args*)}
 
-  dev:db-diff
+  dev:diff-graphs
   {:doc "Diffs two DB graphs"
    :requires ([babashka.fs :as fs])
    :task (apply shell {:dir "deps/db" :extra-env {"ORIGINAL_PWD" (fs/cwd)}}
                 "yarn -s nbb-logseq script/diff_graphs.cljs" *command-line-args*)}
 
-  dev:db-import
+  dev:import
   {:doc "Import a file graph to db graph"
    :requires ([babashka.fs :as fs])
    :task (apply shell {:dir "deps/graph-parser" :extra-env {"ORIGINAL_PWD" (fs/cwd)}}
                 "yarn -s nbb-logseq -cp src:../outliner/src script/db_import.cljs" *command-line-args*)}
 
-  dev:db-import-many
+  dev:import-many
   {:doc "Import multiple file graphs to db graphs"
    :task logseq.tasks.dev/db-import-many}
 
-  dev:db-datoms
+  dev:datoms
   {:doc "Write db's datoms to a file"
    :requires ([babashka.fs :as fs])
    :task (apply shell {:dir "deps/db" :extra-env {"ORIGINAL_PWD" (fs/cwd)}}
@@ -203,9 +203,6 @@
   lint:ns-docstrings
   logseq.bb-tasks.lint.ns-docstrings/-main
 
-  lint:db-and-file-graphs-separate
-  logseq.tasks.dev.db-and-file-graphs/-main
-
   lint:worker-and-frontend-separate
   logseq.tasks.dev.lint/worker-and-frontend-separate
 
@@ -224,9 +221,6 @@
   lang:validate-translations
   logseq.tasks.lang/validate-translations
 
-  file-sync:integration-tests
-  logseq.tasks.file-sync/integration-tests
-
   ai:check-common-errors
   logseq.tasks.common-errors/check-common-errors}
 

clj-e2e/dev/user.clj

@@ -1,6 +1,7 @@
 (ns user
   "fns used on repl"
   (:require [clojure.test :refer [run-tests run-test]]
+            [logseq.e2e.bidirectional-properties-test]
             [logseq.e2e.block :as b]
             [logseq.e2e.commands-basic-test]
             [logseq.e2e.config :as config]
@@ -57,6 +58,11 @@
   (->> (future (run-tests 'logseq.e2e.property-scoped-choices-test))
        (swap! *futures assoc :property-scoped-choices-test)))
 
+(defn run-bidirectional-properties-test
+  []
+  (->> (future (run-tests 'logseq.e2e.bidirectional-properties-test))
+       (swap! *futures assoc :bidirectional-properties-test)))
+
 (defn run-outliner-test
   []
   (->> (future (run-tests 'logseq.e2e.outliner-basic-test))

clj-e2e/test/logseq/e2e/bidirectional_properties_test.clj

@@ -0,0 +1,51 @@
+(ns logseq.e2e.bidirectional-properties-test
+  (:require [clojure.test :refer [deftest is testing use-fixtures]]
+            [logseq.e2e.api :refer [ls-api-call!]]
+            [logseq.e2e.assert :as assert]
+            [logseq.e2e.fixtures :as fixtures]
+            [logseq.e2e.page :as page]
+            [wally.main :as w]))
+
+(use-fixtures :once fixtures/open-page)
+
+(use-fixtures :each
+  fixtures/new-logseq-page
+  fixtures/validate-graph)
+
+(deftest bidirectional-properties-test
+  (testing "shows reverse property references when a class enables bidirectional properties"
+    (let [friend-prop "friend"
+          person-tag "Person"
+          project-tag "Project"
+          target "Bob"
+          container-page "Bidirectional Props"]
+      (ls-api-call! :editor.createTag person-tag
+                    {:tagProperties [{:name friend-prop
+                                      :schema {:type "node"}}]})
+      (ls-api-call! :editor.createTag project-tag)
+      (let [person (ls-api-call! :editor.getTag person-tag)
+            person-uuid (get person "uuid")
+            friend (ls-api-call! :editor.getPage friend-prop)]
+        (ls-api-call! :editor.upsertBlockProperty (get friend "id")
+                      "logseq.property/classes"
+                      (get person "id"))
+        (is (string? person-uuid))
+        (ls-api-call! :editor.upsertBlockProperty person-uuid
+                      "logseq.property.class/bidirectional-property-title"
+                      "People")
+        (ls-api-call! :editor.upsertBlockProperty person-uuid
+                      "logseq.property.class/enable-bidirectional?"
+                      true))
+      (ls-api-call! :editor.createPage target)
+      (ls-api-call! :editor.createPage container-page)
+      (let [bob (ls-api-call! :editor.getPage target)
+            bob-id (get bob "id")]
+        (ls-api-call! :editor.insertBlock container-page (str "Alice #" person-tag)
+                      {:properties {friend-prop bob-id}})
+        (ls-api-call! :editor.insertBlock container-page (str "Charlie #" project-tag)
+                      {:properties {friend-prop bob-id}}))
+
+      (page/goto-page target)
+      (w/wait-for ".property-k:text('People')")
+      (assert/assert-is-visible ".property-value .block-title-wrap:text('Alice')")
+      (assert/assert-have-count ".property-k:text('Projects')" 0))))

clj-e2e/test/logseq/e2e/plugins_basic_test.clj

@@ -56,9 +56,10 @@
           props1 (ls-api-call! :editor.getBlockProperties uuid' "p1")
           props2 (ls-api-call! :editor.getPageProperties "test-block-properties-apis")]
       (w/wait-for ".property-k:text('p1')")
-      (is (= 1 (get prop1 "value")))
-      (is (= (get prop1 "ident") ":plugin.property._test_plugin/p1"))
-      (is (= 1 (get props1 ":plugin.property._test_plugin/p1")))
+      ;; FIXME: Assertions below fail
+      ;; (is (= 1 (get prop1 "value")))
+      ;; (is (= (get prop1 "ident") ":plugin.property._test_plugin/p1"))
+      ;; (is (= 1 (get props1 ":plugin.property._test_plugin/p1")))
       (is (= ["Page"] (get props2 ":block/tags")))
       (ls-api-call! :editor.upsertBlockProperty uuid' "p2" "p2")
       (ls-api-call! :editor.upsertBlockProperty uuid' "p3" true)
@@ -376,3 +377,15 @@
       ;; Verify the result contains valid tag structure
       (is (string? (get (first result) "uuid")) "returned tag should have uuid")
       (is (string? (get (first result) "title")) "returned tag should have title"))))
+
+(deftest set-property-node-tags
+  (testing "set property node tags"
+    (let [property-name (new-property)
+          _ (ls-api-call! :editor.upsertProperty property-name {:type "node"})
+          tag1 (ls-api-call! :editor.createTag "Tag A")
+          tag2 (ls-api-call! :editor.createTag "Tag B")
+          tags [(get tag1 "id") (get tag2 "id")]
+          _ (ls-api-call! :editor.setPropertyNodeTags property-name tags)
+          property (ls-api-call! :editor.getProperty property-name)
+          node-tags (get property ":logseq.property/classes")]
+      (is (= (set node-tags) (set tags)) "property node tags should match the set tags"))))

deps/cli/README.md

@@ -37,16 +37,11 @@ validate [options]   Validate DB graph
 help                 Print a command's help
 
 $ logseq list
-DB Graphs:
 db-test
 docs
 woot
 ...
 
-File Graphs:
-docs
-...
-
 $ logseq show db-test
 
 |                         Name |                                              Value |
@@ -65,8 +60,8 @@ To run a command against the current desktop graph, set `$LOGSEQ_API_SERVER_TOKE
 # Search your current graph and print highlighted results one per line like grep
 $ logseq search woot -a my-token
 Search found 100 results:
-dev:db-export woot woot.edn && dev:db-create woot2 woot.edn
-dev:db-diff woot woot2
+dev:db-export woot woot.edn && dev:create woot2 woot.edn
+dev:diff-graphs woot woot2
 ...
 # Can also authenticate api with $LOGSEQ_API_SERVER_TOKEN
 $ export LOGSEQ_API_SERVER_TOKEN=my-token
@@ -167,7 +162,7 @@ Most of this library is also compatible with ClojureScript for use on the
 frontend. This library follows the practices that [the Logseq frontend
 follows](/docs/dev-practices.md). Most of the same linters are used, with
 configurations that are specific to this library. See [this library's CI
-file](/.github/workflows/cli.yml) for linting examples.
+file](/.github/workflows/deps-cli.yml) for linting examples.
 
 ### Setup
 

deps/cli/src/logseq/cli/commands/export.cljs

@@ -9,7 +9,6 @@
             [logseq.cli.common.file :as common-file]
             [logseq.cli.common.util :as cli-common-util]
             [logseq.cli.util :as cli-util]
-            [logseq.common.config :as common-config]
             [logseq.common.util :as common-util]
             [logseq.db.common.sqlite-cli :as sqlite-cli]))
 
@@ -34,8 +33,8 @@
 
 (defn- get-file-contents
   "Modified version of export.common/<get-file-contents which doesn't have to deal with worker threads"
-  [repo db content-config suffix]
-  (let [page->content (common-file/get-all-page->content repo db content-config)]
+  [db content-config suffix]
+  (let [page->content (common-file/get-all-page->content db content-config)]
     (map (fn [[page-title content]]
            {:path (str page-title "." suffix)
             :content content
@@ -45,26 +44,24 @@
 
 (defn- export-files-as-markdown
   "Modified version of handler.export.text/export-files-as-markdown for the CLI"
-  [repo files options]
+  [files options]
   (mapv
    (fn [{:keys [path title content]}]
-     [(or path title) (cli-export-text/export-helper repo content :markdown options)])
+     [(or path title) (cli-export-text/export-helper content :markdown options)])
    files))
 
 (defn- export-repo-as-markdown!
   "Modified version of handler.export.text/export-repo-as-markdown for the CLI"
-  [repo db {:keys [file]}]
+  [graph db {:keys [file]}]
   (let [content-config (get-content-config db)
-        files* (get-file-contents repo db content-config "md")]
+        files* (get-file-contents db content-config "md")]
     (when (seq files*)
       (let [files (binding [cli-export-common/*current-db* db
-                            cli-export-common/*current-repo* repo
                             cli-export-common/*content-config* content-config]
-                    (export-files-as-markdown repo files* nil))
-            repo' (string/replace repo common-config/db-version-prefix "")
+                    (export-files-as-markdown files* nil))
             zip-file-name (if file
                             (string/replace-first file #"(?i)\.zip$" "")
-                            (str repo' "_markdown_" (quot (common-util/time-ms) 1000)))
+                            (str graph "_markdown_" (quot (common-util/time-ms) 1000)))
             file-name (or file (str zip-file-name ".zip"))
             zip (cli-common-util/make-export-zip zip-file-name files)
             ;; matches behavior in make-export-zip
@@ -78,6 +75,5 @@
     (cli-util/error "Command missing required option 'graph'"))
   (if (fs/existsSync (cli-util/get-graph-path graph))
     (let [conn (apply sqlite-cli/open-db! (cli-util/->open-db-args graph))]
-      (cli-util/ensure-db-graph-for-command @conn)
-      (export-repo-as-markdown! (str common-config/db-version-prefix graph) @conn opts))
+      (export-repo-as-markdown! graph @conn opts))
     (cli-util/error "Graph" (pr-str graph) "does not exist")))

deps/cli/src/logseq/cli/commands/export_edn.cljs

@@ -33,7 +33,6 @@
     (cli-util/error "Command missing required option 'graph'"))
   (if (fs/existsSync (cli-util/get-graph-path graph))
     (let [conn (apply sqlite-cli/open-db! (cli-util/->open-db-args graph))
-          _ (cli-util/ensure-db-graph-for-command @conn)
           export-map (sqlite-export/build-export @conn (build-export-options options))]
       (when validate
         (validate-export export-map options))

deps/cli/src/logseq/cli/commands/graph.cljs

@@ -1,7 +1,6 @@
 (ns logseq.cli.commands.graph
   "Graph related commands"
   (:require ["fs" :as fs]
-            ["path" :as node-path]
             [cljs-time.coerce :as tc]
             [clojure.pprint :as pprint]
             [clojure.string :as string]
@@ -22,7 +21,6 @@
     (let [graph-dir (cli-util/get-graph-path graph)]
       (if (fs/existsSync graph-dir)
         (let [conn (apply sqlite-cli/open-db! (cli-util/->open-db-args graph))
-              _ (cli-util/ensure-db-graph-for-command @conn)
               kv-value #(:kv/value (d/entity @conn %))]
           (pprint/print-table
            (map #(array-map "Name" (first %) "Value" (second %))
@@ -39,16 +37,7 @@
 
 (defn list-graphs
   []
-  (let [[db-graphs* file-graphs*] ((juxt filter remove) #(string/starts-with? % common-config/db-version-prefix)
-                                                        (cli-common-graph/get-db-based-graphs))
-        db-graphs (->> db-graphs*
+  (let [db-graphs (->> (cli-common-graph/get-db-based-graphs)
                        (map #(string/replace-first % common-config/db-version-prefix ""))
-                       sort)
-        file-graphs (->> file-graphs*
-                         (map #(string/replace-first % common-config/file-version-prefix ""))
-                         (map node-path/basename)
-                         sort)]
-    (println "DB Graphs:")
-    (println (string/join "\n" db-graphs))
-    (println "\nFile Graphs:")
-    (println (string/join "\n" file-graphs))))
+                       sort)]
+    (println (string/join "\n" db-graphs))))

deps/cli/src/logseq/cli/commands/import_edn.cljs

@@ -26,7 +26,6 @@
   (if (fs/existsSync (cli-util/get-graph-path graph))
     (let [conn (apply sqlite-cli/open-db! (cli-util/->open-db-args graph))
           _ (db-pipeline/add-listener conn)
-          _ (cli-util/ensure-db-graph-for-command @conn)
           {:keys [init-tx block-props-tx misc-tx]}
           (sqlite-export/build-import import-map @conn {})
           txs (vec (concat init-tx block-props-tx misc-tx))]

deps/cli/src/logseq/cli/commands/query.cljs

@@ -77,7 +77,6 @@
   (doseq [graph graphs]
     (if (fs/existsSync (cli-util/get-graph-path graph))
       (let [conn (apply sqlite-cli/open-db! (cli-util/->open-db-args graph))
-            _ (cli-util/ensure-db-graph-for-command @conn)
             query* (when (string? (first args)) (common-util/safe-read-string {:log-error? false} (first args)))
             results (cond
                       ;; Run datalog query if detected

deps/cli/src/logseq/cli/commands/search.cljs

@@ -55,7 +55,6 @@
     (cli-util/error "Command missing required option 'graph'"))
   (if (fs/existsSync (cli-util/get-graph-path graph))
     (let [conn (apply sqlite-cli/open-db! (cli-util/->open-db-args graph))
-          _ (cli-util/ensure-db-graph-for-command @conn)
           nodes (->> (d/datoms @conn :aevt :block/title)
                      (filter (fn [datom]
                                (string/includes? (:v datom) search-term)))

deps/cli/src/logseq/cli/commands/validate.cljs

@@ -21,8 +21,7 @@
 
 (defn- validate-graph [graph options]
   (if (fs/existsSync (cli-util/get-graph-path graph))
-    (let [conn (apply sqlite-cli/open-db! (cli-util/->open-db-args graph))
-          _ (cli-util/ensure-db-graph-for-command @conn)]
+    (let [conn (apply sqlite-cli/open-db! (cli-util/->open-db-args graph))]
       (validate-db @conn graph options))
     (cli-util/error "Graph" (pr-str graph) "does not exist")))
 

deps/cli/src/logseq/cli/common/export/common.cljs

@@ -55,21 +55,20 @@
 ;; Global vars that are not explicitly passed in all fns
 ;; These vars must be bound in order to use most fns in this namespace
 (def ^:dynamic *current-db* nil)
-(def ^:dynamic *current-repo* nil)
 ;; Config used by logseq.cli.common.file fns
 (def ^:dynamic *content-config* nil)
 
 ;;; internal utils
 (defn ^:api get-blocks-contents
-  [repo root-block-uuid & {:keys [init-level]
-                           :or {init-level 1}}]
+  [root-block-uuid & {:keys [init-level]
+                      :or {init-level 1}}]
   (let [block (d/entity *current-db* [:block/uuid root-block-uuid])
         link (:block/link block)
         block' (or link block)
         root-id (:block/uuid block')
         blocks (ldb/get-block-and-children *current-db* root-id)
-        tree (otree/blocks->vec-tree repo *current-db* blocks root-id {:link link})]
-    (common-file/tree->file-content *current-repo* *current-db* tree
+        tree (otree/blocks->vec-tree *current-db* blocks root-id {:link link})]
+    (common-file/tree->file-content *current-db* tree
                                     {:init-level init-level :link link}
                                     *content-config*)))
 
@@ -78,25 +77,25 @@
 (defn- block-uuid->ast
   [block-uuid]
   (let [block (into {} (d/entity *current-db* [:block/uuid block-uuid]))
-        content (common-file/tree->file-content *current-repo* *current-db* [block] {:init-level 1} *content-config*)
+        content (common-file/tree->file-content *current-db* [block] {:init-level 1} *content-config*)
         format :markdown]
     (when content
       (removev Properties-block-ast?
                (mapv remove-block-ast-pos
-                     (gp-mldoc/->edn *current-repo* content format))))))
+                     (gp-mldoc/->db-edn content format))))))
 
 (defn- block-uuid->ast-with-children
   [block-uuid]
-  (let [content (get-blocks-contents *current-repo* block-uuid)
+  (let [content (get-blocks-contents block-uuid)
         format :markdown]
     (when content
       (removev Properties-block-ast?
                (mapv remove-block-ast-pos
-                     (gp-mldoc/->edn *current-repo* content format))))))
+                     (gp-mldoc/->db-edn content format))))))
 
 (defn ^:api get-page-content
   [page-uuid]
-  (common-file/block->content *current-repo* *current-db* page-uuid nil *content-config*))
+  (common-file/block->content *current-db* page-uuid nil *content-config*))
 
 (defn- page-name->ast
   [page-name]
@@ -106,7 +105,7 @@
         (let [format :markdown]
           (removev Properties-block-ast?
                    (mapv remove-block-ast-pos
-                         (gp-mldoc/->edn *current-repo* content format))))))))
+                         (gp-mldoc/->db-edn content format))))))))
 
 (defn- update-level-in-block-ast-coll
   [block-ast-coll origin-level]

deps/cli/src/logseq/cli/common/export/text.cljs

@@ -451,7 +451,7 @@
 ;;; block-ast, inline-ast -> simple-ast (ends)
 
 (defn ^:large-vars/cleanup-todo export-helper
-  [repo content format options]
+  [content format options]
   (let [remove-options (set (:remove-options options))
         other-options (:other-options options)]
     (binding [*state* (merge *state*
@@ -463,7 +463,7 @@
                                :remove-properties? (contains? remove-options :property)
                                :keep-only-level<=N (:keep-only-level<=N other-options)
                                :newline-after-block (:newline-after-block other-options)}})]
-      (let [ast (gp-mldoc/->edn repo content format)
+      (let [ast (gp-mldoc/->db-edn content format)
             ast (mapv cli-export-common/remove-block-ast-pos ast)
             ast (removev cli-export-common/Properties-block-ast? ast)
             ast* (cli-export-common/replace-block&page-reference&embed ast)

deps/cli/src/logseq/cli/common/file.cljs

@@ -1,14 +1,10 @@
 (ns logseq.cli.common.file
-  "Convert blocks to file content for file and DB graphs. Used for exports and
-  saving file to disk. Shared by CLI, worker and frontend namespaces"
+  "Convert blocks to file content. Used for frontend exports and CLI"
   (:require [clojure.string :as string]
             [datascript.core :as d]
             [logseq.db :as ldb]
-            [logseq.db.common.entity-plus :as entity-plus]
             [logseq.db.frontend.content :as db-content]
             [logseq.db.sqlite.create-graph :as sqlite-create-graph]
-            [logseq.db.sqlite.util :as sqlite-util]
-            [logseq.graph-parser.property :as gp-property]
             [logseq.outliner.tree :as otree]))
 
 (defn- indented-block-content
@@ -16,124 +12,77 @@
   (let [lines (string/split-lines content)]
     (string/join (str "\n" spaces-tabs) lines)))
 
-(defn- content-with-collapsed-state
-  "Only accept nake content (without any indentation)"
-  [repo format content collapsed?]
-  (cond
-    collapsed?
-    (gp-property/insert-property repo format content :collapsed true)
-
-    ;; Don't check properties. Collapsed is an internal state log as property in file, but not counted into properties
-    (false? collapsed?)
-    (gp-property/remove-property format :collapsed content)
-
-    :else
-    content))
-
-(defn- ^:large-vars/cleanup-todo transform-content
-  [repo db {:block/keys [collapsed? format pre-block? properties] :as b} level {:keys [heading-to-list?]} context {:keys [db-based?]}]
-  (let [title (or (:block/raw-title b) (:block/title b))
-        block-ref-not-saved? (and (not db-based?)
-                                  (first (:block/_refs (d/entity db (:db/id b))))
-                                  (not (string/includes? title (str (:block/uuid b)))))
-        heading (:heading properties)
-        title (if db-based?
-                ;; replace [[uuid]] with block's content
-                (db-content/recur-replace-uuid-in-block-title (d/entity db (:db/id b)))
-                title)
+(defn- transform-content
+  [db b level {:keys [heading-to-list?]} context]
+  (let [heading (:logseq.property/heading b)
+        ;; replace [[uuid]] with block's content
+        title (db-content/recur-replace-uuid-in-block-title (d/entity db (:db/id b)))
         content (or title "")
-        content (cond
-                  pre-block?
-                  (let [content (string/trim content)]
-                    (str content "\n"))
-
-                  :else
-                  (let [[prefix spaces-tabs]
-                        (cond
-                          (= format :org)
-                          [(->>
-                            (repeat level "*")
-                            (apply str)) ""]
-
-                          :else
-                          (let [level (if (and heading-to-list? heading)
-                                        (if (> heading 1)
-                                          (dec heading)
-                                          heading)
-                                        level)
-                                spaces-tabs (->>
-                                             (repeat (dec level) (:export-bullet-indentation context))
-                                             (apply str))]
-                            [(str spaces-tabs "-") (str spaces-tabs "  ")]))
-                        content (if heading-to-list?
-                                  (-> (string/replace content #"^\s?#+\s+" "")
-                                      (string/replace #"^\s?#+\s?$" ""))
-                                  content)
-                        content (if db-based? content (content-with-collapsed-state repo format content collapsed?))
-                        new-content (indented-block-content (string/trim content) spaces-tabs)
-                        sep (if (string/blank? new-content)
-                              ""
-                              " ")]
-                    (str prefix sep new-content)))]
-    (if block-ref-not-saved?
-      (gp-property/insert-property repo format content :id (str (:block/uuid b)))
-      content)))
+        content (let [[prefix spaces-tabs]
+                      (let [level (if (and heading-to-list? heading)
+                                    (if (> heading 1)
+                                      (dec heading)
+                                      heading)
+                                    level)
+                            spaces-tabs (->>
+                                         (repeat (dec level) (:export-bullet-indentation context))
+                                         (apply str))]
+                        [(str spaces-tabs "-") (str spaces-tabs "  ")])
+                      content (if heading-to-list?
+                                (-> (string/replace content #"^\s?#+\s+" "")
+                                    (string/replace #"^\s?#+\s?$" ""))
+                                content)
+                      new-content (indented-block-content (string/trim content) spaces-tabs)
+                      sep (if (string/blank? new-content)
+                            ""
+                            " ")]
+                  (str prefix sep new-content))]
+    content))
 
 (defn- tree->file-content-aux
-  [repo db tree {:keys [init-level link] :as opts} context]
-  (let [db-based? (sqlite-util/db-based-graph? repo)
-        block-contents (transient [])]
+  [db tree {:keys [init-level link] :as opts} context]
+  (let [block-contents (transient [])]
     (loop [[f & r] tree level init-level]
       (if (nil? f)
         (->> block-contents persistent! flatten (remove nil?))
         (let [page? (nil? (:block/page f))
-              content (if (and page? (not link)) nil (transform-content repo db f level opts context {:db-based? db-based?}))
+              content (if (and page? (not link)) nil (transform-content db f level opts context))
               new-content
               (if-let [children (seq (:block/children f))]
-                (cons content (tree->file-content-aux repo db children {:init-level (inc level)} context))
+                (cons content (tree->file-content-aux db children {:init-level (inc level)} context))
                 [content])]
           #_:clj-kondo/ignore
           (conj! block-contents new-content)
           (recur r level))))))
 
 (defn tree->file-content
-  "Used by both file and DB graphs for export and for file-graph specific features"
-  [repo db tree opts context]
-  (->> (tree->file-content-aux repo db tree opts context) (string/join "\n")))
-
-(defn- update-block-content
-  [db item eid]
-  ;; This may not be needed if this becomes a file-graph only context
-  (if (entity-plus/db-based-graph? db)
-    (db-content/update-block-content db item eid)
-    item))
+  [db tree opts context]
+  (->> (tree->file-content-aux db tree opts context) (string/join "\n")))
 
 (defn block->content
   "Converts a block including its children (recursively) to plain-text."
-  [repo db root-block-uuid tree->file-opts context]
+  [db root-block-uuid tree->file-opts context]
   (assert (uuid? root-block-uuid))
   (let [init-level (or (:init-level tree->file-opts)
                        (if (ldb/page? (d/entity db [:block/uuid root-block-uuid]))
                          0
                          1))
         blocks (->> (d/pull-many db '[*] (keep :db/id (ldb/get-block-and-children db root-block-uuid)))
-                    (map #(update-block-content db % (:db/id %))))
-        tree (otree/blocks->vec-tree repo db blocks (str root-block-uuid))]
-    (tree->file-content repo db tree
+                    (map #(db-content/update-block-content db % (:db/id %))))
+        tree (otree/blocks->vec-tree db blocks (str root-block-uuid))]
+    (tree->file-content db tree
                         (assoc tree->file-opts :init-level init-level)
                         context)))
 
 (defn get-all-page->content
   "Exports a graph's pages as tuples of page name and page content"
-  [repo db options]
-  (let [filter-fn (if (ldb/db-based-graph? db)
-                    (fn [ent]
-                      (or (not (:logseq.property/built-in? ent))
-                          (contains? sqlite-create-graph/built-in-pages-names (:block/title ent))))
-                    (constantly true))]
+  [db options]
+  (let [filter-fn (fn [ent]
+                    (or (not (:logseq.property/built-in? ent))
+                        (contains? sqlite-create-graph/built-in-pages-names (:block/title ent))))]
     (->> (d/datoms db :avet :block/name)
          (map #(d/entity db (:e %)))
          (filter filter-fn)
          (map (fn [e]
                 [(:block/title e)
-                 (block->content repo db (:block/uuid e) {} options)])))))
+                 (block->content db (:block/uuid e) {} options)])))))

deps/cli/src/logseq/cli/common/mcp/tools.cljs

@@ -5,8 +5,8 @@
             [logseq.common.util :as common-util]
             [logseq.common.util.date-time :as date-time-util]
             [logseq.db :as ldb]
-            [logseq.db.frontend.content :as db-content]
             [logseq.db.frontend.class :as db-class]
+            [logseq.db.frontend.content :as db-content]
             [logseq.db.frontend.entity-util :as entity-util]
             [logseq.db.frontend.property :as db-property]
             [logseq.db.frontend.property.type :as db-property-type]
@@ -16,15 +16,9 @@
             [malli.core :as m]
             [malli.error :as me]))
 
-(defn- ensure-db-graph
-  [db]
-  (when-not (ldb/db-based-graph? db)
-    (throw (ex-info "This tool must be called on a DB graph" {}))))
-
 (defn list-properties
   "Main fn for ListProperties tool"
   [db {:keys [expand]}]
-  (ensure-db-graph db)
   (->> (d/datoms db :avet :block/tags :logseq.class/Property)
        (map #(d/entity db (:e %)))
        #_((fn [x] (prn :prop-keys (distinct (mapcat keys x))) x))
@@ -46,7 +40,6 @@
 (defn list-tags
   "Main fn for ListTags tool"
   [db {:keys [expand]}]
-  (ensure-db-graph db)
   (->> (d/datoms db :avet :block/tags :logseq.class/Tag)
        (map #(d/entity db (:e %)))
        (map (fn [e]
@@ -74,8 +67,7 @@
         block-eids (mapv :e datoms)
         block-ents (map #(d/entity db %) block-eids)
         blocks (map #(assoc % :block/title (db-content/recur-replace-uuid-in-block-title %)) block-ents)]
-      ;; Use repo stub since this is a DB only tool
-    (->> (otree/blocks->vec-tree "logseq_db_repo_stub" db blocks page-id)
+    (->> (otree/blocks->vec-tree db blocks page-id)
          (map #(update % :block/uuid str)))))
 
 (defn ^:api remove-hidden-properties
@@ -89,7 +81,6 @@
 (defn get-page-data
   "Get page data for GetPage tool including the page's entity and its blocks"
   [db page-name-or-uuid]
-  (ensure-db-graph db)
   (when-let [page (ldb/get-page db page-name-or-uuid)]
     {:entity (-> (remove-hidden-properties page)
                  (dissoc :block/tags :block/refs)
@@ -103,7 +94,6 @@
 (defn list-pages
   "Main fn for ListPages tool"
   [db {:keys [expand]}]
-  (ensure-db-graph db)
   (->> (d/datoms db :avet :block/name)
        (map #(d/entity db (:e %)))
        (remove entity-util/hidden?)
@@ -369,7 +359,6 @@
   "Given llm generated operations, builds the import EDN, validates it and returns it. It fails
    fast on anything invalid"
   [db operations*]
-  (ensure-db-graph db)
   ;; Only support these operations with appropriate outliner validations
   (when (seq (filter #(and (#{"page" "tag" "property"} (:entityType %)) (= "edit" (:operation %))) operations*))
     (throw (ex-info "Editing a page, tag or property isn't supported yet" {})))

deps/cli/src/logseq/cli/util.cljs

@@ -4,7 +4,6 @@
             ["path" :as node-path]
             [clojure.string :as string]
             [logseq.cli.common.graph :as cli-common-graph]
-            [logseq.db.common.entity-plus :as entity-plus]
             [logseq.db.common.sqlite :as common-sqlite]
             [nbb.error]
             [promesa.core :as p]))
@@ -82,11 +81,6 @@
          (count (:classes edn-map)) " " (pluralize "class" (count (:classes edn-map))) " and "
          (count (:pages-and-blocks edn-map)) " " (pluralize "page" (count (:pages-and-blocks edn-map))))))
 
-(defn ensure-db-graph-for-command
-  [db]
-  (when-not (entity-plus/db-based-graph? db)
-    (error "This command must be called on a DB graph")))
-
 (defn api-command?
   "Given user options and $LOGSEQ_API_SERVER_TOKEN, determines if
    given command is an api (true) or local (false) command"

deps/common/.carve/config.edn

@@ -8,7 +8,6 @@
                   logseq.common.util.date-time
                   logseq.common.date
                   logseq.common.util.macro
-                  logseq.common.marker
                   logseq.common.config
                   logseq.common.defkeywords]
  :report {:format :ignore}}

deps/common/.carve/ignore

@@ -6,4 +6,9 @@ logseq.common.graph/read-directories
 ;; Profile utils
 logseq.common.profile/profile-fn!
 logseq.common.profile/*key->call-count
-logseq.common.profile/*key->time-sum
+logseq.common.profile/*key->time-sum
+
+;; API fn
+logseq.common.plural/is-plural?
+logseq.common.plural/is-singular?
+logseq.common.plural/pluralize

deps/common/README.md

@@ -16,7 +16,7 @@ This library is under the parent namespace `logseq.common`.
 This follows the practices that [the Logseq frontend
 follows](/docs/dev-practices.md). Most of the same linters are used, with
 configurations that are specific to this library. See [this library's CI
-file](/.github/workflows/logseq-common.yml) for linting examples.
+file](/.github/workflows/deps-common.yml) for linting examples.
 
 ### Setup
 

deps/common/bb.edn

@@ -23,4 +23,5 @@
 
  :tasks/config
  {:large-vars
-  {:max-lines-count 45}}}
+  {:metadata-exceptions #{:large-vars/cleanup-todo}
+   :max-lines-count 45}}}

deps/common/resources/templates/config.edn

@@ -1,137 +1,6 @@
 {:meta/version 1
 
- ;; == FILE GRAPH CONFIG ==
- ;;
- ;; Set the preferred format.
- ;; This is _only_ for file graphs.
- ;; Available options:
- ;; - Markdown (default)
- ;; - Org
- ;; :preferred-format "Markdown"
-
- ;; Set the preferred workflow style.
- ;; This is _only_ for file graphs.
- ;; Available options:
- ;; - :now for NOW/LATER style (default)
- ;; - :todo for TODO/DOING style
- :preferred-workflow :now
-
- ;; Exclude directories/files.
- ;; This is _only_ for file graphs.
- ;; Example usage:
- ;; :hidden ["/archived" "/test.md" "../assets/archived"]
- :hidden []
-
- ;; Define the default journal page template.
- ;; Enter the template name between the quotes.
- ;; This is _only_ for file graphs.
- :default-templates
- {:journals ""}
-
- ;; Set a custom date format for the journal page title.
- ;; This is _only_ for file graphs.
- ;; Default value: "MMM do, yyyy"
- ;; e.g., "Jan 19th, 2038"
- ;; Example usage e.g., "Tue 19th, Jan 2038"
- ;; :journal/page-title-format "EEE do, MMM yyyy"
-
- ;; Specify the journal filename format using a valid date format string.
- ;; !Warning:
- ;;   This configuration is not retroactive and affects only new journals.
- ;;   To show old journal files in the app, manually rename the files in the
- ;;   journal directory to match the new format.
- ;; This is _only_ for file graphs.
- ;; Default value: "yyyy_MM_dd"
- ;; :journal/file-name-format "yyyy_MM_dd"
-
- ;; Set the default location for storing notes.
- ;; This is _only_ for file graphs.
- ;; Default value: "pages"
- ;; :pages-directory "pages"
-
- ;; Set the default location for storing journals.
- ;; This is _only_ for file graphs.
- ;; Default value: "journals"
- ;; :journals-directory "journals"
-
- ;; Set the default location for storing whiteboards.
- ;; This is _only_ for file graphs.
- ;; Default value: "whiteboards"
- ;; :whiteboards-directory "whiteboards"
-
- ;; Enabling this option converts
- ;; [[Grant Ideas]] to [[file:./grant_ideas.org][Grant Ideas]] for org-mode.
- ;; For more information, visit https://github.com/logseq/logseq/issues/672
- ;; This is _only_ for file graphs.
- ;; :org-mode/insert-file-link? false
-
-;; Favorites to list on the left sidebar
- ;; This is _only_ for file graphs.
- :favorites []
-
- ;; Set flashcards interval.
- ;; This is _only_ for file graphs.
- ;; Expected value:
- ;; - Float between 0 and 1
- ;; higher values result in faster changes to the next review interval.
- ;; Default value: 0.5
- ;; :srs/learning-fraction 0.5
-
- ;; Set the initial interval after the first successful review of a card.
- ;; This is _only_ for file graphs.
- ;; Default value: 4
- ;; :srs/initial-interval 4
-
- ;; Hide specific block properties.
- ;; This is _only_ for file graphs.
- ;; Example usage:
- ;; :block-hidden-properties #{:public :icon}
-
- ;; Create a page for all properties.
- ;; This is _only_ for file graphs.
- ;; Default value: true
- :property-pages/enabled? true
-
- ;; Properties to exclude from having property pages
- ;; This is _only_ for file graphs.
- ;; Example usage:
- ;; :property-pages/excludelist #{:duration :author}
-
- ;; By default, property value separated by commas will not be treated as
- ;; page references. You can add properties to enable it.
- ;; This is _only_ for file graphs.
- ;; Example usage:
- ;; :property/separated-by-commas #{:alias :tags}
-
- ;; Properties that are ignored when parsing property values for references
- ;; This is _only_ for file graphs.
- ;; Example usage:
- ;; :ignored-page-references-keywords #{:author :website}
-
- ;; logbook configuration.
- ;; This is _only_ for file graphs.
- ;; :logbook/settings
- ;; {:with-second-support? false ;limit logbook to minutes, seconds will be eliminated
- ;;  :enabled-in-all-blocks true ;display logbook in all blocks after timetracking
- ;;  :enabled-in-timestamped-blocks false ;don't display logbook at all
- ;; }
-
- ;; Configure the escaping method for special characters in page titles.
- ;; This is _only_ for file graphs.
- ;; Warning:
- ;;   This is a dangerous operation. To modify the setting,
- ;;   you'll need to manually rename all affected files and
- ;;   re-index them on all clients after synchronization.
- ;;   Incorrect handling may result in messy page titles.
- ;; Available options:
- ;;   - :triple-lowbar (default)
- ;;      ;use triple underscore `___` for slash `/` in page title
- ;;      ;use Percent-encoding for other invalid characters
- :file/name-format :triple-lowbar
- ;; == END OF FILE GRAPH CONFIG ==
-
  ;; Hide empty block properties
- ;; This is _only_ for DB graphs.
  ;; Default value: false
  ;; :ui/hide-empty-properties? false
 
@@ -164,10 +33,6 @@
  ;; Default value: true
  ;; :feature/enable-flashcards? true
 
- ;; Enable whiteboards.
- ;; Default value: true
- ;; :feature/enable-whiteboards? true
-
  ;; Disable the journal's built-in 'Scheduled tasks and deadlines' query.
  ;; Default value: false
  ;; :feature/disable-scheduled-and-deadline-query? false

deps/common/src/logseq/common/async.clj

@@ -1,4 +1,4 @@
-(ns logseq.common.async
+(ns ^:no-doc logseq.publish.async
   (:require [shadow.cljs.modern]))
 
 (defmacro js-await

deps/common/src/logseq/common/config.cljs

@@ -66,28 +66,6 @@
         (string/replace-first asset-protocol "file://"))
     s))
 
-(defonce default-draw-directory "draws")
-;; TODO read configurable value?
-(defonce default-whiteboards-directory "whiteboards")
-
-(defn draw?
-  [path]
-  (string/starts-with? path default-draw-directory))
-
-(defn whiteboard?
-  [path]
-  (and path
-       (string/includes? path (str default-whiteboards-directory "/"))
-       (string/ends-with? path ".edn")))
-
-;; TODO: rename
-(defonce mldoc-support-formats
-  #{:org :markdown :md})
-
-(defn mldoc-support?
-  [format]
-  (contains? mldoc-support-formats (keyword format)))
-
 (defn text-formats
   []
   #{:json :org :md :yml :dat :asciidoc :rst :txt :markdown :adoc :html :js :ts :edn :clj :ml :rb :ex :erl :java :php :c :css
@@ -97,34 +75,7 @@
   []
   #{:gif :svg :jpeg :ico :png :jpg :bmp :webp})
 
-(defn get-date-formatter
-  [config]
-  (or
-   (:journal/page-title-format config)
-   ;; for compatibility
-   (:date-formatter config)
-   "MMM do, yyyy"))
-
-(defn get-preferred-format
-  [config]
-  (or
-   (when-let [fmt (:preferred-format config)]
-     (keyword (string/lower-case (name fmt))))
-   :markdown))
-
-(defn get-block-pattern
-  [format]
-  (let [format' (keyword format)]
-    (case format'
-      :org
-      "*"
-
-      "-")))
-
-(defn create-config-for-db-graph
-  "Given a new config.edn file string, creates a config.edn for use with only DB graphs"
-  [config]
-  (string/replace config #"(?m)[\s]*;; == FILE GRAPH CONFIG ==(?:.|\n)*?;; == END OF FILE GRAPH CONFIG ==\n?" ""))
+(defonce block-pattern "-")
 
 (def file-only-config
   "File only config keys that are deprecated in DB graphs along with
@@ -146,7 +97,8 @@
      :property-pages/excludelist
      :srs/learning-fraction
      :srs/initial-interval
-     :whiteboards-directory]
+     :whiteboards-directory
+     :feature/enable-whiteboards?]
     (repeat "is not used in DB graphs"))
    {:preferred-format
     "is not used in DB graphs as there is only markdown mode."

deps/common/src/logseq/common/date.cljs

@@ -2,10 +2,8 @@
   "Date related fns shared by worker and frontend namespaces. Eventually some
    of this should go to logseq.common.util.date-time"
   (:require [cljs-time.format :as tf]
-            [logseq.common.util :as common-util]
-            [clojure.string :as string]))
-
-(def default-journal-filename-formatter "yyyy_MM_dd")
+            [clojure.string :as string]
+            [logseq.common.util :as common-util]))
 
 (defonce built-in-journal-title-formatters
   (list
@@ -79,12 +77,4 @@
 
 (defn ^:api valid-journal-title-with-slash?
   [title]
-  (some #(valid-journal-title? title %) slash-journal-title-formatters))
-
-(defn ^:api date->file-name
-  "Date object to filename format"
-  [date journal-filename-formatter]
-  (let [formatter (if journal-filename-formatter
-                    (tf/formatter journal-filename-formatter)
-                    (tf/formatter default-journal-filename-formatter))]
-    (tf/unparse formatter date)))
+  (some #(valid-journal-title? title %) slash-journal-title-formatters))

deps/common/src/logseq/common/marker.cljs

@@ -1,13 +0,0 @@
-(ns logseq.common.marker
-  "Marker patterns. File graph only"
-  (:require [clojure.string :as string]))
-
-(defn marker-pattern [format]
-  (re-pattern
-   (str "^" (if (= format :markdown) "(#+\\s+)?" "(\\*+\\s+)?")
-        "(NOW|LATER|TODO|DOING|DONE|WAITING|WAIT|CANCELED|CANCELLED|IN-PROGRESS)?\\s?")))
-
-
-(defn clean-marker
-  [content format]
-  (string/replace-first content (marker-pattern format) ""))

deps/common/src/logseq/common/path.cljs

@@ -51,17 +51,6 @@
   [path]
   (second (split-ext path)))
 
-(defn safe-filename?
-  "Safe filename on all platforms"
-  [fname]
-  (and (not (string/blank? fname))
-       (< (count fname) 255)
-       (not (or (re-find #"[\/?<>\\:*|\"]" fname)
-                (re-find #"^\.+$" fname)
-                (re-find #"[\. ]$" fname)
-                (re-find #"(?i)^(COM[0-9]|CON|LPT[0-9]|NUL|PRN|AUX|com[0-9]|con|lpt[0-9]|nul|prn|aux)\..+" fname)
-                (re-find #"[\u0000-\u001f\u0080-\u009f]" fname)))))
-
 (defn- path-join-internal
   "Joins the given path segments into a single path, handling relative paths,
   '..' and '.' normalization."
@@ -253,31 +242,6 @@
         (js/console.error "unhandled trim-base" base-path sub-path)
         nil))))
 
-(defn relative-path
-  "Get relative path from base path.
-   Works for both path and URL."
-  [base-path sub-path]
-  (let [base-path (path-normalize base-path)
-        sub-path (path-normalize sub-path)
-        is-url? (is-file-url? base-path)]
-    (if (string/starts-with? sub-path base-path)
-      (if is-url?
-        (safe-decode-uri-component (string/replace (subs sub-path (count base-path)) #"^/+", ""))
-        (string/replace (subs sub-path (count base-path)) #"^/+", ""))
-      ;; append as many ..
-      ;; NOTE: This is a buggy impl, relative-path is different when base-path is a file or a dir
-      (let [base-segs (string/split base-path #"/" -1)
-            path-segs (string/split sub-path #"/" -1)
-            common-segs (take-while #(= (first %) (second %)) (map vector base-segs path-segs))
-            base-segs (drop (count common-segs) base-segs)
-            remain-segs (drop (count common-segs) path-segs)
-            base-prefix (apply str (repeat (max 0 (dec (count base-segs))) "../"))]
-        (js/console.error (js/Error. "buggy relative-path") base-path sub-path)
-        #_{:clj-kondo/ignore [:path-invalid-construct/string-join]}
-        (if is-url?
-          (safe-decode-uri-component (str base-prefix (string/join "/" remain-segs)))
-          (str base-prefix (string/join "/" remain-segs)))))))
-
 (defn parent
   "Parent, containing directory"
   [path]
@@ -286,45 +250,12 @@
     (path-normalize (str path "/.."))
     nil))
 
-(defn resolve-relative-path
-  "Assume current-path is a file"
-  [current-path rel-path]
-  (if-let [base-dir (parent current-path)]
-    (path-join base-dir rel-path)
-    rel-path))
-
-(defn get-relative-path
-  "Assume current-path is a file, and target-path is a file or directory.
-   Return relative path from current-path to target-path.
-   Works for both path and URL. Also works for relative path.
-   The opposite operation is `resolve-relative-path`"
-  [current-path target-path]
-  (let [base-path (parent current-path)
-        sub-path (path-normalize target-path)
-        is-url? (is-file-url? base-path)
-        base-segs (if base-path
-                    (string/split base-path #"/" -1)
-                    [])
-        path-segs (string/split sub-path #"/" -1)
-        common-segs (take-while #(= (first %) (second %)) (map vector base-segs path-segs))
-        base-segs (drop (count common-segs) base-segs)
-        remain-segs (drop (count common-segs) path-segs)
-        base-prefix (apply str (repeat (max 0 (count base-segs)) "../"))]
-    #_{:clj-kondo/ignore [:path-invalid-construct/string-join]}
-    (if is-url?
-      (safe-decode-uri-component (str base-prefix (string/join "/" remain-segs)))
-      (str base-prefix (string/join "/" remain-segs)))))
-
 ;; compat
 (defn basename
   [path]
   (let [path (string/replace path #"/+$" "")]
     (filename path)))
 
-(defn dirname
-  [path]
-  (parent path))
-
 (defn absolute?
   "Whether path `p` is absolute."
   [p]

deps/common/src/logseq/common/plural.cljs

@@ -0,0 +1,333 @@
+(ns logseq.common.plural
+  "ClojureScript port of pluralize.js core (rules + API).
+
+  Usage:
+    (pluralize \"duck\" 2 true)     ;; => \"2 ducks\"
+    (plural \"person\")            ;; => \"people\"
+    (singular \"people\")          ;; => \"person\"
+    (is-plural? \"ducks\")         ;; => true
+    (is-singular? \"duck\")        ;; => true
+
+  You can add rules at runtime:
+    (add-plural-rule! #\"(ox)$\" \"$1en\")
+    (add-uncountable-rule! \"metadata\")"
+  (:require [clojure.string :as string]))
+
+;; -----------------------------------------------------------------------------
+;; Rule storage (mirrors original semantics)
+;; pluralize and singularize must run rules sequentially.
+;; -----------------------------------------------------------------------------
+
+(defonce ^:private plural-rules (atom []))      ;; vector of [js/RegExp replacement]
+(defonce ^:private singular-rules (atom []))    ;; vector of [js/RegExp replacement]
+(defonce ^:private uncountables (atom {}))      ;; token -> true
+(defonce ^:private irregular-plurals (atom {})) ;; plural -> singular
+(defonce ^:private irregular-singles (atom {})) ;; singular -> plural
+
+;; -----------------------------------------------------------------------------
+;; Helpers
+;; -----------------------------------------------------------------------------
+
+(defn- sanitize-rule
+  "If rule is a string, compile to case-insensitive regexp that matches the whole string.
+   Else keep it (assumed to be js/RegExp)."
+  [rule]
+  (if (string? rule)
+    (js/RegExp. (str "^" rule "$") "i")
+    rule))
+
+(defn- restore-case
+  "Replicate casing of `word` onto `token`."
+  [word token]
+  (cond
+    (= word token)
+    token
+
+    (= word (string/lower-case word))
+    (string/lower-case token)
+
+    (= word (string/upper-case word))
+    (string/upper-case token)
+
+    (and (seq word)
+         (= (subs word 0 1) (string/upper-case (subs word 0 1))))
+    (str (string/upper-case (subs token 0 1))
+         (string/lower-case (subs token 1)))
+
+    :else
+    (string/lower-case token)))
+
+(defn- interpolate
+  "Replace $1..$12 etc in `s` using JS replace args (match, g1, g2 ...)."
+  [s js-args]
+  (.replace s (js/RegExp. "\\$(\\d{1,2})" "g")
+            (fn [_ idx]
+              (let [i (js/parseInt idx 10)
+                    v (aget js-args i)]
+                (or v "")))))
+
+(defn- replace-with-rule
+  "Apply a [re repl] rule to word with casing restoration (matches JS behavior)."
+  [word [re repl]]
+  (.replace word re
+            (fn [& args]
+              ;; args: [match g1 g2 ... offset string]
+              (let [match  (nth args 0)
+                    ;; In JS replace callback, second-to-last is offset
+                    offset (nth args (- (count args) 2))
+                    ;; interpolate expects JS-ish indexed args;
+                    ;; easiest is to turn args into a JS array.
+                    js-args (to-array args)
+                    result (interpolate repl js-args)]
+                (if (= match "")
+                  ;; match empty => restore based on char before match
+                  (restore-case (subs word (dec offset) offset) result)
+                  (restore-case match result))))))
+
+(defn- sanitize-word
+  "Return sanitized `word` based on `token` and `rules`."
+  [token word rules]
+  (cond
+    (or (zero? (count token))
+        (contains? @uncountables token))
+    word
+
+    :else
+    (let [rs rules
+          ;; JS iterates from end to start
+          n  (count rs)]
+      (loop [i (dec n)]
+        (if (neg? i)
+          word
+          (let [[re _ :as rule] (nth rs i)]
+            (if (.test re word)
+              (replace-with-rule word rule)
+              (recur (dec i)))))))))
+
+(defn- replace-word-fn
+  "Build a word transformer (plural or singular)."
+  [replace-map-atom keep-map-atom rules-atom]
+  (fn [word]
+    (let [token (string/lower-case word)
+          keep-map @keep-map-atom
+          replace-map @replace-map-atom
+          rules @rules-atom]
+      (cond
+        (contains? keep-map token)
+        (restore-case word token)
+
+        (contains? replace-map token)
+        (restore-case word (get replace-map token))
+
+        :else
+        (sanitize-word token word rules)))))
+
+(defn- check-word-fn
+  "Build a predicate for whether word is plural/singular (mirrors JS `checkWord`)."
+  [replace-map-atom keep-map-atom rules-atom]
+  (fn [word]
+    (let [token (string/lower-case word)
+          keep-map @keep-map-atom
+          replace-map @replace-map-atom
+          rules @rules-atom]
+      (cond
+        (contains? keep-map token) true
+        (contains? replace-map token) false
+        :else (= (sanitize-word token token rules) token)))))
+
+;; -----------------------------------------------------------------------------
+;; Public API (matches original surface)
+;; -----------------------------------------------------------------------------
+
+(def plural (replace-word-fn irregular-singles irregular-plurals plural-rules))
+(def singular (replace-word-fn irregular-plurals irregular-singles singular-rules))
+
+(def is-plural? (check-word-fn irregular-singles irregular-plurals plural-rules))
+(def is-singular? (check-word-fn irregular-plurals irregular-singles singular-rules))
+
+(defn pluralize
+  "Pluralize or singularize based on count. If inclusive, prefix with count."
+  ([word item-count] (pluralize word item-count false))
+  ([word item-count inclusive]
+   (let [pluralized (if (= item-count 1) (singular word) (plural word))]
+     (str (when inclusive (str item-count " "))
+          pluralized))))
+
+(defn add-plural-rule!
+  [rule replacement]
+  (swap! plural-rules conj [(sanitize-rule rule) replacement]))
+
+(defn add-singular-rule!
+  [rule replacement]
+  (swap! singular-rules conj [(sanitize-rule rule) replacement]))
+
+(defn add-uncountable-rule!
+  "If word is string => mark as uncountable.
+   If regexp => add plural+singular passthrough rules ($0)."
+  [word]
+  (if (string? word)
+    (swap! uncountables assoc (string/lower-case word) true)
+    (do
+      (add-plural-rule! word "$0")
+      (add-singular-rule! word "$0"))))
+
+(defn add-irregular-rule!
+  [single plural-word]
+  (let [p (string/lower-case plural-word)
+        s (string/lower-case single)]
+    (swap! irregular-singles assoc s p)
+    (swap! irregular-plurals assoc p s)))
+
+;; -----------------------------------------------------------------------------
+;; Data initialization (same as original JS)
+;; -----------------------------------------------------------------------------
+
+(defn- ^:large-vars/cleanup-todo init-irregulars! []
+  (doseq [[s p]
+          ;; Pronouns + irregulars
+          [["I" "we"]
+           ["me" "us"]
+           ["he" "they"]
+           ["she" "they"]
+           ["them" "them"]
+           ["myself" "ourselves"]
+           ["yourself" "yourselves"]
+           ["itself" "themselves"]
+           ["herself" "themselves"]
+           ["himself" "themselves"]
+           ["themself" "themselves"]
+           ["is" "are"]
+           ["was" "were"]
+           ["has" "have"]
+           ["this" "these"]
+           ["that" "those"]
+           ["my" "our"]
+           ["its" "their"]
+           ["his" "their"]
+           ["her" "their"]
+           ;; Words ending with consonant + o
+           ["echo" "echoes"]
+           ["dingo" "dingoes"]
+           ["volcano" "volcanoes"]
+           ["tornado" "tornadoes"]
+           ["torpedo" "torpedoes"]
+           ;; Ends with us
+           ["genus" "genera"]
+           ["viscus" "viscera"]
+           ;; Ends with ma
+           ["stigma" "stigmata"]
+           ["stoma" "stomata"]
+           ["dogma" "dogmata"]
+           ["lemma" "lemmata"]
+           ["schema" "schemata"]
+           ["anathema" "anathemata"]
+           ;; Other irregular
+           ["ox" "oxen"]
+           ["axe" "axes"]
+           ["die" "dice"]
+           ["yes" "yeses"]
+           ["foot" "feet"]
+           ["eave" "eaves"]
+           ["goose" "geese"]
+           ["tooth" "teeth"]
+           ["quiz" "quizzes"]
+           ["human" "humans"]
+           ["proof" "proofs"]
+           ["carve" "carves"]
+           ["valve" "valves"]
+           ["looey" "looies"]
+           ["thief" "thieves"]
+           ["groove" "grooves"]
+           ["pickaxe" "pickaxes"]
+           ["passerby" "passersby"]
+           ["canvas" "canvases"]]]
+    (add-irregular-rule! s p)))
+
+(defn- init-plural-rules! []
+  (doseq [[rule repl]
+          [[(js/RegExp. "s?$" "i") "s"]
+           [(js/RegExp. "[^\\u0000-\\u007F]$" "i") "$0"]
+           [(js/RegExp. "([^aeiou]ese)$" "i") "$1"]
+           [(js/RegExp. "(ax|test)is$" "i") "$1es"]
+           [(js/RegExp. "(alias|[^aou]us|t[lm]as|gas|ris)$" "i") "$1es"]
+           [(js/RegExp. "(e[mn]u)s?$" "i") "$1s"]
+           [(js/RegExp. "([^l]ias|[aeiou]las|[ejzr]as|[iu]am)$" "i") "$1"]
+           [(js/RegExp. "(alumn|syllab|vir|radi|nucle|fung|cact|stimul|termin|bacill|foc|uter|loc|strat)(?:us|i)$" "i") "$1i"]
+           [(js/RegExp. "(alumn|alg|vertebr)(?:a|ae)$" "i") "$1ae"]
+           [(js/RegExp. "(seraph|cherub)(?:im)?$" "i") "$1im"]
+           [(js/RegExp. "(her|at|gr)o$" "i") "$1oes"]
+           [(js/RegExp. "(agend|addend|millenni|dat|extrem|bacteri|desiderat|strat|candelabr|errat|ov|symposi|curricul|automat|quor)(?:a|um)$" "i") "$1a"]
+           [(js/RegExp. "(apheli|hyperbat|periheli|asyndet|noumen|phenomen|criteri|organ|prolegomen|hedr|automat)(?:a|on)$" "i") "$1a"]
+           [(js/RegExp. "sis$" "i") "ses"]
+           [(js/RegExp. "(?:(kni|wi|li)fe|(ar|l|ea|eo|oa|hoo)f)$" "i") "$1$2ves"]
+           [(js/RegExp. "([^aeiouy]|qu)y$" "i") "$1ies"]
+           [(js/RegExp. "([^ch][ieo][ln])ey$" "i") "$1ies"]
+           [(js/RegExp. "(x|ch|ss|sh|zz)$" "i") "$1es"]
+           [(js/RegExp. "(matr|cod|mur|sil|vert|ind|append)(?:ix|ex)$" "i") "$1ices"]
+           [(js/RegExp. "\\b((?:tit)?m|l)(?:ice|ouse)$" "i") "$1ice"]
+           [(js/RegExp. "(pe)(?:rson|ople)$" "i") "$1ople"]
+           [(js/RegExp. "(child)(?:ren)?$" "i") "$1ren"]
+           [(js/RegExp. "eaux$" "i") "$0"]
+           [(js/RegExp. "m[ae]n$" "i") "men"]
+           ["thou" "you"]]]
+    (add-plural-rule! rule repl)))
+
+(defn- init-singular-rules! []
+  (doseq [[rule repl]
+          [[(js/RegExp. "s$" "i") ""]
+           [(js/RegExp. "(ss)$" "i") "$1"]
+           [(js/RegExp. "(wi|kni|(?:after|half|high|low|mid|non|night|[^\\w]|^)li)ves$" "i") "$1fe"]
+           [(js/RegExp. "(ar|(?:wo|[ae])l|[eo][ao])ves$" "i") "$1f"]
+           [(js/RegExp. "ies$" "i") "y"]
+           [(js/RegExp. "(dg|ss|ois|lk|ok|wn|mb|th|ch|ec|oal|is|ck|ix|sser|ts|wb)ies$" "i") "$1ie"]
+           [(js/RegExp. "\\b(l|(?:neck|cross|hog|aun)?t|coll|faer|food|gen|goon|group|hipp|junk|vegg|(?:pork)?p|charl|calor|cut)ies$" "i") "$1ie"]
+           [(js/RegExp. "\\b(mon|smil)ies$" "i") "$1ey"]
+           [(js/RegExp. "\\b((?:tit)?m|l)ice$" "i") "$1ouse"]
+           [(js/RegExp. "(seraph|cherub)im$" "i") "$1"]
+           [(js/RegExp. "(x|ch|ss|sh|zz|tto|go|cho|alias|[^aou]us|t[lm]as|gas|(?:her|at|gr)o|[aeiou]ris)(?:es)?$" "i") "$1"]
+           [(js/RegExp. "(analy|diagno|parenthe|progno|synop|the|empha|cri|ne)(?:sis|ses)$" "i") "$1sis"]
+           [(js/RegExp. "(movie|twelve|abuse|e[mn]u)s$" "i") "$1"]
+           [(js/RegExp. "(test)(?:is|es)$" "i") "$1is"]
+           [(js/RegExp. "(alumn|syllab|vir|radi|nucle|fung|cact|stimul|termin|bacill|foc|uter|loc|strat)(?:us|i)$" "i") "$1us"]
+           [(js/RegExp. "(agend|addend|millenni|dat|extrem|bacteri|desiderat|strat|candelabr|errat|ov|symposi|curricul|quor)a$" "i") "$1um"]
+           [(js/RegExp. "(apheli|hyperbat|periheli|asyndet|noumen|phenomen|criteri|organ|prolegomen|hedr|automat)a$" "i") "$1on"]
+           [(js/RegExp. "(alumn|alg|vertebr)ae$" "i") "$1a"]
+           [(js/RegExp. "(cod|mur|sil|vert|ind)ices$" "i") "$1ex"]
+           [(js/RegExp. "(matr|append)ices$" "i") "$1ix"]
+           [(js/RegExp. "(pe)(rson|ople)$" "i") "$1rson"]
+           [(js/RegExp. "(child)ren$" "i") "$1"]
+           [(js/RegExp. "(eau)x?$" "i") "$1"]
+           [(js/RegExp. "men$" "i") "man"]]]
+    (add-singular-rule! rule repl)))
+
+(defn- init-uncountables! []
+  (doseq [w
+          ["adulthood" "advice" "agenda" "aid" "aircraft" "alcohol" "ammo"
+           "analytics" "anime" "athletics" "audio" "bison" "blood" "bream"
+           "buffalo" "butter" "carp" "cash" "chassis" "chess" "clothing" "cod"
+           "commerce" "cooperation" "corps" "debris" "diabetes" "digestion" "elk"
+           "energy" "equipment" "excretion" "expertise" "firmware" "flounder"
+           "fun" "gallows" "garbage" "graffiti" "hardware" "headquarters" "health"
+           "herpes" "highjinks" "homework" "housework" "information" "jeans"
+           "justice" "kudos" "labour" "literature" "machinery" "mackerel" "mail"
+           "media" "mews" "moose" "music" "mud" "manga" "news" "only" "personnel"
+           "pike" "plankton" "pliers" "police" "pollution" "premises" "rain"
+           "research" "rice" "salmon" "scissors" "series" "sewage" "shambles"
+           "shrimp" "software" "staff" "swine" "tennis" "traffic"
+           "transportation" "trout" "tuna" "wealth" "welfare" "whiting"
+           "wildebeest" "wildlife" "you"]]
+    (add-uncountable-rule! w))
+  (doseq [re [(js/RegExp. "pok[eé]mon$" "i")
+              (js/RegExp. "[^aeiou]ese$" "i")
+              (js/RegExp. "deer$" "i")
+              (js/RegExp. "fish$" "i")
+              (js/RegExp. "measles$" "i")
+              (js/RegExp. "o[iu]s$" "i")
+              (js/RegExp. "pox$" "i")
+              (js/RegExp. "sheep$" "i")]]
+    (add-uncountable-rule! re)))
+
+(init-irregulars!)
+(init-plural-rules!)
+(init-singular-rules!)
+(init-uncountables!)

deps/common/src/logseq/common/util.cljs

@@ -48,16 +48,6 @@
   (when (string? tag-name)
     (not (re-find #"[#\t\r\n]+" tag-name))))
 
-(defn tag?
-  "Whether `s` is a tag."
-  [s]
-  (and (string? s)
-       (string/starts-with? s "#")
-       (or
-        (not (string/includes? s " "))
-        (string/starts-with? s "#[[")
-        (string/ends-with? s "]]"))))
-
 (defn safe-subs
   ([s start]
    (let [c (count s)]
@@ -66,10 +56,6 @@
    (let [c (count s)]
      (subs s (min c start) (min c end)))))
 
-(defn unquote-string
-  [v]
-  (string/trim (subs v 1 (dec (count v)))))
-
 (defn wrapped-by
   [v start end]
   (and (string? v) (>= (count v) 2)
@@ -254,15 +240,6 @@
   [fmt & args]
   (apply gstring/format fmt args))
 
-(defn remove-first [pred coll]
-  ((fn inner [coll]
-     (lazy-seq
-      (when-let [[x & xs] (seq coll)]
-        (if (pred x)
-          xs
-          (cons x (inner xs))))))
-   coll))
-
 (defn concat-without-nil
   [& cols]
   (->> (apply concat cols)
@@ -298,10 +275,6 @@
   [s old-value new-value]
   (string/replace s (re-pattern (str "(?i)" (escape-regex-chars old-value))) new-value))
 
-(defn replace-first-ignore-case
-  [s old-value new-value]
-  (string/replace-first s (re-pattern (str "(?i)" (escape-regex-chars old-value))) new-value))
-
 (defn sort-coll-by-dependency
   "Sort the elements in the collection based on dependencies.
 coll:  [{:id 1 :depend-on 2} {:id 2 :depend-on 3} {:id 3}]

deps/common/src/logseq/common/util/block_ref.cljs

@@ -8,10 +8,6 @@
   (str left-parens right-parens))
 (def block-ref-re #"\(\(([a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12})\)\)")
 
-(defn get-all-block-ref-ids
-  [content]
-  (map second (re-seq block-ref-re content)))
-
 (defn get-block-ref-id
   "Extracts block id from block-ref using regex"
   [s]

deps/common/src/logseq/common/util/macro.cljs

@@ -4,19 +4,12 @@
 
 (def left-braces "Opening characters for macro" "{{")
 (def right-braces "Closing characters for macro" "}}")
-(def query-macro (str left-braces "query"))
 
 (defn macro?
   [*s]
   (when-let [s (and (string? *s) (string/trim *s))]
     (and (string/starts-with? s left-braces) (string/ends-with? s right-braces))))
 
-(defn query-macro?
-  [s]
-  (and (string? s)
-       (string/includes? s (str query-macro " "))
-       (not (string/includes? s (str "`" query-macro)))))
-
 (defn macro-subs
   [macro-content arguments]
   (loop [s macro-content

deps/common/src/logseq/common/util/page_ref.cljs

@@ -1,8 +1,8 @@
 (ns logseq.common.util.page-ref
   "Core vars and util fns for page-ref. Currently this only handles a logseq
   page-ref e.g. [[page name]]"
-  (:require [clojure.string :as string]
-            ["path" :as path]))
+  (:require ["path" :as node-path]
+            [clojure.string :as string]))
 
 (def left-brackets "Opening characters for page-ref" "[[")
 (def right-brackets "Closing characters for page-ref" "]]")
@@ -13,20 +13,13 @@
 (def page-ref-re "Inner capture and doesn't match nested brackets" #"\[\[(.*?)\]\]")
 (def page-ref-without-nested-re "Matches most inner nested brackets" #"\[\[([^\[\]]+)\]\]")
 (def page-ref-any-re "Inner capture that matches anything between brackets" #"\[\[(.*)\]\]")
-(def org-page-ref-re #"\[\[(file:.*)\]\[.+?\]\]")
 (def markdown-page-ref-re #"\[(.*)\]\(file:.*\)")
 
 (defn get-file-basename
   "Returns the basename of a file path. e.g. /a/b/c.md -> c.md"
   [path]
   (when-not (string/blank? path)
-    (.-base (path/parse (string/replace path "+" "/")))))
-
-(defn get-file-rootname
-  "Returns the rootname of a file path. e.g. /a/b/c.md -> c"
-  [path]
-  (when-not (string/blank? path)
-    (.-name (path/parse (string/replace path "+" "/")))))
+    (.-base (node-path/parse (string/replace path "+" "/")))))
 
 (defn page-ref?
   "Determines if string is page-ref. Avoid using with format-specific page-refs e.g. org"
@@ -40,16 +33,12 @@
   (str left-brackets page-name right-brackets))
 
 (defn get-page-name
-  "Extracts page names from format-specific page-refs e.g. org/md specific and
-  logseq page-refs. Only call in contexts where format-specific page-refs are
-  used. For logseq page-refs use page-ref/get-page-name"
+  "Extracts page names from md page-refs or logseq page-refs. Only call in
+  contexts where md page-refs are used"
   [s]
   (and (string? s)
        (or (when-let [[_ label _path] (re-matches markdown-page-ref-re s)]
              (string/trim label))
-           (when-let [[_ path _label] (re-matches org-page-ref-re s)]
-             (some-> (get-file-rootname path)
-                     (string/replace "." "/")))
            (-> (re-matches page-ref-any-re s)
                second))))
 

deps/common/test/logseq/common/config_test.cljc

@@ -1,9 +1,5 @@
 (ns logseq.common.config-test
-  (:require ["fs" :as fs]
-            ["path" :as node-path]
-            #?(:org.babashka/nbb [nbb.classpath :as cp])
-            [cljs.test :refer [deftest is]]
-            [clojure.string :as string]
+  (:require [cljs.test :refer [deftest is]]
             [logseq.common.config :as common-config]))
 
 (deftest remove-hidden-files
@@ -23,23 +19,4 @@
             (map #(str "/" %) files)
             {:hidden ["script" "/dev"]}
             identity))
-        "Removes hidden files if they start with '/'")))
-
-(defn find-on-classpath [classpath rel-path]
-  (some (fn [dir]
-          (let [f (node-path/join dir rel-path)]
-            (when (fs/existsSync f) f)))
-        (string/split classpath #":")))
-
-#?(:org.babashka/nbb
-   (deftest create-config-for-db-graph
-     (let [original-config (some-> (find-on-classpath (cp/get-classpath) "templates/config.edn") fs/readFileSync str)
-           _ (assert original-config "config.edn must not be blank")
-           migrated-config (common-config/create-config-for-db-graph original-config)
-           forbidden-kws-regex (re-pattern (str (string/join "|" (keys common-config/file-only-config))))]
-      ;;  (println migrated-config)
-       (is (not (string/includes? migrated-config "== FILE ONLY CONFIG"))
-           "No longer includes file config header")
-       (assert (re-find forbidden-kws-regex original-config) "File config keys present in original config")
-       (is (not (re-find forbidden-kws-regex migrated-config))
-           "File config keys no longer present in migrated config"))))
+        "Removes hidden files if they start with '/'")))

deps/common/test/logseq/common/path_test.cljs

@@ -12,21 +12,6 @@
   (is (= ["some-song" ""] (path/split-ext "some-song")))
   (is (= ["some-file.edn" "txt"] (path/split-ext "some-file.edn.txt"))))
 
-(deftest safe-file-name?
-  (testing "safe-file-name"
-    (is (path/safe-filename? "foo"))
-    (is (path/safe-filename? "foo bar"))
-    (is (path/safe-filename? "foo-bar"))
-    (is (path/safe-filename? "foo_bar"))
-    (is (path/safe-filename? "foo.bar"))
-    (is (path/safe-filename? "foo..bar"))
-    (is (path/safe-filename? "foo...bar"))
-    (is (not (path/safe-filename? "foo/bar")))
-    (is (not (path/safe-filename? "foo?bar")))
-    (is (not (path/safe-filename? "foo<bar")))
-    (is (not (path/safe-filename? "foo>bar")))))
-
-
 (deftest path-join
   (testing "path-join"
     (is (= "foo/bar" (path/path-join "foo" "bar")))

deps/db/.carve/config.edn

@@ -4,7 +4,6 @@
                   logseq.db.common.sqlite-cli
                   logseq.db.frontend.property
                   logseq.db.frontend.property.build
-                  logseq.db.common.property-util
                   logseq.db.frontend.content
                   logseq.db.common.order
                   logseq.db.sqlite.create-graph

deps/db/.carve/ignore

@@ -1,8 +1,4 @@
 ;; API
-logseq.db.file-based.rules/rules
-;; API
-logseq.db.file-based.schema/retract-attributes
-;; API
 logseq.db.frontend.rules/db-query-dsl-rules
 ;; API
 logseq.db.frontend.rules/extract-rules
@@ -51,6 +47,6 @@ logseq.db.sqlite.gc/gc-kvs-table-node-version!
 ;; API
 logseq.db.sqlite.gc/ensure-no-garbage
 ;; API
-logseq.db.common.entity-util/entity->map
+logseq.db.frontend.entity-util/entity->map
 ;; documenting keywords
 logseq.db.frontend.kv-entity/kv-entities

deps/db/.clj-kondo/config.edn

@@ -10,9 +10,7 @@
   {:aliases {clojure.string string
              logseq.db ldb
              logseq.db.common.entity-plus entity-plus
-             logseq.db.common.entity-util common-entity-util
              logseq.db.common.order db-order
-             logseq.db.common.property-util db-property-util
              logseq.db.common.sqlite common-sqlite
              logseq.db.common.view db-view
              logseq.db.frontend.content db-content
@@ -23,9 +21,6 @@
              logseq.db.frontend.property db-property
              logseq.db.frontend.property.build db-property-build
              logseq.db.frontend.property.type db-property-type
-             logseq.db.file-based.rules file-rules
-             logseq.db.file-based.schema file-schema
-             logseq.db.file-based.entity-util file-entity-util
              logseq.db.frontend.rules rules
              logseq.db.frontend.schema db-schema
              logseq.db.frontend.validate db-validate

deps/db/README.md

@@ -3,20 +3,21 @@
 This library provides an API to the
 frontend([datascript](https://github.com/tonsky/datascript)) and
 backend([SQLite](https://www.sqlite.org/index.html)) databases from the Logseq
-app and the CLI. The majority of this library is focused on supporting DB graphs
-but there are a few namespaces that support file graphs. This library is
-compatible with ClojureScript and with
+app and the CLI. This library defines the core schema and fns for DB graphs.
+There are a few namespaces that also support file graphs for the graph-parser.
+This library is compatible with ClojureScript and with
 [nbb-logseq](https://github.com/logseq/nbb-logseq) to respectively provide
 frontend and commandline functionality.
 
 ## API
 
 This library is under the parent namespace `logseq.db`. It provides the following namespaces:
-* `logseq.db` - main entry point serving both file and DB graphs
+* `logseq.db` - main public ns. Most often used by frontend
 * `logseq.db.frontend.*` - frontend namespaces for DB graphs
 * `logseq.db.sqlite.*` - backend/sqlite namespaces for DB graphs
-* `logseq.db.file-based.*` - namespaces for file graphs, mostly old namespaces
-* `logseq.db.common.*` - namespaces for both file and DB graphs
+* `logseq.db.common.*` - frontend and backend namespaces for DB graphs
+
+The following namespaces are used with file graphs via the graph-parser: `logseq.db.common.order, logseq.db.frontend.entity-util and logseq.db.common.entity-plus`.
 
 ## Usage
 
@@ -27,7 +28,7 @@ See the frontend for example usage.
 This follows the practices that [the Logseq frontend
 follows](/docs/dev-practices.md). Most of the same linters are used, with
 configurations that are specific to this library. See [this library's CI
-file](/.github/workflows/db.yml) for linting examples.
+file](/.github/workflows/deps-db.yml) for linting examples.
 
 ### Setup
 

deps/db/bb.edn

@@ -24,14 +24,11 @@
 
   lint:rules
   {:requires ([logseq.bb-tasks.lint.datalog :as datalog]
-              [logseq.db.file-based.rules :as file-rules]
               [logseq.db.frontend.rules :as rules])
    :doc "Lint datalog rules for parsability and unbound variables"
    :task (datalog/lint-rules
           (set
-           (concat (mapcat val (merge file-rules/rules (dissoc rules/rules :self-ref)))
-                   ;; TODO: Update linter to handle false positive on ?str-val for :property
-                   (rules/extract-rules (dissoc file-rules/query-dsl-rules :property))
+           (concat (mapcat val (dissoc rules/rules :self-ref))
                    ;; TODO: Update linter to handle false positive on :task, :priority, :*property* rules
                    (rules/extract-rules (dissoc rules/db-query-dsl-rules
                                                 :task :priority

deps/db/script/create_graph/inferred.edn

@@ -1,6 +1,6 @@
 ;; Script that generates classes and properties for a demo of inferring properties.
 ;; To generate this graph:
-;; bb dev:db-create inferred deps/db/create_graph/inferred.edn
+;; bb dev:create inferred deps/db/create_graph/inferred.edn
 ;;
 ;; To try the demo in the UI, in any page type:
 ;; - Good Will Hunting #Movie #Ben-Affleck

deps/db/script/query.cljs

@@ -39,9 +39,7 @@
         results (if (:entity options)
                   (map #(when-let [ent (d/entity @conn
                                                  (if (string? %) (edn/read-string %) %))]
-                          (cond-> (into {:db/id (:db/id ent)} ent)
-                            (seq (:block/properties ent))
-                            (update :block/properties (fn [props] (map (fn [m] (into {} m)) props)))))
+                          (into {:db/id (:db/id ent)} ent))
                        (:entity options))
                   ;; assumes no :in are in queries
                   (let [query (into (edn/read-string (first args'')) [:in '$ '%])

deps/db/src/logseq/db.cljs

@@ -1,7 +1,5 @@
 (ns logseq.db
-  "Main namespace for db fns that handles DB and file graphs. For db graph only
-  fns, use logseq.db.frontend.db and for file graph only fns, use
-  logseq.graph-parser.db"
+  "Main namespace for db fns. All fns are only for DB graphs"
   (:require [clojure.set :as set]
             [clojure.string :as string]
             [clojure.walk :as walk]
@@ -9,18 +7,16 @@
             [datascript.core :as d]
             [datascript.impl.entity :as de]
             [logseq.common.config :as common-config]
+            [logseq.common.plural :as common-plural]
             [logseq.common.util :as common-util]
             [logseq.common.uuid :as common-uuid]
             [logseq.db.common.delete-blocks :as delete-blocks] ;; Load entity extensions
             [logseq.db.common.entity-plus :as entity-plus]
-            [logseq.db.common.entity-util :as common-entity-util]
             [logseq.db.common.initial-data :as common-initial-data]
-            [logseq.db.file-based.schema :as file-schema]
             [logseq.db.frontend.class :as db-class]
             [logseq.db.frontend.db :as db-db]
             [logseq.db.frontend.entity-util :as entity-util]
             [logseq.db.frontend.property :as db-property]
-            [logseq.db.frontend.schema :as db-schema]
             [logseq.db.frontend.validate :as db-validate]
             [logseq.db.sqlite.util :as sqlite-util])
   (:refer-clojure :exclude [object?]))
@@ -223,13 +219,12 @@
                         remove-conflict-datoms)]
           (transact! conn tx-data' tx-meta))))))
 
-(def page? common-entity-util/page?)
+(def page? entity-util/page?)
 (def internal-page? entity-util/internal-page?)
 (def class? entity-util/class?)
 (def property? entity-util/property?)
 (def closed-value? entity-util/closed-value?)
-(def whiteboard? common-entity-util/whiteboard?)
-(def journal? common-entity-util/journal?)
+(def journal? entity-util/journal?)
 (def hidden? entity-util/hidden?)
 (def object? entity-util/object?)
 (def asset? entity-util/asset?)
@@ -329,44 +324,37 @@
 
 (def get-first-page-by-name common-initial-data/get-first-page-by-name)
 
-(def db-based-graph? entity-plus/db-based-graph?)
-
 (defn page-exists?
-  "Returns truthy value if page exists.
-   For db graphs, returns all page db ids that given title and one of the given `tags`.
-   For file graphs, returns page db/id vector if it exists"
+  "Returns all page db ids that given title and one of the given `tags`."
   [db page-name tags]
   (when page-name
-    (if (db-based-graph? db)
-      (let [tags' (if (coll? tags) (set tags) #{tags})]
+    (let [tags' (if (coll? tags) (set tags) #{tags})]
         ;; Classes and properties are case sensitive and can be looked up
         ;; as such in case-sensitive contexts e.g. no Page
-        (if (and (seq tags') (every? #{:logseq.class/Tag :logseq.class/Property} tags'))
-          (seq
-           (d/q
-            '[:find [?p ...]
-              :in $ ?name [?tag-ident ...]
-              :where
-              [?p :block/title ?name]
-              [?p :block/tags ?tag]
-              [?tag :db/ident ?tag-ident]]
-            db
-            page-name
-            tags'))
-          ;; TODO: Decouple db graphs from file specific :block/name
-          (seq
-           (d/q
-            '[:find [?p ...]
-              :in $ ?name [?tag-ident ...]
-              :where
-              [?p :block/name ?name]
-              [?p :block/tags ?tag]
-              [?tag :db/ident ?tag-ident]]
-            db
-            (common-util/page-name-sanity-lc page-name)
-            tags'))))
-      (when-let [id (:db/id (d/entity db [:block/name (common-util/page-name-sanity-lc page-name)]))]
-        [id]))))
+      (if (and (seq tags') (every? #{:logseq.class/Tag :logseq.class/Property} tags'))
+        (seq
+         (d/q
+          '[:find [?p ...]
+            :in $ ?name [?tag-ident ...]
+            :where
+            [?p :block/title ?name]
+            [?p :block/tags ?tag]
+            [?tag :db/ident ?tag-ident]]
+          db
+          page-name
+          tags'))
+        ;; TODO: Decouple db graphs from file specific :block/name
+        (seq
+         (d/q
+          '[:find [?p ...]
+            :in $ ?name [?tag-ident ...]
+            :where
+            [?p :block/name ?name]
+            [?p :block/tags ?tag]
+            [?tag :db/ident ?tag-ident]]
+          db
+          (common-util/page-name-sanity-lc page-name)
+          tags'))))))
 
 (defn get-page
   "Get a page given its unsanitized name or uuid"
@@ -440,8 +428,6 @@
                                     (= 1 (count children))
                                     (contains? #{"" "-" "*"} (string/trim (:block/title first-child))))))
                                 (not (contains? built-in-pages name'))
-                                (not (whiteboard? page))
-                                (not (:block/_namespace page))
                                 (not (property? page))
                                  ;; a/b/c might be deleted but a/b/c/d still exists (for backward compatibility)
                                 (not (and (string/includes? name' "/")
@@ -640,30 +626,18 @@
 
 (defn get-pages-relation
   [db with-journal?]
-  (if (entity-plus/db-based-graph? db)
-    (let [q (if with-journal?
-              '[:find ?p ?ref-page
-                :where
-                [?block :block/page ?p]
-                [?block :block/refs ?ref-page]]
-              '[:find ?p ?ref-page
-                :where
-                [?block :block/page ?p]
-                [?p :block/tags]
-                (not [?p :block/tags :logseq.class/Journal])
-                [?block :block/refs ?ref-page]])]
-      (d/q q db))
-    (let [q (if with-journal?
-              '[:find ?p ?ref-page
-                :where
-                [?block :block/page ?p]
-                [?block :block/refs ?ref-page]]
-              '[:find ?p ?ref-page
-                :where
-                [?block :block/page ?p]
-                (not [?p :block/type "journal"])
-                [?block :block/refs ?ref-page]])]
-      (d/q q db))))
+  (let [q (if with-journal?
+            '[:find ?p ?ref-page
+              :where
+              [?block :block/page ?p]
+              [?block :block/refs ?ref-page]]
+            '[:find ?p ?ref-page
+              :where
+              [?block :block/page ?p]
+              [?p :block/tags]
+              (not [?p :block/tags :logseq.class/Journal])
+              [?block :block/refs ?ref-page]])]
+    (d/q q db)))
 
 (defn get-all-tagged-pages
   [db]
@@ -672,13 +646,6 @@
          [?page :block/tags ?tag]]
        db))
 
-(defn get-schema
-  "Returns schema for given repo"
-  [repo]
-  (if (sqlite-util/db-based-graph? repo)
-    db-schema/schema
-    file-schema/schema))
-
 (defn page-in-library?
   "Check whether a `page` exists on the Library page"
   [db page]
@@ -694,3 +661,69 @@
           (recur (:block/parent parent)))))))
 
 (def get-class-title-with-extends db-db/get-class-title-with-extends)
+
+(defn- bidirectional-property-attr?
+  [db attr]
+  (when (qualified-keyword? attr)
+    (let [attr-ns (namespace attr)]
+      (and (or (db-property/user-property-namespace? attr-ns)
+               (db-property/plugin-property? attr))
+           (when-let [property (d/entity db attr)]
+             (= :db.type/ref (:db/valueType property)))))))
+
+(defn- get-ea-by-v
+  [db v]
+  (d/q '[:find ?e ?a
+         :in $ ?v
+         :where
+         [?e ?a ?v]
+         [?ea :db/ident ?a]
+         [?ea :logseq.property/classes]]
+       db
+       v))
+
+(defn get-bidirectional-properties
+  "Given a target entity id, returns a seq of maps with:
+   * :class - class entity
+   * :title - pluralized class title
+   * :entities - node entities that reference the target via ref properties"
+  [db target-id]
+  (when (and db target-id (d/entity db target-id))
+    (let [add-entity
+          (fn [acc class-id entity]
+            (if class-id
+              (update acc class-id (fnil conj #{}) entity)
+              acc))]
+      (->> (get-ea-by-v db target-id)
+           (keep (fn [[e a]]
+                   (when (bidirectional-property-attr? db a)
+                     (when-let [entity (d/entity db e)]
+                       (when (and (not= (:db/id entity) target-id)
+                                  (not (entity-util/class? entity))
+                                  (not (entity-util/property? entity)))
+                         (let [classes (filter entity-util/class? (:block/tags entity))]
+                           (when (seq classes)
+                             (keep (fn [class-ent]
+                                     (when-not (built-in? class-ent)
+                                       [(:db/id class-ent) entity]))
+                                   classes))))))))
+           (mapcat identity)
+           (reduce (fn [acc [class-ent entity]]
+                     (add-entity acc class-ent entity))
+                   {})
+           (keep (fn [[class-id entities]]
+                   (let [class (d/entity db class-id)]
+                     (when (true? (:logseq.property.class/enable-bidirectional? class))
+                       (let [custom-title (when-let [custom (:logseq.property.class/bidirectional-property-title class)]
+                                            (if (string? custom)
+                                              custom
+                                              (db-property/property-value-content custom)))
+                             title (if (string/blank? custom-title)
+                                     (common-plural/plural (:block/title class))
+                                     custom-title)]
+                         {:title title
+                          :class (-> (into {} class)
+                                     (assoc :db/id (:db/id class)))
+                          :entities (->> entities
+                                         (sort-by :block/created-at))})))))
+           (sort-by (comp :block/created-at :class))))))

deps/db/src/logseq/db/common/delete_blocks.cljs

@@ -1,12 +1,10 @@
 (ns logseq.db.common.delete-blocks
-  "For file and DB graphs, provides fn to handle any deletion to occur per ldb/transact!"
+  "Provides fn to handle any deletion to occur per ldb/transact!"
   (:require [clojure.string :as string]
             [datascript.core :as d]
             [logseq.common.util :as common-util]
             [logseq.common.util.block-ref :as block-ref]
             [logseq.common.util.page-ref :as page-ref]
-            [logseq.db.common.entity-plus :as entity-plus]
-            [logseq.db.common.entity-util :as common-entity-util]
             [logseq.db.frontend.entity-util :as entity-util]))
 
 (defn- replace-ref-with-deleted-block-title
@@ -55,7 +53,7 @@
                                                     (contains? #{:db.fn/retractEntity :db/retractEntity} (first tx)))
                                            (second tx))) txs)
                                  (filter (fn [id]
-                                           (not (common-entity-util/page? (d/entity db id))))))]
+                                           (not (entity-util/page? (d/entity db id))))))]
     (when (seq retracted-block-ids)
       (let [retracted-blocks (map #(d/entity db %) retracted-block-ids)
             retracted-tx (build-retracted-tx retracted-blocks)

deps/db/src/logseq/db/common/entity_plus.cljc

@@ -1,5 +1,7 @@
 (ns logseq.db.common.entity-plus
   "Add map ops such as assoc/dissoc to datascript Entity.
+   The db-based-graph? checks are still in this ns because it's unclear which of these
+   lower level fns are still used by the graph-parser.
 
    NOTE: This doesn't work for nbb/sci yet because of https://github.com/babashka/sci/issues/639"
   ;; Disable clj linters since we don't support clj
@@ -17,12 +19,9 @@
 
 (def nil-db-ident-entities
   "No such entities with these :db/ident, but `(d/entity <db> <ident>)` has been called somewhere."
-  #{:block/tx-id :block/uuid :block/journal-day :block/_refs :block/level :block/heading-level :block/warning
-    ;; File graph only attributes. Can these be removed if this is only called in db graphs?
-    :block/pre-block? :block/scheduled :block/deadline :block/type :block/name :block/marker
-
-    :block.temp/ast-title
-    :block.temp/load-status :block.temp/has-children? :block.temp/ast-body
+  #{:block/tx-id :block/uuid :block/journal-day :block/_refs :block/level :block/heading-level
+    :block/warning :block/name
+    :block.temp/ast-title :block.temp/load-status :block.temp/has-children? :block.temp/ast-body
 
     :db/valueType :db/cardinality :db/ident :db/index
 
@@ -78,7 +77,8 @@
   (Entity. db e (volatile! false) (volatile! {})))
 
 (defn db-based-graph?
-  "Whether the current graph is db-only"
+  "Whether the current graph is db-only. This should only be used in contexts
+   where both DB and file graph databases are possible e.g. graph-parser"
   [db]
   (when db
     (identical? "db" (:kv/value (entity-memoized db :logseq.kv/db-type)))))

deps/db/src/logseq/db/common/entity_util.cljs

@@ -1,26 +0,0 @@
-(ns logseq.db.common.entity-util
-  "Lower level entity util fns for DB and file graphs"
-  (:require [datascript.impl.entity :as de]
-            [logseq.db.file-based.entity-util :as file-entity-util]
-            [logseq.db.frontend.entity-util :as entity-util]))
-
-(defn whiteboard?
-  [entity]
-  (or (entity-util/whiteboard? entity)
-      (file-entity-util/whiteboard? entity)))
-
-(defn journal?
-  [entity]
-  (or (entity-util/journal? entity)
-      (file-entity-util/journal? entity)))
-
-(defn page?
-  [entity]
-  (or (entity-util/page? entity)
-      (file-entity-util/page? entity)))
-
-(defn entity->map
-  "Convert a db Entity to a map"
-  [e]
-  (assert (de/entity? e))
-  (assoc (into {} e) :db/id (:db/id e)))

deps/db/src/logseq/db/common/initial_data.cljs

@@ -6,8 +6,6 @@
             [logseq.common.config :as common-config]
             [logseq.common.util :as common-util]
             [logseq.common.util.date-time :as date-time-util]
-            [logseq.db.common.entity-plus :as entity-plus]
-            [logseq.db.common.entity-util :as common-entity-util]
             [logseq.db.common.order :as db-order]
             [logseq.db.frontend.class :as db-class]
             [logseq.db.frontend.db :as db-db]
@@ -29,7 +27,7 @@
   (->> (d/datoms db :avet :block/title page-name)
        (filter (fn [d]
                  (let [e (d/entity db (:e d))]
-                   (common-entity-util/page? e))))
+                   (entity-util/page? e))))
        (map :e)
        sort
        first))
@@ -98,7 +96,7 @@
                           (let [e (d/entity db eid)]
                             (when (or include-collapsed-children?
                                       (not (:block/collapsed? e))
-                                      (common-entity-util/page? e))
+                                      (entity-util/page? e))
                               (:block/_parent e)))) eids-to-expand)
                 ids-to-add (keep :db/id children)]
             (vswap! seen (partial apply conj) ids-to-add)
@@ -165,23 +163,18 @@
 (defn hidden-ref?
   "Whether ref-block (for block with the `id`) should be hidden."
   [db ref-block id]
-  (let [db-based? (entity-plus/db-based-graph? db)]
-    (if db-based?
-      (let [entity (d/entity db id)]
-        (or
-         (= (:db/id ref-block) id)
-         (= id (:db/id (:block/page ref-block)))
-         (= id (:db/id (:logseq.property/view-for ref-block)))
-         (entity-util/hidden? (:block/page ref-block))
-         (entity-util/hidden? ref-block)
-         (and (entity-util/class? entity)
-              (let [children (db-class/get-structured-children db id)
-                    class-ids (set (conj children id))]
-                (some class-ids (map :db/id (:block/tags ref-block)))))
-         (some? (get ref-block (:db/ident entity)))))
-      (or
-       (= (:db/id ref-block) id)
-       (= id (:db/id (:block/page ref-block)))))))
+  (let [entity (d/entity db id)]
+    (or
+     (= (:db/id ref-block) id)
+     (= id (:db/id (:block/page ref-block)))
+     (= id (:db/id (:logseq.property/view-for ref-block)))
+     (entity-util/hidden? (:block/page ref-block))
+     (entity-util/hidden? ref-block)
+     (and (entity-util/class? entity)
+          (let [children (db-class/get-structured-children db id)
+                class-ids (set (conj children id))]
+            (some class-ids (map :db/id (:block/tags ref-block)))))
+     (some? (get ref-block (:db/ident entity))))))
 
 (defn get-block-refs
   [db id]
@@ -263,7 +256,7 @@
          (keep (fn [d]
                  (when (<= (:v d) today)
                    (let [e (d/entity db (:e d))]
-                     (when (and (common-entity-util/journal? e) (:db/id e))
+                     (when (and (entity-util/journal? e) (:db/id e))
                        e))))))))
 
 (defn- get-structured-datoms
@@ -311,7 +304,7 @@
                                (string/blank? title)))
                  (let [e (d/entity db (:e datom))]
                    (when (and
-                          (common-entity-util/page? e)
+                          (entity-util/page? e)
                           (not (entity-util/hidden? e)))
                      e))))))
      (take 15))))
@@ -325,8 +318,7 @@
      (d/datoms db :avet :logseq.property.user/email))))
 
 (defn get-initial-data
-  "Returns current database schema and initial data.
-   NOTE: This fn is called by DB and file graphs"
+  "Returns current database schema and initial data"
   [db]
   (let [_ (reset! db-order/*max-key (db-order/get-max-order db))
         schema (:schema db)
@@ -336,6 +328,7 @@
                        [:logseq.kv/db-type
                         :logseq.kv/schema-version
                         :logseq.kv/graph-uuid
+                        :logseq.kv/local-graph-uuid
                         :logseq.kv/graph-rtc-e2ee?
                         :logseq.kv/latest-code-lang
                         :logseq.kv/graph-backup-folder

deps/db/src/logseq/db/common/property_util.cljs

@@ -1,44 +0,0 @@
-(ns logseq.db.common.property-util
-  "Property related util fns. Fns used in both DB and file graphs should go here"
-  (:require [datascript.core :as d]
-            [logseq.db.frontend.property :as db-property]
-            [logseq.db.sqlite.util :as sqlite-util]))
-
-(defn get-file-pid
-  "Gets file graph property id given the db graph ident"
-  [db-ident]
-  ;; Map of unique cases where the db graph keyword name is different than the file graph id
-  (let [unique-file-ids {:logseq.property/order-list-type :logseq.order-list-type
-                         :logseq.property.tldraw/page :logseq.tldraw.page
-                         :logseq.property.tldraw/shape :logseq.tldraw.shape
-                         :logseq.property/publishing-public? :public}]
-    (or (get unique-file-ids db-ident)
-        (keyword (name db-ident)))))
-
-;; TODO: replace repo with db later to remove this fn
-(defn get-pid
-  "Get a built-in property's id (keyword name for file graph and db-ident for db
-  graph) given its db-ident. No need to use this fn in a db graph only context"
-  [repo db-ident]
-  (if (sqlite-util/db-based-graph? repo)
-    db-ident
-    (get-file-pid db-ident)))
-
-(defn lookup
-  "Get the property value by a built-in property's db-ident from coll. For file and db graphs"
-  [repo block db-ident]
-  (if (sqlite-util/db-based-graph? repo)
-    (let [val (get block db-ident)]
-      (if (db-property/built-in-has-ref-value? db-ident) (db-property/property-value-content val) val))
-    (get (:block/properties block) (get-pid repo db-ident))))
-
-(defn get-block-property-value
-  "Get the value of built-in block's property by its db-ident"
-  [repo db block db-ident]
-  (when db
-    (let [block (or (d/entity db (:db/id block)) block)]
-      (lookup repo block db-ident))))
-
-(defn shape-block?
-  [repo db block]
-  (= :whiteboard-shape (get-block-property-value repo db block :logseq.property/ls-type)))

deps/db/src/logseq/db/common/reference.cljs

@@ -1,12 +1,9 @@
 (ns logseq.db.common.reference
   "References"
-  (:require [cljs.reader :as reader]
-            [clojure.set :as set]
+  (:require [clojure.set :as set]
             [clojure.string :as string]
             [datascript.core :as d]
-            [logseq.common.log :as log]
             [logseq.db :as ldb]
-            [logseq.db.common.entity-plus :as entity-plus]
             [logseq.db.common.initial-data :as common-initial-data]
             [logseq.db.frontend.class :as db-class]
             [logseq.db.frontend.entity-util :as entity-util]))
@@ -240,34 +237,17 @@
 ;; -----------------------------------------------------------------------------
 
 (defn get-filters
-  [db page]
-  (let [db-based? (entity-plus/db-based-graph? db)]
-    (if db-based?
-      (let [included-pages (:logseq.property.linked-references/includes page)
-            excluded-pages (:logseq.property.linked-references/excludes page)]
-        (when (or (seq included-pages) (seq excluded-pages))
-          {:included included-pages
-           :excluded excluded-pages}))
-      (let [k :filters
-            properties (:block/properties page)
-            properties-str (or (get properties k) "{}")]
-        (try (let [result (reader/read-string properties-str)]
-               (when (seq result)
-                 (let [excluded-pages (->> (filter #(false? (second %)) result)
-                                           (keep first)
-                                           (keep #(ldb/get-page db %)))
-                       included-pages (->> (filter #(true? (second %)) result)
-                                           (keep first)
-                                           (keep #(ldb/get-page db %)))]
-                   {:included included-pages
-                    :excluded excluded-pages})))
-             (catch :default e
-               (log/error :syntax/filters e)))))))
+  [page]
+  (let [included-pages (:logseq.property.linked-references/includes page)
+        excluded-pages (:logseq.property.linked-references/excludes page)]
+    (when (or (seq included-pages) (seq excluded-pages))
+      {:included included-pages
+       :excluded excluded-pages})))
 
 (defn get-linked-references [db id]
   (let [entity       (d/entity db id)
         ids          (set (cons id (ldb/get-block-alias db id)))
-        page-filters (get-filters db entity)
+        page-filters (get-filters entity)
         excludes     (map :db/id (:excluded page-filters))
         includes     (map :db/id (:included page-filters))
         has-filters? (or (seq excludes) (seq includes))

deps/db/src/logseq/db/common/sqlite.cljs

@@ -1,10 +1,8 @@
 (ns logseq.db.common.sqlite
-  "Provides common sqlite util fns for file and DB graphs. These fns work on
-  browser and node"
+  "Provides common sqlite util fns that work on browser and node"
   (:require ["path" :as node-path]
             [clojure.string :as string]
             [datascript.core :as d]
-            [logseq.common.config :as common-config]
             [logseq.db.sqlite.util :as sqlite-util]))
 
 (defn create-kvs-table!
@@ -18,22 +16,13 @@
   (or (d/restore-conn storage)
       (d/create-conn schema {:storage storage})))
 
-(defn local-file-based-graph?
-  [s]
-  (and (string? s)
-       (string/starts-with? s common-config/file-version-prefix)))
-
 (defn sanitize-db-name
   [db-name]
-  (if (string/starts-with? db-name common-config/file-version-prefix)
-    (-> db-name
-        (string/replace ":" "+3A+")
-        (string/replace "/" "++"))
-    (-> db-name
-        (string/replace sqlite-util/db-version-prefix "")
-        (string/replace "/" "_")
-        (string/replace "\\" "_")
-        (string/replace ":" "_"))));; windows
+  (-> db-name
+      (string/replace sqlite-util/db-version-prefix "")
+      (string/replace "/" "_")
+      (string/replace "\\" "_")
+      (string/replace ":" "_")));; windows
 
 (defn get-db-full-path
   [graphs-dir db-name]

deps/db/src/logseq/db/common/sqlite_cli.cljs

@@ -1,24 +1,15 @@
 (ns ^:node-only logseq.db.common.sqlite-cli
-  "Primary ns to interact with DB files for DB and file graphs with node.js based CLIs"
+  "Primary ns to interact with DB files with node.js based CLIs"
   (:require ["better-sqlite3" :as sqlite3]
-            ["fs" :as fs]
             ["os" :as os]
             ["path" :as node-path]
             [cljs-bean.core :as bean]
             [clojure.string :as string]
             [datascript.storage :refer [IStorage]]
             [logseq.db.common.sqlite :as common-sqlite]
-            [logseq.db.file-based.schema :as file-schema]
             [logseq.db.frontend.schema :as db-schema]
             [logseq.db.sqlite.util :as sqlite-util]))
 
-;; Should this check directory name instead if file graphs also
-;; have this file?
-(defn db-graph-directory?
-  "Returns boolean indicating if the given directory is a DB graph"
-  [graph-dir]
-  (fs/existsSync (node-path/join graph-dir "db.sqlite")))
-
 ;; Reference same sqlite default class in cljs + nbb without needing .cljc
 (def sqlite (if (find-ns 'nbb.core) (aget sqlite3 "default") sqlite3))
 
@@ -76,18 +67,14 @@
   ([db-full-path]
    (open-sqlite-datascript! nil db-full-path))
   ([graphs-dir db-name]
-   (let [[base-name db-full-path]
+   (let [db-full-path
          (if (nil? graphs-dir)
-           [(node-path/basename db-name) db-name]
-           [db-name (second (common-sqlite/get-db-full-path graphs-dir db-name))])
-         db (new sqlite db-full-path nil)
-        ;; For both desktop and CLI, only file graphs have db-name that indicate their db type
-         schema (if (common-sqlite/local-file-based-graph? base-name)
-                  file-schema/schema
-                  db-schema/schema)]
+           db-name
+           (second (common-sqlite/get-db-full-path graphs-dir db-name)))
+         db (new sqlite db-full-path nil)]
      (common-sqlite/create-kvs-table! db)
      (let [storage (new-sqlite-storage db)
-           conn (common-sqlite/get-storage-conn storage schema)]
+           conn (common-sqlite/get-storage-conn storage db-schema/schema)]
        {:sqlite db
         :conn conn}))))
 

deps/db/src/logseq/db/file_based/entity_util.cljs

@@ -1,14 +0,0 @@
-(ns logseq.db.file-based.entity-util
-  "Lower level entity util fns for file graphs")
-
-(defn whiteboard?
-  [entity]
-  (identical? "whiteboard" (:block/type entity)))
-
-(defn journal?
-  [entity]
-  (identical? "journal" (:block/type entity)))
-
-(defn page?
-  [entity]
-  (contains? #{"page" "journal" "whiteboard"} (:block/type entity)))

deps/db/src/logseq/db/file_based/rules.cljc

@@ -1,92 +0,0 @@
-(ns ^:bb-compatible logseq.db.file-based.rules
-  "Datalog rules for file graphs")
-
-(def rules
-  "File graph rules used in db.model queries"
-  {:namespace
-   '[[(namespace ?p ?c)
-      [?c :block/namespace ?p]]
-     [(namespace ?p ?c)
-      [?t :block/namespace ?p]
-      (namespace ?t ?c)]]})
-
-(def ^:large-vars/data-var query-dsl-rules
-  "Rules used by frontend.db.query-dsl for file graphs. The symbols ?b and ?p
-  respectively refer to block and page. Do not alter them as they are
-  programmatically built by the query-dsl ns"
-  {:page-property
-   '[(page-property ?p ?key ?val)
-     [?p :block/name]
-     [?p :block/properties ?prop]
-     [(get ?prop ?key) ?v]
-     (or [(= ?v ?val)] [(contains? ?v ?val)])]
-
-   :has-page-property
-   '[(has-page-property ?p ?key)
-     [?p :block/name]
-     [?p :block/properties ?prop]
-     [(get ?prop ?key)]]
-
-   :task
-   '[(task ?b ?markers)
-     [?b :block/marker ?marker]
-     [(contains? ?markers ?marker)]]
-
-   :priority
-   '[(priority ?b ?priorities)
-     [?b :block/priority ?priority]
-     [(contains? ?priorities ?priority)]]
-
-   :page-tags
-   '[(page-tags ?p ?tags)
-     [?p :block/tags ?t]
-     [?t :block/name ?tag]
-     [(contains? ?tags ?tag)]]
-
-   :all-page-tags
-   '[(all-page-tags ?p)
-     [_ :block/tags ?p]]
-
-   :between
-   '[(between ?b ?start ?end)
-     [?b :block/page ?p]
-     [?p :block/type "journal"]
-     [?p :block/journal-day ?d]
-     [(>= ?d ?start)]
-     [(<= ?d ?end)]]
-
-   :has-property
-   '[(has-property ?b ?prop)
-     [?b :block/properties ?bp]
-     [(missing? $ ?b :block/name)]
-     [(get ?bp ?prop)]]
-
-   :block-content
-   '[(block-content ?b ?query)
-     [?b :block/title ?content]
-     [(clojure.string/includes? ?content ?query)]]
-
-   :page
-   '[(page ?b ?page-name)
-     [?b :block/page ?bp]
-     [?bp :block/name ?page-name]]
-
-   :namespace
-   '[(namespace ?p ?namespace)
-     [?p :block/namespace ?parent]
-     [?parent :block/name ?namespace]]
-
-   :property
-   '[(property ?b ?key ?val)
-     [?b :block/properties ?prop]
-     [(missing? $ ?b :block/name)]
-     [(get ?prop ?key) ?v]
-     [(str ?val) ?str-val]
-     (or [(= ?v ?val)]
-         [(contains? ?v ?val)]
-         ;; For integer pages that aren't strings
-         [(contains? ?v ?str-val)])]
-
-   :page-ref
-   '[(page-ref ?b ?ref)
-     (has-ref ?b ?ref)]})

deps/db/src/logseq/db/file_based/schema.cljs

@@ -1,120 +0,0 @@
-(ns logseq.db.file-based.schema
-  "Schema related vars for file graphs")
-
-;; A page is a special block, a page can corresponds to multiple files with the same ":block/name".
-(def ^:large-vars/data-var schema
-  "Schema for file graphs"
-  {:db/ident        {:db/unique :db.unique/identity}
-   :kv/value       {}
-   :recent/pages {}
-
-   ;; :block/type is a string type of the current block
-   ;; "whiteboard" for whiteboards
-   ;; "property" for property blocks
-   ;; "class" for structured page
-   :block/type {:db/index true}
-   :block/uuid {:db/unique :db.unique/identity}
-   :block/parent {:db/valueType :db.type/ref
-                  :db/index true}
-   :block/order {:db/index true}
-   :block/collapsed? {}
-
-   ;; :markdown, :org
-   :block/format {}
-
-   ;; belongs to which page
-   :block/page {:db/valueType :db.type/ref
-                :db/index true}
-   ;; reference blocks
-   :block/refs {:db/valueType :db.type/ref
-                :db/cardinality :db.cardinality/many}
-   :block/tags {:db/valueType :db.type/ref
-                :db/cardinality :db.cardinality/many}
-
-   ;; which block this block links to, used for tag, embeds
-   :block/link {:db/valueType :db.type/ref
-                :db/index true}
-
-   ;; page's namespace
-   :block/namespace {:db/valueType :db.type/ref}
-
-   ;; for pages
-   :block/alias {:db/valueType :db.type/ref
-                 :db/cardinality :db.cardinality/many
-                 :db/index true}
-
-   ;; todo keywords, e.g. "TODO", "DOING", "DONE"
-   :block/marker {}
-
-   ;; "A", "B", "C"
-   :block/priority {}
-
-   ;; map, key -> set of refs in property value or full text if none are found
-   :block/properties {}
-   ;; vector
-   :block/properties-order {}
-   ;; map, key -> original property value's content
-   :block/properties-text-values {}
-
-   ;; first block that's not a heading or unordered list
-   :block/pre-block? {}
-
-   ;; scheduled day
-   :block/scheduled {}
-
-   ;; deadline day
-   :block/deadline {}
-
-   ;; whether blocks is a repeated block (usually a task)
-   :block/repeated? {}
-
-   :block/created-at {:db/index true}
-   :block/updated-at {:db/index true}
-
-   ;; page additional attributes
-   ;; page's name, lowercase
-   :block/name {:db/unique :db.unique/identity}
-
-   ;; page's original name
-   :block/title {:db/index true}
-
-   ;; page's journal day
-   :block/journal-day {:db/index true}
-
-   ;; macros in block
-   :block/macros {:db/valueType :db.type/ref
-                  :db/cardinality :db.cardinality/many}
-
-   ;; block's file
-   :block/file {:db/valueType :db.type/ref}
-
-   ;; latest tx that affected the block
-   :block/tx-id {}
-
-   ;; file
-   :file/path {:db/unique :db.unique/identity}
-   :file/content {}
-   :file/created-at {}
-   :file/last-modified-at {}
-   :file/size {}})
-
-(def file-only-attributes
-  [:block/namespace :block/properties-text-values :block/pre-block? :recent/pages :block/file
-   :block/properties :block/properties-order :block/repeated? :block/deadline :block/scheduled :block/priority
-   :block/marker :block/macros :block/type :block/format])
-
-(def retract-attributes
-  #{:block/tags
-    :block/alias
-    :block/marker
-    :block/priority
-    :block/scheduled
-    :block/deadline
-    :block/repeated?
-    :block/pre-block?
-    :block/properties
-    :block/properties-order
-    :block/properties-text-values
-    :block/macros
-    :block/invalid-properties
-    :block/warning})

deps/db/src/logseq/db/frontend/class.cljs

@@ -32,6 +32,7 @@
       :properties {:logseq.property.class/extends :logseq.class/Page
                    :logseq.property.journal/title-format "MMM do, yyyy"}}
 
+     ;; TODO: Remove deprecated
      :logseq.class/Whiteboard
      {:title "Whiteboard"
       :properties {:logseq.property.class/extends :logseq.class/Page}}
@@ -111,8 +112,7 @@
     :logseq.class/Asset})
 
 (def private-tags
-  "Built-in classes that are private and should not be used by a user directly.
-  These used to be in block/type"
+  "Built-in classes that are private and should not be used by a user directly."
   (set/union (disj internal-tags :logseq.class/Root)
              #{:logseq.class/Journal :logseq.class/Whiteboard
                :logseq.class/Pdf-annotation}))

deps/db/src/logseq/db/frontend/content.cljs

@@ -4,7 +4,6 @@
             [datascript.core :as d]
             [logseq.common.util :as common-util]
             [logseq.common.util.page-ref :as page-ref]
-            [logseq.db.common.entity-util :as common-entity-util]
             [logseq.db.frontend.entity-util :as entity-util]))
 
 ;; [[uuid]]
@@ -49,7 +48,7 @@
                  ;; The caller need to handle situations including
                  ;; mutual references and circle references.
                  refs*
-                 (cond->> (filter common-entity-util/page? refs*)
+                 (cond->> (filter entity-util/page? refs*)
                    (and db (false? replace-pages-with-same-name?))
                    (remove (fn [e]
                              (> (count (entity-util/get-pages-by-name db (:block/title e))) 1)))))

deps/db/src/logseq/db/frontend/db.cljs

@@ -29,7 +29,7 @@
         (or (entity-util/class? page) (entity-util/internal-page? page))
         false
         ;; Default to true for closed value and future internal types.
-        ;; Other types like whiteboard are not considered because they aren't built-in
+        ;; Other types are not considered because they aren't built-in
         :else
         true))
 

deps/db/src/logseq/db/frontend/db_ident.cljc

@@ -75,8 +75,7 @@
   (assert (not (re-find #"^(logseq|block)(\.|$)" (name user-namespace)))
           "New ident is not allowed to use an internal namespace")
   (if #?(:org.babashka/nbb true
-         :cljs             (and (exists? js/process)
-                                (or js/process.env.REPEATABLE_IDENTS js/process.env.DB_GRAPH))
+         :cljs             (exists? js/process)
          :default          false)
      ;; Used for contexts where we want repeatable idents e.g. tests and CLIs
     (keyword user-namespace (normalize-ident-name-part name-string))

deps/db/src/logseq/db/frontend/entity_util.cljs

@@ -33,11 +33,6 @@
   [entity]
   (has-tag? entity :logseq.class/Property))
 
-(defn whiteboard?
-  "Given a page entity or map, check if it is a whiteboard page"
-  [entity]
-  (has-tag? entity :logseq.class/Whiteboard))
-
 (defn closed-value?
   [entity]
   (some? (:block/closed-value-property entity)))
@@ -52,8 +47,7 @@
   (or (internal-page? entity)
       (journal? entity)
       (class? entity)
-      (property? entity)
-      (whiteboard? entity)))
+      (property? entity)))
 
 (defn asset?
   "Given an entity or map, check if it is an asset block"
@@ -80,7 +74,6 @@
   (let [ident->type {:logseq.class/Tag :class
                      :logseq.class/Property :property
                      :logseq.class/Journal :journal
-                     :logseq.class/Whiteboard :whiteboard
                      :logseq.class/Page :page}]
     (set (map #(ident->type (:db/ident %)) (:block/tags entity)))))
 
@@ -92,3 +85,9 @@
 (defn get-pages-by-name
   [db page-name]
   (d/datoms db :avet :block/name (common-util/page-name-sanity-lc page-name)))
+
+(defn entity->map
+  "Convert a db Entity to a map"
+  [e]
+  (assert (de/entity? e))
+  (assoc (into {} e) :db/id (:db/id e)))

deps/db/src/logseq/db/frontend/malli_schema.cljs

@@ -3,6 +3,7 @@
   (:require [clojure.set :as set]
             [clojure.string :as string]
             [datascript.core :as d]
+            [datascript.impl.entity :as de]
             [logseq.db.common.entity-plus :as entity-plus]
             [logseq.db.common.order :as db-order]
             [logseq.db.frontend.class :as db-class]
@@ -408,6 +409,7 @@
    [:block/link {:optional true} :int]
    [:logseq.property/created-from-property {:optional true} :int]])
 
+;; TODO: Remove deprecated
 (def whiteboard-block
   "A (shape) block for whiteboard"
   (vec
@@ -526,6 +528,14 @@
    [:block/created-at {:optional true} :int]
    [:block/updated-at {:optional true} :int]])
 
+(defn- whiteboard?
+  [entity]
+  (when (or (map? entity) (de/entity? entity))
+    (some (fn [t]
+            (or (keyword-identical? (:db/ident t) :logseq.class/Whiteboard)
+                (keyword-identical? t :logseq.class/Whiteboard)))
+          (:block/tags entity))))
+
 (defn entity-dispatch-key [db ent]
   (let [d (if (:block/uuid ent) (d/entity db [:block/uuid (:block/uuid ent)]) ent)
         ;; order matters as some block types are a subset of others e.g. :whiteboard
@@ -536,7 +546,8 @@
                        :class
                        (entity-util/hidden? d)
                        :hidden
-                       (entity-util/whiteboard? d)
+                       ;; TODO: Remove deprecated
+                       (whiteboard? d)
                        :normal-page
                        (entity-util/page? d)
                        :normal-page

deps/db/src/logseq/db/frontend/property.cljs

@@ -182,6 +182,16 @@
                                                  :cardinality :many
                                                  :public? true
                                                  :view-context :never}}
+     :logseq.property.class/bidirectional-property-title {:title "Bidirectional property title"
+                                                          :schema {:type :string
+                                                                   :public? true
+                                                                   :view-context :class}}
+     :logseq.property.class/enable-bidirectional? {:title "Enable bidirectional properties"
+                                                   :schema {:type :checkbox
+                                                            :public? true
+                                                            :view-context :class}
+                                                   :properties
+                                                   {:logseq.property/description "When enabled, this tag will show reverse nodes that link to the current node via properties."}}
      :logseq.property/hide-empty-value {:title "Hide empty value"
                                         :schema {:type :checkbox
                                                  :public? true
@@ -219,7 +229,7 @@
      :logseq.property/asset   {:title "Asset"
                                :schema {:type :entity
                                         :hide? true}}
-     ;; used by pdf and whiteboard
+     ;; used by pdf
      ;; TODO: remove ls-type
      :logseq.property/ls-type {:schema {:type :keyword
                                         :hide? true}}
@@ -258,9 +268,11 @@
                                                   :schema {:type :node
                                                            :cardinality :many
                                                            :hide? true}}
+     ;; TODO: Remove deprecated
      :logseq.property.tldraw/page {:title "Tldraw Page"
                                    :schema {:type :map
                                             :hide? true}}
+     ;; TODO: Remove deprecated
      :logseq.property.tldraw/shape {:title "Tldraw Shape"
                                     :schema {:type :map
                                              :hide? true}}
@@ -870,3 +882,16 @@
                   :else
                   false)))
             values)))
+
+(defn lookup
+  "Get the property value by a built-in property's db-ident from coll"
+  [block db-ident]
+  (let [val (get block db-ident)]
+    (if (built-in-has-ref-value? db-ident) (property-value-content val) val)))
+
+(defn get-block-property-value
+  "Get the value of built-in block's property by its db-ident"
+  [db block db-ident]
+  (when db
+    (let [block (or (d/entity db (:db/id block)) block)]
+      (lookup block db-ident))))

deps/db/src/logseq/db/frontend/rules.cljc

@@ -1,10 +1,8 @@
 (ns ^:bb-compatible logseq.db.frontend.rules
-  "Datalog rules mostly for DB graphs. `rules`
-   is the only var also used by file graphs"
-  (:require [logseq.db.file-based.rules :as file-rules]))
+  "Datalog rules for DB graphs")
 
 (def ^:large-vars/data-var rules
-  "Rules used mainly in frontend.db.model for both DB and file graphs"
+  "Rules used mainly in frontend.db.model"
   ;; rule "parent" is optimized for parent node -> child node nesting queries
   {:parent
    '[[(parent ?p ?c)
@@ -83,12 +81,23 @@
 (def ^:large-vars/data-var db-query-dsl-rules
   "Rules used by frontend.query.dsl for DB graphs"
   (merge
-   (dissoc file-rules/query-dsl-rules :namespace
-           :page-property :has-page-property
-           :page-tags :all-page-tags)
    rules
 
-   {:between
+   {:page-ref
+    '[(page-ref ?b ?ref)
+      (has-ref ?b ?ref)]
+
+    :block-content
+    '[(block-content ?b ?query)
+      [?b :block/title ?content]
+      [(clojure.string/includes? ?content ?query)]]
+
+    :page
+    '[(page ?b ?page-name)
+      [?b :block/page ?bp]
+      [?bp :block/name ?page-name]]
+
+    :between
     '[(between ?b ?start ?end)
       [?b :block/page ?p]
       [?p :block/tags :logseq.class/Journal]

deps/db/src/logseq/db/frontend/schema.cljs

@@ -1,17 +1,10 @@
 (ns logseq.db.frontend.schema
-  "Schema related fns for DB and file graphs"
+  "Schema related fns"
   (:require [clojure.set :as set]
-            [clojure.string :as string]
-            [logseq.db.file-based.schema :as file-schema]))
+            [clojure.string :as string]))
 
 (def schema-version? (every-pred map? :major))
 
-(def major-schema-version-string-schema
-  [:and :string
-   [:fn
-    {:error/message "should be a major schema-version"}
-    (fn [s] (some? (parse-long s)))]])
-
 (defn parse-schema-version
   "Return schema-version({:major <num> :minor <num>}).
   supported input: 10, \"10.1\", [10 1]"
@@ -37,7 +30,7 @@
          (map (juxt :major :minor)
               [(parse-schema-version x) (parse-schema-version y)])))
 
-(def version (parse-schema-version "65.19"))
+(def version (parse-schema-version "65.20"))
 
 (defn major-version
   "Return a number.
@@ -60,14 +53,59 @@
       (str (:major schema-version)))
     :else (throw (ex-info "Not a schema-version" {:data schema-version}))))
 
-(def schema
-  "Schema for DB graphs. :block/tags are classes in this schema"
-  (merge
-   (apply dissoc file-schema/schema file-schema/file-only-attributes)
-   {:block/name {:db/index true}        ; remove db/unique for :block/name
-    ;; closed value
-    :block/closed-value-property {:db/valueType :db.type/ref
-                                  :db/cardinality :db.cardinality/many}}))
+(def ^:large-vars/data-var schema
+  "Schema for DB graphs"
+  {:db/ident        {:db/unique :db.unique/identity}
+   :kv/value       {}
+
+   :block/uuid {:db/unique :db.unique/identity}
+   :block/parent {:db/valueType :db.type/ref
+                  :db/index true}
+   :block/order {:db/index true}
+   :block/collapsed? {}
+
+   ;; belongs to which page
+   :block/page {:db/valueType :db.type/ref
+                :db/index true}
+   ;; reference blocks
+   :block/refs {:db/valueType :db.type/ref
+                :db/cardinality :db.cardinality/many}
+   :block/tags {:db/valueType :db.type/ref
+                :db/cardinality :db.cardinality/many}
+
+   ;; which block this block links to, used for tag, embeds
+   :block/link {:db/valueType :db.type/ref
+                :db/index true}
+
+   ;; for pages
+   :block/alias {:db/valueType :db.type/ref
+                 :db/cardinality :db.cardinality/many
+                 :db/index true}
+
+   :block/created-at {:db/index true}
+   :block/updated-at {:db/index true}
+
+   ;; page additional attributes
+   ;; page's name, lowercase
+   :block/name {:db/index true} ; remove db/unique for :block/name
+
+   ;; page's original name
+   :block/title {:db/index true}
+
+   ;; page's journal day
+   :block/journal-day {:db/index true}
+
+   ;; latest tx that affected the block
+   :block/tx-id {}
+   :block/closed-value-property {:db/valueType :db.type/ref
+                                 :db/cardinality :db.cardinality/many}
+
+   ;; file
+   :file/path {:db/unique :db.unique/identity}
+   :file/content {}
+   :file/created-at {}
+   :file/last-modified-at {}
+   :file/size {}})
 
 ;; If only block/title changes
 (def retract-attributes

deps/db/src/logseq/db/sqlite/build.cljs

@@ -156,7 +156,7 @@
 
                              :else
                              v)})]
-                      (if (set? v) (set (map build-pvalue v)) (build-pvalue v)))])))
+                    (if (set? v) (set (map build-pvalue v)) (build-pvalue v)))])))
        ((fn [x]
           (db-property-build/build-property-values-tx-m new-block x {:pvalue-map? true})))))
 
@@ -550,8 +550,8 @@
                      (remove existing-pages))))
              distinct
              (map #(hash-map :page {:block/title %})))]
-    (when (seq new-pages-from-refs)
-      (println "Building additional pages from content refs:" (pr-str (mapv #(get-in % [:page :block/title]) new-pages-from-refs))))
+    ;; (when (seq new-pages-from-refs)
+    ;;   (prn :debug "Building additional pages from content refs:" (pr-str (mapv #(get-in % [:page :block/title]) new-pages-from-refs))))
     (concat new-pages-from-refs pages-and-blocks)))
 
 (defn- add-new-pages-from-properties
@@ -565,9 +565,9 @@
                        distinct
                        (remove existing-pages)
                        (map #(hash-map :page %)))]
-    (when (seq new-pages)
-      (println "Building additional pages from property values:"
-               (pr-str (mapv #(or (get-in % [:page :block/title]) (get-in % [:page :build/journal])) new-pages))))
+    ;; (when (seq new-pages)
+    ;;   (prn :debug "Building additional pages from property values:"
+    ;;            (pr-str (mapv #(or (get-in % [:page :block/title]) (get-in % [:page :build/journal])) new-pages))))
     ;; new-pages must come first because they are referenced by pages-and-blocks
     (concat new-pages pages-and-blocks)))
 
@@ -662,7 +662,7 @@
         classes' (merge new-classes classes)
         used-properties (get-used-properties-from-options options)
         new-properties (->> (set/difference (set (keys used-properties)) (set (keys properties)))
-                            (remove db-property/logseq-property?)
+                            (remove db-property/internal-property?)
                             (map (fn [prop]
                                    [prop (infer-property-schema (get used-properties prop))]))
                             (into {}))

deps/db/test/logseq/db/common/initial_data_test.cljs

@@ -31,6 +31,7 @@
     (create-graph-dir "tmp/graphs" "test-db")
 
     (let [conn* (sqlite-cli/open-db! "tmp/graphs" "test-db")
+          _ (d/transact! conn* (sqlite-create-graph/build-db-initial-data "{}"))
           blocks [{:file/path "logseq/config.edn"
                    :file/content "{:foo :bar}"}]
           _ (d/transact! conn* blocks)
@@ -39,7 +40,8 @@
           conn (d/conn-from-datoms initial-data schema)]
       (is (= blocks
              (->> @conn
-                  (d/q '[:find (pull ?b [:block/uuid :file/path :file/content]) :where [?b :file/content]])
+                  (d/q '[:find (pull ?b [:file/path :file/content])
+                         :where [?b :file/content] [?b :file/path "logseq/config.edn"]])
                   (map first)))
           "Correct file with content is found"))))
 

deps/db/test/logseq/db_test.cljs

@@ -109,3 +109,42 @@
          (ldb/transact! temp-conn [{:db/ident :logseq.class/Task
                                     :block/tags :logseq.class/Property}])
          (ldb/transact! temp-conn [[:db/retract :logseq.class/Task :block/tags :logseq.class/Property]]))))))
+
+(deftest get-bidirectional-properties
+  (testing "disabled by default"
+    (let [conn (db-test/create-conn-with-blocks
+                {:properties {:friend {:logseq.property/type :node
+                                       :build/property-classes [:Person]}}
+                 :classes {:Person {}
+                           :Project {}}
+                 :pages-and-blocks
+                 [{:page {:block/title "Alice"
+                          :build/tags [:Person]
+                          :build/properties {:friend [:build/page {:block/title "Bob"}]}}}
+                  {:page {:block/title "Bob"}}
+                  {:page {:block/title "Charlie"
+                          :build/tags [:Project]
+                          :build/properties {:friend [:build/page {:block/title "Bob"}]}}}]})
+          target (db-test/find-page-by-title @conn "Bob")]
+      (is (empty? (ldb/get-bidirectional-properties @conn (:db/id target))))))
+
+  (testing "enabled per class"
+    (let [conn (db-test/create-conn-with-blocks
+                {:properties {:friend {:logseq.property/type :node
+                                       :build/property-classes [:Person]}}
+                 :classes {:Person {:build/properties {:logseq.property.class/enable-bidirectional? true}}
+                           :Project {}}
+                 :pages-and-blocks
+                 [{:page {:block/title "Alice"
+                          :build/tags [:Person]
+                          :build/properties {:friend [:build/page {:block/title "Bob"}]}}}
+                  {:page {:block/title "Bob"}}
+                  {:page {:block/title "Charlie"
+                          :build/tags [:Project]
+                          :build/properties {:friend [:build/page {:block/title "Bob"}]}}}]})
+          target (db-test/find-page-by-title @conn "Bob")
+          results (ldb/get-bidirectional-properties @conn (:db/id target))]
+      (is (= 1 (count results)))
+      (is (= "People" (:title (first results))))
+      (is (= ["Alice"]
+             (map :block/title (:entities (first results))))))))

deps/graph-parser/.carve/config.edn

@@ -1,8 +1,4 @@
 {:paths ["src"]
- :api-namespaces [logseq.graph-parser.property
-                  logseq.graph-parser.exporter
-                  logseq.graph-parser.db
-                  ;; Used in tests
-                  logseq.graph-parser.test.docs-graph-helper
+ :api-namespaces [logseq.graph-parser.exporter
                   logseq.graph-parser.schema.mldoc]
  :report {:format :ignore}}

deps/graph-parser/.carve/ignore

@@ -1,25 +1,13 @@
 ;; For CLI
-logseq.graph-parser.cli/parse-graph
-;; For CLI
 logseq.graph-parser.mldoc/ast-export-markdown
 ;; API
-logseq.graph-parser.mldoc/link?
+logseq.graph-parser.mldoc/block-with-title?
 ;; API
-logseq.graph-parser/get-blocks-to-delete
+logseq.graph-parser.mldoc/->db-edn
 ;; API
 logseq.graph-parser.text/get-file-basename
 ;; API
 logseq.graph-parser.mldoc/mldoc-link?
-;; public var
-logseq.graph-parser.schema.mldoc/block-ast-coll-schema
-;; API
-logseq.graph-parser/import-file-to-db-graph
-;; API
-logseq.graph-parser.block/extract-plain
-;; API
-logseq.graph-parser.block/extract-refs-from-text
 ;; API
 logseq.graph-parser.text/get-page-name
-logseq.graph-parser.text/get-namespace-last-part
-;; API
-logseq.graph-parser.whiteboard/shape->block
+logseq.graph-parser.text/get-namespace-last-part

deps/graph-parser/.clj-kondo/config.edn

@@ -9,7 +9,6 @@
   :consistent-alias
   {:aliases {clojure.string string
              datascript.core d
-             logseq.graph-parser graph-parser
              logseq.graph-parser.text text
              logseq.graph-parser.block gp-block
              logseq.graph-parser.mldoc gp-mldoc

deps/graph-parser/README.md

@@ -1,24 +1,20 @@
 ## Description
 
-This library parses a logseq graph directory and returns it as a datascript
-database connection. This library powers the Logseq app and also runs from the
-commandline, _independent_ of the app. This is powerful as this can run anywhere
-that a Node.js script has access to a Logseq graph e.g. on CI processes like
-Github Actions. This library is compatible with ClojureScript and with
+This library parses a file graph directory and returns it as a datascript
+database connection. This library mainly exists to convert file graphs to DB graphs.
+This library is compatible with ClojureScript and with
 [nbb-logseq](https://github.com/logseq/nbb-logseq) to respectively provide
 frontend and commandline functionality.
 
 ## API
 
 This library is under the parent namespace `logseq.graph-parser`. This library
-provides two main namespaces for parsing, `logseq.graph-parser` and
-`logseq.graph-parser.cli`. `logseq.graph-parser/parse-file` is the main fn for
-the frontend. `logseq.graph-parser.cli/parse-graph` is the main fn for node.js
-CLIs.
+provides two main namespaces, `logseq.graph-parser.exporter` and
+`logseq.graph-parser.extract`.
 
 ## Usage
 
-See `logseq.graph-parser.cli-test` and [nbb-logseq example
+See [nbb-logseq example
 scripts](https://github.com/logseq/nbb-logseq/tree/main/examples) for example
 usage.
 
@@ -27,7 +23,7 @@ usage.
 This follows the practices that [the Logseq frontend
 follows](/docs/dev-practices.md). Most of the same linters are used, with
 configurations that are specific to this library. See [this library's CI
-file](/.github/workflows/graph-parser.yml) for linting examples.
+file](/.github/workflows/deps-graph-parser.yml) for linting examples.
 
 ### Setup
 
@@ -56,7 +52,7 @@ $ yarn test -i focus
 
 ClojureScript tests use https://github.com/Olical/cljs-test-runner. To run tests:
 ```
-REPEATABLE_IDENTS=true clojure -M:test
+clojure -M:test
 ```
 
 To see available options that can run specific tests or namespaces: `clojure -M:test --help`

deps/graph-parser/src/logseq/graph_parser/block.cljs

@@ -5,7 +5,6 @@
             [clojure.string :as string]
             [clojure.walk :as walk]
             [datascript.core :as d]
-            [datascript.impl.entity :as de]
             [logseq.common.config :as common-config]
             [logseq.common.date :as common-date]
             [logseq.common.util :as common-util]
@@ -14,8 +13,10 @@
             [logseq.common.util.page-ref :as page-ref]
             [logseq.common.uuid :as common-uuid]
             [logseq.db :as ldb]
+            [logseq.db.common.entity-plus :as entity-plus]
             [logseq.db.common.order :as db-order]
             [logseq.db.frontend.class :as db-class]
+            [logseq.db.frontend.entity-util :as entity-util]
             [logseq.graph-parser.mldoc :as gp-mldoc]
             [logseq.graph-parser.property :as gp-property]
             [logseq.graph-parser.text :as text]
@@ -52,8 +53,7 @@
                   (and
                    (= url-type "Page_ref")
                    (and (string? value)
-                        (not (or (common-config/local-relative-asset? value)
-                                 (common-config/draw? value))))
+                        (not (common-config/local-relative-asset? value)))
                    value)
 
                   (and
@@ -305,21 +305,28 @@
 ;; Hack to detect export as some fns are too deeply nested to be refactored to get explicit option
 (def *export-to-db-graph? (atom false))
 
+(defn- get-page
+  "Similar to get-page but only for file graphs"
+  [db page-name]
+  (when (and db (string? page-name))
+    (d/entity db
+              (first (sort (map :e (entity-util/get-pages-by-name db page-name)))))))
+
 (defn- page-name-string->map
   [original-page-name db date-formatter
    {:keys [with-timestamp? page-uuid from-page class? skip-existing-page-check?]}]
-  (let [db-based? (ldb/db-based-graph? db)
+  (let [db-based? (entity-plus/db-based-graph? db)
         original-page-name (common-util/remove-boundary-slashes original-page-name)
         [original-page-name' page-name journal-day] (convert-page-if-journal original-page-name date-formatter {:export-to-db-graph? @*export-to-db-graph?})
         namespace? (and (or (not db-based?) @*export-to-db-graph?)
                         (not (boolean (text/get-nested-page-name original-page-name')))
                         (text/namespace-page? original-page-name'))
         page-entity (when (and db (not skip-existing-page-check?))
-                      (if class?
+                      (if (and class? db-based?)
                         (some->> (ldb/page-exists? db original-page-name' #{:logseq.class/Tag})
                                  first
                                  (d/entity db))
-                        (ldb/get-page db original-page-name')))
+                        (get-page db original-page-name')))
         original-page-name' (or from-page (:block/title page-entity) original-page-name')
         page (merge
               {:block/name page-name
@@ -366,6 +373,12 @@
   [s]
   (string/replace s "#" "HashTag-"))
 
+(defn- page-entity?
+  "Support DB or file graphs because of exporter"
+  [entity]
+  (or (entity-util/page? entity)
+      (contains? #{"page" "journal"} (:block/type entity))))
+
 ;; TODO: refactor
 (defn page-name->map
   "Create a page's map structure given a original page name (string).
@@ -377,8 +390,8 @@
   [original-page-name db with-timestamp? date-formatter
    & {:keys [page-uuid class?] :as options}]
   (when-not (and db (common-util/uuid-string? original-page-name)
-                 (not (ldb/page? (d/entity db [:block/uuid (uuid original-page-name)]))))
-    (let [db-based? (ldb/db-based-graph? db)
+                 (not (page-entity? (d/entity db [:block/uuid (uuid original-page-name)]))))
+    (let [db-based? (entity-plus/db-based-graph? db)
           original-page-name (cond-> (string/trim original-page-name)
                                db-based?
                                sanitize-hashtag-name)
@@ -457,7 +470,7 @@
 (defn- with-page-refs-and-tags
   [{:keys [title body tags refs marker priority] :as block} db date-formatter {:keys [structured-tags]
                                                                                :or {structured-tags #{}}}]
-  (let [db-based? (and (ldb/db-based-graph? db) (not @*export-to-db-graph?))
+  (let [db-based? (and (entity-plus/db-based-graph? db) (not @*export-to-db-graph?))
         refs (->> (concat tags refs (when-not db-based? [marker priority]))
                   (remove string/blank?)
                   (distinct))
@@ -866,61 +879,4 @@
                                [others parents' result'])))]
                      (recur blocks parents result))))
         result' (map (fn [block] (assoc block :block/order (db-order/gen-key))) result)]
-    (concat result' other-blocks)))
-
-(defn extract-plain
-  "Extract plain elements including page refs"
-  [repo content]
-  (let [ast (gp-mldoc/->edn repo content :markdown)
-        *result (atom [])]
-    (walk/prewalk
-     (fn [f]
-       (cond
-           ;; tag
-         (and (vector? f)
-              (= "Tag" (first f)))
-         nil
-
-           ;; nested page ref
-         (and (vector? f)
-              (= "Nested_link" (first f)))
-         (swap! *result conj (:content (second f)))
-
-           ;; page ref
-         (and (vector? f)
-              (= "Link" (first f))
-              (map? (second f))
-              (vector? (:url (second f)))
-              (= "Page_ref" (first (:url (second f)))))
-         (swap! *result conj
-                (:full_text (second f)))
-
-           ;; plain
-         (and (vector? f)
-              (= "Plain" (first f)))
-         (swap! *result conj (second f))
-
-         :else
-         f))
-     ast)
-    (-> (string/trim (apply str @*result))
-        text/page-ref-un-brackets!)))
-
-(defn extract-refs-from-text
-  [repo db text date-formatter]
-  (when (string? text)
-    (let [ast-refs (gp-mldoc/get-references text (gp-mldoc/get-default-config repo :markdown))
-          page-refs (map #(get-page-reference % :markdown) ast-refs)
-          block-refs (map get-block-reference ast-refs)
-          refs' (->> (concat page-refs block-refs)
-                     (remove string/blank?)
-                     distinct)]
-      (-> (map #(cond
-                  (de/entity? %)
-                  {:block/uuid (:block/uuid %)}
-                  (common-util/uuid-string? %)
-                  {:block/uuid (uuid %)}
-                  :else
-                  (page-name->map % db true date-formatter))
-               refs')
-          set))))
+    (concat result' other-blocks)))

deps/graph-parser/src/logseq/graph_parser/cli.cljs

@@ -1,81 +0,0 @@
-(ns ^:node-only logseq.graph-parser.cli
-  "For file graphs, primary ns to parse graphs with node.js based CLIs"
-  (:require ["fs" :as fs]
-            ["path" :as path]
-            [clojure.edn :as edn]
-            [logseq.common.config :as common-config]
-            [logseq.common.graph :as common-graph]
-            [logseq.common.util :as common-util]
-            [logseq.graph-parser :as graph-parser]
-            [logseq.graph-parser.db :as gp-db]))
-
-(defn- slurp
-  "Return file contents like clojure.core/slurp"
-  [file]
-  (str (fs/readFileSync file)))
-
-(defn- remove-hidden-files [dir config files]
-  (if (seq (:hidden config))
-    (->> files
-         (map #(assoc % ::rel-path (path/relative dir (:file/path %))))
-         ((fn [files] (common-config/remove-hidden-files files config ::rel-path)))
-         (map #(dissoc % ::rel-path)))
-    files))
-
-(defn- build-graph-files
-  "Given a graph directory, return absolute, allowed file paths and their contents in preparation
-   for parsing"
-  [dir* config]
-  (let [dir (path/resolve dir*)]
-    (->> (common-graph/get-files dir)
-         (map #(hash-map :file/path %))
-         graph-parser/filter-files
-         (remove-hidden-files dir config)
-         (mapv #(assoc % :file/content (slurp (:file/path %)))))))
-
-(defn- read-config
-  "Reads repo-specific config from logseq/config.edn"
-  [dir]
-  (let [config-file (str dir "/" common-config/app-name "/config.edn")]
-    (if (fs/existsSync config-file)
-      (-> config-file fs/readFileSync str edn/read-string)
-      {})))
-
-(defn- parse-files
-  [conn files {:keys [config] :as options}]
-  (let [extract-options (merge {:date-formatter (common-config/get-date-formatter config)
-                                :user-config config
-                                :filename-format (or (:file/name-format config) :legacy)}
-                               (select-keys options [:verbose]))]
-    (mapv
-     (fn [{:file/keys [path content]}]
-       (let [{:keys [ast]}
-             (let [parse-file-options
-                   (merge {:extract-options
-                           (assoc extract-options
-                                  :block-pattern (common-config/get-block-pattern (common-util/get-format path)))}
-                          (:parse-file-options options))]
-               (graph-parser/parse-file conn path content parse-file-options))]
-         {:file path :ast ast}))
-     files)))
-
-(defn parse-graph
-  "Parses a given graph directory and returns a datascript connection and all
-  files that were processed. The directory is parsed as if it were a new graph
-  as it can't assume that the metadata in logseq/ is up to date. Directory is
-  assumed to be using git. This fn takes the following options:
-* :verbose - When enabled prints more information during parsing. Defaults to true
-* :files - Specific files to parse instead of parsing the whole directory
-* :conn - Database connection to use instead of creating new one
-* :parse-file-options - Options map to pass to graph-parser/parse-file"
-  ([dir]
-   (parse-graph dir {}))
-  ([dir options]
-   (let [config (read-config dir)
-         files (or (:files options) (build-graph-files dir config))
-         conn (or (:conn options) (gp-db/start-conn))
-         _ (when-not (:files options) (println "Parsing" (count files) "files..."))
-         asts (parse-files conn files (merge options {:config config}))]
-     {:conn conn
-      :files (map :file/path files)
-      :asts asts})))

deps/graph-parser/src/logseq/graph_parser/exporter.cljs

@@ -22,11 +22,11 @@
             [logseq.common.uuid :as common-uuid]
             [logseq.db :as ldb]
             [logseq.db.common.order :as db-order]
-            [logseq.db.common.property-util :as db-property-util]
             [logseq.db.frontend.asset :as db-asset]
             [logseq.db.frontend.class :as db-class]
             [logseq.db.frontend.content :as db-content]
             [logseq.db.frontend.db-ident :as db-ident]
+            [logseq.db.frontend.entity-util :as entity-util]
             [logseq.db.frontend.malli-schema :as db-malli-schema]
             [logseq.db.frontend.property :as db-property]
             [logseq.db.frontend.property.build :as db-property-build]
@@ -35,7 +35,7 @@
             [logseq.db.sqlite.util :as sqlite-util]
             [logseq.graph-parser.block :as gp-block]
             [logseq.graph-parser.extract :as extract]
-            [logseq.graph-parser.property :as gp-property]
+            [logseq.graph-parser.text :as text]
             [logseq.graph-parser.utf8 :as utf8]
             [promesa.core :as p]))
 
@@ -314,11 +314,29 @@
           (dissoc :block/priority)))
     block))
 
+(defn- get-date-formatter
+  [config]
+  (or
+   (:journal/page-title-format config)
+   ;; for compatibility
+   (:date-formatter config)
+   "MMM do, yyyy"))
+
+(defn- journal-entity?
+  [entity]
+  (or (entity-util/journal? entity)
+      (identical? "journal" (:block/type entity))))
+
+(defn- page-entity?
+  [entity]
+  (or (entity-util/page? entity)
+      (contains? #{"page" "journal"} (:block/type entity))))
+
 (defn- find-or-create-deadline-scheduled-value
   "Given a :block/scheduled or :block/deadline value, creates the datetime property value
    and any optional journal tx associated with that value"
   [date-int page-names-to-uuids user-config]
-  (let [title (date-time-util/int->journal-title date-int (common-config/get-date-formatter user-config))
+  (let [title (date-time-util/int->journal-title date-int (get-date-formatter user-config))
         existing-journal-page (some->> title
                                        common-util/page-name-sanity-lc
                                        (get @page-names-to-uuids)
@@ -399,7 +417,7 @@
   (let [prop-type (cond (and (coll? prop-val)
                              (seq prop-val)
                              (set/subset? prop-val
-                                          (set (keep #(when (ldb/journal? %)
+                                          (set (keep #(when (journal-entity? %)
                                                         (:block/title %)) refs))))
                         :date
                         (and (coll? prop-val) (seq prop-val) (text-with-refs? prop-val prop-val-text))
@@ -422,11 +440,20 @@
     (when (and prev-type (not= prev-type prop-type))
       {:type {:from prev-type :to prop-type}})))
 
+(defn- get-file-pid
+  "Gets file graph property id given the db graph ident"
+  [db-ident]
+  ;; Map of unique cases where the db graph keyword name is different than the file graph id
+  (let [unique-file-ids {:logseq.property/order-list-type :logseq.order-list-type
+                         :logseq.property/publishing-public? :public}]
+    (or (get unique-file-ids db-ident)
+        (keyword (name db-ident)))))
+
 (def built-in-property-file-to-db-idents
   "Map of built-in property file ids to their db graph idents"
   (->> (keys db-property/built-in-properties)
        (map (fn [k]
-              [(db-property-util/get-file-pid k) k]))
+              [(get-file-pid k) k]))
        (into {})))
 
 (def all-built-in-property-file-ids
@@ -451,8 +478,7 @@
   #{:alias :tags :background-color :heading
     :query-table :query-properties :query-sort-by :query-sort-desc
     :ls-type :hl-type :hl-color :hl-page :hl-stamp :hl-value :file :file-path
-    :logseq.order-list-type :logseq.tldraw.page :logseq.tldraw.shape
-    :icon :public :exclude-from-graph-view :filters})
+    :logseq.order-list-type :icon :public :exclude-from-graph-view :filters})
 
 (assert (set/subset? file-built-in-property-names all-built-in-property-file-ids)
         "All file-built-in properties are used in db graph")
@@ -998,35 +1024,31 @@
 (defn- update-block-refs
   "Updates the attributes of a block ref as this is where a new page is defined. Also
    updates block content effected by refs"
-  [block page-names-to-uuids {:keys [whiteboard?]}]
-  (let [ref-to-ignore? (if whiteboard?
-                         #(and (map? %) (:block/uuid %))
-                         #(and (vector? %) (= :block/uuid (first %))))]
-    (if (seq (:block/refs block))
-      (cond-> block
-        true
-        (update
-         :block/refs
-         (fn [refs]
-           (mapv (fn [ref]
-                   ;; Only keep :block/uuid as we don't want to re-transact page refs
-                   (if (map? ref)
-                     ;; a new page's uuid can change across blocks so rely on consistent one from pages-tx
-                     (if-let [existing-uuid (some->> (:block/name ref) (get @page-names-to-uuids))]
-                       [:block/uuid existing-uuid]
-                       [:block/uuid (:block/uuid ref)])
-                     ref))
-                 refs)))
-        (:block/title block)
-        (assoc :block/title
-               ;; TODO: Handle refs for whiteboard block which has none
-               (let [refs (->> (:block/refs block)
-                               (remove #(or (ref-to-ignore? %)
-                                  ;; ignore deadline related refs that don't affect content
-                                            (and (keyword? %) (db-malli-schema/internal-ident? %))))
-                               (map #(add-uuid-to-page-map % page-names-to-uuids)))]
-                 (db-content/title-ref->id-ref (:block/title block) refs {:replace-tag? false}))))
-      block)))
+  [block page-names-to-uuids]
+  (if (seq (:block/refs block))
+    (cond-> block
+      true
+      (update
+       :block/refs
+       (fn [refs]
+         (mapv (fn [ref]
+                 ;; Only keep :block/uuid as we don't want to re-transact page refs
+                 (if (map? ref)
+                   ;; a new page's uuid can change across blocks so rely on consistent one from pages-tx
+                   (if-let [existing-uuid (some->> (:block/name ref) (get @page-names-to-uuids))]
+                     [:block/uuid existing-uuid]
+                     [:block/uuid (:block/uuid ref)])
+                   ref))
+               refs)))
+      (:block/title block)
+      (assoc :block/title
+             (let [refs (->> (:block/refs block)
+                             (remove #(or (and (vector? %) (= :block/uuid (first %)))
+                                          ;; ignore deadline related refs that don't affect content
+                                          (and (keyword? %) (db-malli-schema/internal-ident? %))))
+                             (map #(add-uuid-to-page-map % page-names-to-uuids)))]
+               (db-content/title-ref->id-ref (:block/title block) refs {:replace-tag? false}))))
+    block))
 
 (defn- fix-pre-block-references
   "Point pre-block children to parents since pre blocks don't exist in db graphs"
@@ -1299,7 +1321,7 @@
     (cond
       (page-ref/page-ref? (str (first (:arguments (second embed-node)))))
       (let [page-uuid (get-page-uuid page-names-to-uuids
-                                     (some-> (page-ref/get-page-name (first (:arguments (second embed-node))))
+                                     (some-> (text/get-page-name (first (:arguments (second embed-node))))
                                              common-util/page-name-sanity-lc)
                                      {:block block})]
         (merge block
@@ -1337,15 +1359,14 @@
           block' (-> prepared-block
                      (fix-pre-block-references pre-blocks page-names-to-uuids)
                      (fix-block-name-lookup-ref page-names-to-uuids)
-                     (update-block-refs page-names-to-uuids options)
+                     (update-block-refs page-names-to-uuids)
                      (update-block-tags db (:user-options options) per-file-state (:all-idents import-state))
                      (handle-embeds page-names-to-uuids walked-ast-blocks (select-keys options [:log-fn]))
                      (handle-quotes (select-keys options [:log-fn]))
                      (update-block-marker options)
                      (update-block-priority options)
                      add-missing-timestamps
-                   ;; old whiteboards may have :block/left
-                     (dissoc :block/left :block/format :block.temp/ast-blocks)
+                     (dissoc :block/format :block.temp/ast-blocks)
                   ;;  ((fn [x] (prn ::block-out x) x))
                      )]
     ;; Order matters as previous txs are referenced in block
@@ -1368,7 +1389,6 @@
         (journal-created-ats (:block/name m))
         (assoc :block/created-at (journal-created-ats (:block/name m))))
       add-missing-timestamps
-      (dissoc :block/whiteboard?)
       (update-page-tags db user-options per-file-state all-idents)))
 
 (defn- get-page-parents
@@ -1464,7 +1484,7 @@
 
 (defn- build-pages-tx
   "Given all the pages and blocks parsed from a file, return a map containing
-  all non-whiteboard pages to be transacted, pages' properties and additional
+  all pages to be transacted, pages' properties and additional
   data for subsequent steps"
   [conn pages blocks {:keys [import-state user-options]
                       :as options}]
@@ -1681,16 +1701,6 @@
      :property-pages-tx (concat property-pages-tx converted-property-pages-tx retract-page-tag-from-properties-tx)
      :property-page-properties-tx property-page-properties-tx}))
 
-(defn- update-whiteboard-blocks [blocks format]
-  (map (fn [b]
-         (if (seq (:block/properties b))
-           (-> (dissoc b :block/content)
-               (update :block/title #(gp-property/remove-properties format %)))
-           (cond-> (dissoc b :block/content)
-             (:block/content b)
-             (assoc :block/title (:block/content b)))))
-       blocks))
-
 (defn- fix-extracted-block-tags-and-refs
   "A tag or ref can have different :block/uuid's across extracted blocks. This makes
    sense for most in-app uses but not for importing where we want consistent identity.
@@ -1726,11 +1736,16 @@
              (update :block/refs fix-block-uuids {:ref? true :properties (:block/properties b)})))
          blocks)))
 
+(defn- get-block-pattern
+  [format]
+  (let [format' (keyword format)]
+    (if (= format' :org) "*" "-")))
+
 (defn- extract-pages-and-blocks
   "Main fn which calls graph-parser to convert markdown into data"
   [db file content {:keys [extract-options import-state]}]
   (let [format (common-util/get-format file)
-        extract-options' (merge {:block-pattern (common-config/get-block-pattern format)
+        extract-options' (merge {:block-pattern (get-block-pattern format)
                                  :date-formatter "MMM do, yyyy"
                                  :uri-encoded? false
                                  ;; Alters behavior in gp-block
@@ -1739,26 +1754,16 @@
                                 extract-options
                                 {:db db})
         extracted
-        (cond (contains? common-config/mldoc-support-formats format)
+        (cond (contains? #{:org :markdown :md} format)
               (-> (extract/extract file content extract-options')
                   (update :pages (fn [pages]
                                    (map #(dissoc % :block.temp/original-page-name) pages)))
                   (update :blocks fix-extracted-block-tags-and-refs))
 
-              (common-config/whiteboard? file)
-              (-> (extract/extract-whiteboard-edn file content extract-options')
-                  (update :pages (fn [pages]
-                                   (->> pages
-                                        ;; migrate previous attribute for :block/title
-                                        (map #(-> %
-                                                  (assoc :block/title (or (:block/original-name %) (:block/title %))
-                                                         :block/tags #{:logseq.class/Whiteboard})
-                                                  (dissoc :block/type :block/original-name))))))
-                  (update :blocks update-whiteboard-blocks format))
-
               :else
-              (swap! (:ignored-files import-state) conj
-                     {:path file :reason :unsupported-file-format}))]
+              (when-not (re-find #"whiteboards/.*\.edn$" (str file))
+                (swap! (:ignored-files import-state) conj
+                       {:path file :reason :unsupported-file-format})))]
     ;; Annotation markdown pages are saved for later as they are dependant on the asset being annotated
     (if (string/starts-with? (str (path/basename file)) "hls__")
       (do
@@ -1847,16 +1852,8 @@
           old-properties (keys @(get-in options [:import-state :property-schemas]))
           ;; Build page and block txs
           {:keys [pages-tx page-properties-tx per-file-state existing-pages]} (build-pages-tx conn pages blocks tx-options)
-          whiteboard-pages (->> pages-tx
-                                ;; support old and new whiteboards
-                                (filter ldb/whiteboard?)
-                                (map (fn [page-block]
-                                       (-> page-block
-                                           (assoc :logseq.property/ls-type :whiteboard-page)))))
           pre-blocks (->> blocks (keep #(when (:block/pre-block? %) (:block/uuid %))) set)
-
-          blocks-tx (let [tx-options' (assoc tx-options :whiteboard? (some? (seq whiteboard-pages)))]
-                      (<build-blocks-tx conn blocks pre-blocks per-file-state tx-options'))
+          blocks-tx (<build-blocks-tx conn blocks pre-blocks per-file-state tx-options)
           {:keys [property-pages-tx property-page-properties-tx] pages-tx' :pages-tx}
           (split-pages-and-properties-tx pages-tx old-properties existing-pages (:import-state options) @(:upstream-properties tx-options))
           ;; _ (when (seq property-pages-tx) (cljs.pprint/pprint {:property-pages-tx property-pages-tx}))
@@ -1881,12 +1878,12 @@
           blocks-index (set/union (set block-ids) (set block-refs-ids))
           ;; Order matters. pages-index and blocks-index needs to come before their corresponding tx for
           ;; uuids to be valid. Also upstream-properties-tx comes after blocks-tx to possibly override blocks
-          tx (concat whiteboard-pages pages-index page-properties-tx property-page-properties-tx pages-tx'' classes-tx' blocks-index blocks-tx)
+          tx (concat pages-index page-properties-tx property-page-properties-tx pages-tx'' classes-tx' blocks-index blocks-tx)
           tx' (common-util/fast-remove-nils tx)
-          ;; (prn :tx-counts (map #(vector %1 (count %2))
-          ;;                        [:whiteboard-pages :pages-index :page-properties-tx :property-page-properties-tx :pages-tx' :classes-tx :blocks-index :blocks-tx]
-          ;;                        [whiteboard-pages pages-index page-properties-tx property-page-properties-tx pages-tx' classes-tx blocks-index blocks-tx]))
-          ;; _ (when (not (seq whiteboard-pages)) (cljs.pprint/pprint {#_:property-pages-tx #_property-pages-tx :pages-tx pages-tx :tx tx'}))
+          ;; _ (prn :tx-counts (map #(vector %1 (count %2))
+          ;;                        [:pages-index :page-properties-tx :property-page-properties-tx :pages-tx' :classes-tx :blocks-index :blocks-tx]
+          ;;                        [pages-index page-properties-tx property-page-properties-tx pages-tx' classes-tx blocks-index blocks-tx]))
+          ;; _ (cljs.pprint/pprint {#_:property-pages-tx #_property-pages-tx :pages-tx pages-tx :tx tx'})
           main-tx-report (d/transact! conn tx' {::new-graph? true ::path file})
           _ (save-from-tx tx' options)
 
@@ -2102,7 +2099,7 @@
 (defn build-doc-options
   "Builds options for use with export-doc-files and assets"
   [config options]
-  (-> {:extract-options {:date-formatter (common-config/get-date-formatter config)
+  (-> {:extract-options {:date-formatter (get-date-formatter config)
                          ;; Remove config keys that break importing
                          :user-config (dissoc config :property-pages/excludelist :property-pages/enabled?)
                          :filename-format (or (:file/name-format config) :legacy)
@@ -2122,7 +2119,7 @@
                               (keep (fn [d]
                                       (let [child (d/entity db (:e d))
                                             parent (d/entity db (:v d))]
-                                        (when (and (nil? (:block/parent parent)) (ldb/page? child) (ldb/page? parent))
+                                        (when (and (nil? (:block/parent parent)) (page-entity? child) (page-entity? parent))
                                           parent))))
                               (common-util/distinct-by :block/uuid))
         tx-data (map

deps/graph-parser/src/logseq/graph_parser/extract.cljc

@@ -10,24 +10,16 @@
             [clojure.string :as string]
             [clojure.walk :as walk]
             [datascript.core :as d]
-            [logseq.common.config :as common-config]
             [logseq.common.util :as common-util]
             [logseq.db :as ldb]
             [logseq.graph-parser.block :as gp-block]
             [logseq.graph-parser.mldoc :as gp-mldoc]
             [logseq.graph-parser.property :as gp-property]
-            [logseq.graph-parser.text :as text]
-            [logseq.graph-parser.whiteboard :as gp-whiteboard]
-            [medley.core :as medley]))
+            [logseq.graph-parser.text :as text]))
 
-(defn- filepath->page-name
-  [filepath]
-  (when-let [file-name (last (string/split filepath #"/"))]
-    (let [result (first (common-util/split-last "." file-name))
-          ext (string/lower-case (common-util/get-file-ext filepath))]
-      (if (or (common-config/mldoc-support? ext) (= "edn" ext))
-        (common-util/safe-decode-uri-component (string/replace result "." "/"))
-        result))))
+(defn- mldoc-support?
+  [format']
+  (contains? #{:org :markdown :md} (keyword format')))
 
 (defn- path->file-name
   ;; Only for internal paths, as they are converted to POXIS already
@@ -117,7 +109,7 @@
                                     (string? title)
                                     title))
             file-name (when-let [result (path->file-body file)]
-                        (if (common-config/mldoc-support? (common-util/get-file-ext file))
+                        (if (mldoc-support? (common-util/get-file-ext file))
                           (title-parsing result filename-format)
                           result))]
         (or property-name
@@ -143,12 +135,14 @@
                  (assoc :block/alias aliases')
 
                  (:tags properties)
-                 (assoc :block/tags (let [tags (:tags properties)
-                                          tags (if (coll? tags) tags [(str tags)])
-                                          tags (remove string/blank? tags)]
-                                      (map (fn [tag] {:block/name (common-util/page-name-sanity-lc tag)
-                                                      :block/title tag})
-                                           tags))))]
+                 (assoc :block/tags (concat
+                                     (:block/tags page-m)
+                                     (let [tags (:tags properties)
+                                           tags (if (coll? tags) tags [(str tags)])
+                                           tags (remove string/blank? tags)]
+                                       (map (fn [tag] {:block/name (common-util/page-name-sanity-lc tag)
+                                                       :block/title tag})
+                                            tags)))))]
     (update result :block/properties #(apply dissoc % gp-property/editable-linkable-built-in-properties))))
 
 (defn- build-page-map
@@ -301,42 +295,6 @@
          :blocks blocks
          :ast ast}))))
 
-(defn extract-whiteboard-edn
-  "Extracts whiteboard page from given edn file
-   Whiteboard page edn is a subset of page schema
-   - it will only contain a single page (for now). The page properties are stored under :logseq.tldraw.* properties and contain 'bindings' etc
-   - blocks will be adapted to tldraw shapes. All blocks's parent is the given page."
-  [file content {:keys [verbose] :or {verbose true}}]
-  (let [_ (when verbose (println "Parsing start: " file))
-        {:keys [pages blocks]} (common-util/safe-read-map-string content)
-        blocks (map
-                (fn [block]
-                  (-> block
-                      (medley/dissoc-in [:block/parent :block/name])
-                      ;; :block/left here for backward compatibility
-                      (medley/dissoc-in [:block/left :block/name])))
-                blocks)
-        serialized-page (first pages)
-        ;; whiteboard edn file should normally have valid :block/title, :block/name, :block/uuid
-        page-name (-> (or (:block/name serialized-page)
-                          (filepath->page-name file))
-                      (common-util/page-name-sanity-lc))
-        title (or (:block/title serialized-page)
-                  page-name)
-        page-block (merge {:block/name page-name
-                           :block/title title
-                           :block/file {:file/path (common-util/path-normalize file)}}
-                          serialized-page
-                          ;; Ensure old whiteboards have correct type
-                          {:block/type "whiteboard"})
-        page-block (gp-whiteboard/migrate-page-block page-block)
-        blocks (->> blocks
-                    (map gp-whiteboard/migrate-shape-block)
-                    (map #(merge % (gp-whiteboard/with-whiteboard-block-props % [:block/uuid (:block/uuid page-block)]))))
-        _ (when verbose (println "Parsing finished: " file))]
-    {:pages (list page-block)
-     :blocks blocks}))
-
 (defn- with-block-uuid
   [pages]
   (->> (common-util/distinct-by :block/name pages)

deps/graph-parser/src/logseq/graph_parser/mldoc.cljc

@@ -169,6 +169,11 @@
   ([repo content format]
    (->edn content (get-default-config repo format))))
 
+(defn ->db-edn
+  "Wrapper around ->edn for DB graphs"
+  [content format]
+  (->edn "logseq_db_repo_stub" content format))
+
 (defn inline->edn
   [text config]
   (try
@@ -189,18 +194,7 @@
           (not (contains? #{"Page_ref" "Block_ref"} ref-type))
 
           (and (contains? #{"Page_ref"} ref-type)
-               (or
-                ;; 2. excalidraw link
-                (common-config/draw? ref-value)
-
-                ;; 3. local asset link
-                (boolean (common-config/local-relative-asset? ref-value))))))))
-
-(defn link?
-  [format link]
-  (when (string? link)
-    (some-> (first (inline->edn link (default-config format)))
-            ast-link?)))
+               (boolean (common-config/local-relative-asset? ref-value)))))))
 
 (defn mldoc-link?
   "Check whether s is a link (including page/block refs)."
@@ -212,27 +206,9 @@
        (or (contains? #{"Nested_link"} (first result'))
            (contains? #{"Page_ref" "Block_ref" "Complex"} (first (:url (second result')))))))))
 
-(defn properties?
-  [ast]
-  (contains? #{"Properties" "Property_Drawer"} (ffirst ast)))
-
 (defn block-with-title?
   [type]
   (contains? #{"Paragraph"
                "Raw_Html"
                "Hiccup"
-               "Heading"} type))
-
-(defn- has-title?
-  [repo content format]
-  (let [ast (->edn repo content format)]
-    (block-with-title? (ffirst (map first ast)))))
-
-(defn get-title&body
-  "parses content and returns [title body]
-   returns nil if no title"
-  [repo content format]
-  (let [lines (string/split-lines content)]
-    (if (has-title? repo content format)
-      [(first lines) (string/join "\n" (rest lines))]
-      [nil (string/join "\n" lines)])))
+               "Heading"} type))

deps/graph-parser/src/logseq/graph_parser/property.cljs

@@ -4,9 +4,7 @@
             [clojure.string :as string]
             [goog.string :as gstring]
             [goog.string.format]
-            [logseq.common.util :as common-util]
-            [logseq.common.util.page-ref :as page-ref]
-            [logseq.graph-parser.mldoc :as gp-mldoc]))
+            [logseq.common.util :as common-util]))
 
 (def colons "Property delimiter for markdown mode" "::")
 (defn colons-org
@@ -14,13 +12,6 @@
   [property]
   (str ":" property ":"))
 
-(defn ->block-content
-  "Creates a block content string from properties map"
-  [properties]
-  (->> properties
-       (map #(str (name (key %)) (str colons " ") (val %)))
-       (string/join "\n")))
-
 (defn properties-ast?
   [block]
   (and
@@ -40,11 +31,6 @@
 ;; these properties are hidden from the user but a few like the editable ones
 ;; are visible for the user to edit.
 
-(def built-in-extended-properties (atom #{}))
-(defn register-built-in-properties
-  [props]
-  (reset! built-in-extended-properties (set/union @built-in-extended-properties props)))
-
 (def editable-linkable-built-in-properties
   "Properties used by logseq that user can edit and that can have linkable property values"
   #{:alias :aliases :tags})
@@ -61,16 +47,14 @@
 (defn hidden-built-in-properties
   "Properties used by logseq that user can't edit or see"
   []
-  (set/union
-   #{:custom-id :background_color :created_at :last_modified_at ; backward compatibility only
-     :id :background-color :heading :collapsed
-     :created-at :updated-at :last-modified-at
-     :query-table :query-properties :query-sort-by :query-sort-desc :ls-type
-     :hl-type :hl-page :hl-stamp :hl-color :hl-value :logseq.macro-name :logseq.macro-arguments
-     :logseq.order-list-type :logseq.tldraw.page :logseq.tldraw.shape
+  #{:custom-id :background_color :created_at :last_modified_at ; backward compatibility only
+    :id :background-color :heading :collapsed
+    :created-at :updated-at :last-modified-at
+    :query-table :query-properties :query-sort-by :query-sort-desc :ls-type
+    :hl-type :hl-page :hl-stamp :hl-color :hl-value :logseq.macro-name :logseq.macro-arguments
+    :logseq.order-list-type
      ; task markers
-     :todo :doing :now :later :done}
-   @built-in-extended-properties))
+    :todo :doing :now :later :done})
 
 (def built-in-property-types
   "Types for built-in properties. Built-in properties whose values are to be
@@ -106,9 +90,6 @@
   []
   (set/difference (set/union (hidden-built-in-properties)
                              (editable-built-in-properties))
-                  ;; Most of these need to be auto-parsed as integers so exclude
-                  ;; them until we have ones that must be unparsed
-                  @built-in-extended-properties
                   ;; Refs need to be parsed
                   editable-linkable-built-in-properties
                   ;; All these should be parsed by gp-text/parse-non-string-property-value
@@ -119,7 +100,7 @@
 (defonce properties-end-pattern
   (re-pattern (gstring/format "%s[\t\r ]*\n|(%s\\s*$)" properties-end properties-end)))
 
-(defn contains-properties?
+(defn- contains-properties?
   [content]
   (when content
     (and (string/includes? content properties-start)
@@ -150,143 +131,12 @@
         content))
     content))
 
-(defn- build-properties-str
-  [format properties]
-  (when (seq properties)
-    (let [org? (= format :org)
-          kv-format (if org? ":%s: %s" (str "%s" colons " %s"))
-          full-format (if org? ":PROPERTIES:\n%s\n:END:" "%s\n")
-          properties-content (->> (map (fn [[k v]] (gstring/format kv-format (name k) v)) properties)
-                                  (string/join "\n"))]
-      (gstring/format full-format properties-content))))
-
 (defn simplified-property?
   [line]
   (boolean
    (and (string? line)
         (re-find (re-pattern (str "^\\s?[^ ]+" colons)) line))))
 
-(defn- front-matter-property?
-  [line]
-  (boolean
-   (and (string? line)
-        (common-util/safe-re-find #"^\s*[^ ]+:" line))))
-
-(defn- insert-property-not-org
-  [key* value lines {:keys [front-matter? has-properties? title?]}]
-  (let [exists? (atom false)
-        sym (if front-matter? ": " (str colons " "))
-        new-property-s (str key* sym value)
-        property-f (if front-matter? front-matter-property? simplified-property?)
-        groups (partition-by property-f lines)
-        compose-lines (fn []
-                        (mapcat (fn [lines]
-                                  (if (property-f (first lines))
-                                    (let [lines (doall
-                                                 (mapv (fn [text]
-                                                         (let [[k v] (common-util/split-first sym text)]
-                                                           (if (and k v)
-                                                             (let [key-exists? (= k key*)
-                                                                   _ (when key-exists? (reset! exists? true))
-                                                                   v (if key-exists? value v)]
-                                                               (str k sym  (string/trim v)))
-                                                             text)))
-                                                       lines))
-                                          lines (if @exists? lines (conj lines new-property-s))]
-                                      lines)
-                                    lines))
-                                groups))
-        lines (cond
-                has-properties?
-                (compose-lines)
-
-                title?
-                (cons (first lines) (cons new-property-s (rest lines)))
-
-                :else
-                (cons new-property-s lines))]
-    (string/join "\n" lines)))
-
-(defn insert-property
-  "Only accept nake content (without any indentation)"
-  ([repo format content key value]
-   (insert-property repo format content key value false))
-  ([repo format content key value front-matter?]
-   (when (string? content)
-     (let [ast (gp-mldoc/->edn repo content format)
-           title? (gp-mldoc/block-with-title? (ffirst (map first ast)))
-           has-properties? (or (and title?
-                                    (or (gp-mldoc/properties? (second ast))
-                                        (gp-mldoc/properties? (second
-                                                               (remove
-                                                                (fn [[x _]]
-                                                                  (contains? #{"Hiccup" "Raw_Html"} (first x)))
-                                                                ast)))))
-                               (gp-mldoc/properties? (first ast)))
-           lines (string/split-lines content)
-           [title body] (gp-mldoc/get-title&body repo content format)
-           scheduled (filter #(string/starts-with? % "SCHEDULED") lines)
-           deadline (filter #(string/starts-with? % "DEADLINE") lines)
-           body-without-timestamps (filter
-                                    #(not (or (string/starts-with? % "SCHEDULED")
-                                              (string/starts-with? % "DEADLINE")))
-                                    (string/split-lines body))
-           org? (= :org format)
-           key (string/lower-case (name key))
-           value (string/trim (str value))
-           start-idx (.indexOf lines properties-start)
-           end-idx (.indexOf lines properties-end)
-           result (cond
-                    (and org? (not has-properties?))
-                    (let [properties (build-properties-str format {key value})]
-                      (if title
-                        (string/join "\n" (concat [title] scheduled deadline [properties] body-without-timestamps))
-                        (str properties "\n" content)))
-
-                    (and has-properties? (>= start-idx 0) (> end-idx 0) (> end-idx start-idx))
-                    (let [exists? (atom false)
-                          before (subvec lines 0 start-idx)
-                          middle (doall
-                                  (->> (subvec lines (inc start-idx) end-idx)
-                                       (mapv (fn [text]
-                                               (let [[k v] (common-util/split-first ":" (subs text 1))]
-                                                 (if (and k v)
-                                                   (let [key-exists? (= k key)
-                                                         _ (when key-exists? (reset! exists? true))
-                                                         v (if key-exists? value v)]
-                                                     (str ":" k ": "  (string/trim v)))
-                                                   text))))))
-                          middle (if @exists? middle (conj middle (str ":" key ": "  value)))
-                          after (subvec lines (inc end-idx))
-                          lines (concat before [properties-start] middle [properties-end] after)]
-                      (string/join "\n" lines))
-
-                    (not org?)
-                    (insert-property-not-org key value lines {:has-properties? has-properties?
-                                                              :title? title?
-                                                              :front-matter? front-matter?})
-
-                    :else
-                    content)]
-       (string/trimr result)))))
-
-(defn remove-property
-  ([format key content]
-   (remove-property format key content true))
-  ([format key content first?]
-   (when (not (string/blank? (name key)))
-     (let [format (or format :markdown)
-           key (string/lower-case (name key))
-           remove-f (if first? common-util/remove-first remove)]
-       (if (and (= format :org) (not (contains-properties? content)))
-         content
-         (let [lines (->> (string/split-lines content)
-                          (remove-f (fn [line]
-                                      (let [s (string/triml (string/lower-case line))]
-                                        (or (string/starts-with? s (str ":" key ":"))
-                                            (string/starts-with? s (str key colons " ")))))))]
-           (string/join "\n" lines)))))))
-
 (defn remove-properties
   [format content]
   (cond
@@ -325,22 +175,4 @@
       (string/join "\n" lines))
 
     :else
-    content))
-
-(defn insert-properties
-  [repo format content kvs]
-  (reduce
-   (fn [content [k v]]
-     (let [k (if (string? k)
-               (keyword (-> (string/lower-case k)
-                            (string/replace " " "-")))
-               k)
-           v (if (coll? v)
-               (some->>
-                (seq v)
-                (distinct)
-                (map (fn [item] (page-ref/->page-ref (page-ref/page-ref-un-brackets! item))))
-                (string/join ", "))
-               (if (keyword? v) (name v) v))]
-       (insert-property repo format content k v)))
-   content kvs))
+    content))

deps/graph-parser/src/logseq/graph_parser/schema/mldoc.cljc

@@ -215,6 +215,3 @@
 
 (def block-ast-with-pos-coll-schema
   [:sequential [:cat block-ast-schema [:maybe pos-schema]]])
-
-(def block-ast-coll-schema
-  [:sequential block-ast-schema])

deps/graph-parser/src/logseq/graph_parser/test/docs_graph_helper.cljs

@@ -1,170 +0,0 @@
-(ns logseq.graph-parser.test.docs-graph-helper
-  "Helper fns for setting up and running tests against docs graph"
-  (:require ["child_process" :as child-process]
-            ["fs" :as fs]
-            [cljs.test :refer [is testing]]
-            [clojure.string :as string]
-            [datascript.core :as d]
-            [logseq.common.config :as common-config]))
-
-;; Helper fns for test setup
-;; =========================
-(defn- sh
-  "Run shell cmd synchronously and print to inherited streams by default. Aims
-    to be similar to babashka.tasks/shell"
-  [cmd opts]
-  (child-process/spawnSync (first cmd)
-                           (clj->js (rest cmd))
-                           (clj->js (merge {:stdio "inherit"} opts))))
-
-(defn clone-docs-repo-if-not-exists
-  [dir branch]
-  (when-not (.existsSync fs dir)
-    (sh ["git" "clone" "--depth" "1" "-b" branch "-c" "advice.detachedHead=false"
-         "https://github.com/logseq/docs" dir] {})))
-
-;; Fns for common test assertions
-;; ==============================
-(defn get-top-block-properties
-  [db]
-  (->> (d/q '[:find (pull ?b [*])
-              :where
-              [?b :block/properties]
-              [(missing? $ ?b :block/name)]]
-            db)
-       (map first)
-       (map (fn [m] (zipmap (keys (:block/properties m)) (repeat 1))))
-       (apply merge-with +)
-       (filter #(>= (val %) 5))
-       (into {})))
-
-(defn get-all-page-properties
-  [db]
-  (->> (d/q '[:find (pull ?b [*])
-              :where
-              [?b :block/properties]
-              [?b :block/name]]
-            db)
-       (map first)
-       (map (fn [m] (zipmap (keys (:block/properties m)) (repeat 1))))
-       (apply merge-with +)
-       (into {})))
-
-(defn get-block-format-counts
-  [db]
-  (->> (d/q '[:find (pull ?b [*]) :where [?b :block/format]] db)
-       (map first)
-       (group-by :block/format)
-       (map (fn [[k v]] [k (count v)]))
-       (into {})))
-
-(defn- get-journal-page-count [db]
-  (->> (d/q '[:find (count ?b)
-              :where
-              [?b :block/journal-day]
-              [?b :block/name]
-              [?b :block/file]]
-            db)
-       ffirst))
-
-(defn- get-counts-for-common-attributes [db]
-  (->> [:block/scheduled :block/priority :block/deadline :block/collapsed?
-        :block/repeated?]
-       (map (fn [attr]
-              [attr
-               (ffirst (d/q [:find (list 'count '?b) :where ['?b attr]]
-                            db))]))
-       (into {})))
-
-(defn- query-assertions
-  [db graph-dir files]
-  (testing "Query based stats"
-    (is (= (->> files
-                ;; logseq files aren't saved under :block/file
-                (remove #(string/includes? % (str graph-dir "/" common-config/app-name "/")))
-                ;; edn files being listed in docs by parse-graph aren't graph files
-                (remove #(and (not (common-config/whiteboard? %)) (string/ends-with? % ".edn")))
-                set)
-           (->> (d/q '[:find (pull ?b [* {:block/file [:file/path]}])
-                       :where [?b :block/name] [?b :block/file]]
-                     db)
-                (map (comp #(get-in % [:block/file :file/path]) first))
-                set))
-        "Files on disk should equal ones in db")
-
-    (is (= (count (filter #(re-find #"journals/" %) files))
-           (get-journal-page-count db))
-        "Journal page count on disk equals count in db")
-
-    (is (= {"CANCELED" 2 "DONE" 6 "LATER" 4 "NOW" 5 "WAIT" 1 "IN-PROGRESS" 1 "CANCELLED" 1 "TODO" 19}
-           (->> (d/q '[:find (pull ?b [*]) :where [?b :block/marker]]
-                     db)
-                (map first)
-                (group-by :block/marker)
-                (map (fn [[k v]] [k (count v)]))
-                (into {})))
-        "Task marker counts")
-
-    (is (= {:markdown 7372 :org 500} (get-block-format-counts db))
-        "Block format counts")
-
-    (is (= {:rangeincludes 13, :description 137, :updated-at 46, :tags 5, :logseq.order-list-type 16, :query-table 8,
-            :logseq.macro-arguments 105, :parent 14, :logseq.tldraw.shape 79, :card-last-score 5, :card-repeats 5,
-            :name 16, :card-next-schedule 5, :ls-type 79, :card-last-interval 5, :type
-            166, :template 5, :domainincludes 7, :title 114, :alias 62, :supports 6, :id
-            146, :url 30, :card-ease-factor 5, :logseq.macro-name 105, :created-at 46,
-            :card-last-reviewed 5, :platforms 79, :initial-version 16, :heading 332}
-           (get-top-block-properties db))
-        "Counts for top block properties")
-
-    (is (= {:rangeincludes 13, :description 117, :tags 5, :unique 2, :meta 2, :parent 14,
-            :ls-type 1, :type 147, :source 1, :domainincludes 7, :sameas 4, :title 113, :author 1,
-            :alias 62, :logseq.tldraw.page 1, :supports 6, :url 30, :platforms 78,
-            :initial-version 15, :full-title 1}
-           (get-all-page-properties db))
-        "Counts for all page properties")
-
-    (is (= {:block/scheduled 2
-            :block/priority 4
-            :block/deadline 1
-            :block/collapsed? 90
-            :block/repeated? 1}
-           (get-counts-for-common-attributes db))
-        "Counts for blocks with common block attributes")
-
-    (let [no-name (->> (d/q '[:find (pull ?n [*]) :where [?b :block/namespace ?n]] db)
-                       (filter (fn [x]
-                                 (when-not (:block/title (first x))
-                                   x))))
-          all-namespaces (->> (d/q '[:find (pull ?n [*]) :where [?b :block/namespace ?n]] db)
-                              (map (comp :block/title first))
-                              set)]
-      (is (= #{"term" "setting" "book" "templates" "page" "Community" "Tweet"
-               "Whiteboard" "Whiteboard/Tool" "Whiteboard/Tool/Shape" "Whiteboard/Object" "Whiteboard/Action Bar"}
-             all-namespaces)
-          (str "Has correct namespaces: " no-name)))
-
-    (is (empty? (->> (d/q '[:find ?n :where [?b :block/name ?n]] db)
-                     (map first)
-                     (filter #(string/includes? % "___"))))
-        "Block names don't have the slash/triple-lowbar delimiter")))
-
-(defn docs-graph-assertions
-  "These are common assertions that should pass in both graph-parser and main
-  logseq app. It is important to run these in both contexts to ensure that the
-  functionality in frontend.handler.repo and logseq.graph-parser remain the
-  same"
-  [db graph-dir files]
-  ;; Counts assertions help check for no major regressions. These counts should
-  ;; only increase over time as the docs graph rarely has deletions
-  (testing "Counts"
-    (is (= 339 (count files)) "Correct file count")
-    (is (= 33
-           (ffirst
-            (d/q '[:find (count ?b)
-                   :where [?b :block/title ?content]
-                   [(clojure.string/includes? ?content "+BEGIN_QUERY")]]
-                 db)))
-        "Advanced query count"))
-
-  (query-assertions db graph-dir files))

deps/graph-parser/src/logseq/graph_parser/text.cljs

@@ -1,6 +1,7 @@
 (ns logseq.graph-parser.text
   "Miscellaneous text util fns for the parser. Used by file and DB graphs"
-  (:require [clojure.set :as set]
+  (:require ["path" :as node-path]
+            [clojure.set :as set]
             [clojure.string :as string]
             [goog.string :as gstring]
             [logseq.common.util :as common-util]
@@ -11,7 +12,23 @@
 
 (def get-file-basename page-ref/get-file-basename)
 
-(def get-page-name page-ref/get-page-name)
+(defn- get-file-rootname
+  "Returns the rootname of a file path. e.g. /a/b/c.md -> c"
+  [path]
+  (when-not (string/blank? path)
+    (.-name (node-path/parse (string/replace path "+" "/")))))
+
+(defn get-page-name
+  "Similar to page-ref/get-page-name but handles format-specific page-refs e.g. org/md"
+  [s]
+  (and (string? s)
+       (or (when-let [[_ label _path] (re-matches page-ref/markdown-page-ref-re s)]
+             (string/trim label))
+           (when-let [[_ path _label] (re-matches #"\[\[(file:.*)\]\[.+?\]\]" s)]
+             (some-> (get-file-rootname path)
+                     (string/replace "." "/")))
+           (-> (re-matches page-ref/page-ref-any-re s)
+               second))))
 
 (def page-ref-un-brackets! page-ref/page-ref-un-brackets!)
 
@@ -75,7 +92,7 @@
       nil)
 
     "Nested_link"
-    (page-ref/get-page-name (:content data))
+    (get-page-name (:content data))
 
     "Tag"
     (if (= "Plain" (ffirst data))

deps/graph-parser/src/logseq/graph_parser/whiteboard.cljs

@@ -1,90 +0,0 @@
-(ns logseq.graph-parser.whiteboard
-  "Whiteboard related parser utilities"
-  (:require [logseq.db.common.property-util :as db-property-util]
-            [logseq.db.sqlite.util :as sqlite-util]))
-
-(defn block->shape [block]
-  (get-in block [:block/properties :logseq.tldraw.shape]))
-
-(defn shape-block? [block]
-  (= :whiteboard-shape (get-in block [:block/properties :ls-type])))
-
-;; tldraw shape props is now saved into [:block/properties :logseq.tldraw.shape]
-;; migrate
-(defn shape-block-needs-migrate? [block]
-  (let [properties (:block/properties block)]
-    (and (seq properties)
-         (and (= :whiteboard-shape (:ls-type properties))
-              (not (seq (get properties :logseq.tldraw.shape)))))))
-
-(defn page-block-needs-migrate? [block]
-  (let [properties (:block/properties block)]
-    (and (seq properties)
-         (and (= :whiteboard-page (:ls-type properties))
-              (not (seq (get properties :logseq.tldraw.page)))))))
-
-(defn migrate-shape-block [block]
-  (if (shape-block-needs-migrate? block)
-    (let [properties (:block/properties block)
-          properties (assoc properties :logseq.tldraw.shape properties)]
-      (assoc block :block/properties properties))
-    block))
-
-(defn migrate-page-block [block]
-  (if (page-block-needs-migrate? block)
-    (let [properties (:block/properties block)
-          properties (assoc properties :logseq.tldraw.page properties)]
-      (assoc block :block/properties properties))
-    block))
-
-(defn- get-shape-refs [shape]
-  (let [portal-refs (when (= "logseq-portal" (:type shape))
-                      [{:block/uuid (uuid (:pageId shape))}])
-        shape-link-refs (->> (:refs shape)
-                             (filter (complement empty?))
-                             (keep (fn [ref] (when (parse-uuid ref)
-                                               {:block/uuid (parse-uuid ref)}))))]
-    (concat portal-refs shape-link-refs)))
-
-(defn- with-whiteboard-block-refs
-  [shape]
-  (let [refs (or (get-shape-refs shape) [])]
-    {:block/refs (if (seq refs) refs [])}))
-
-(defn- with-whiteboard-content
-  "Main purpose of this function is to populate contents when shapes are used as references in outliner."
-  [shape]
-  {:block/title (case (:type shape)
-                  "text" (:text shape)
-                  "logseq-portal" ""
-                  "line" (str "whiteboard arrow" (when-let [label (:label shape)] (str ": " label)))
-                  (str "whiteboard " (:type shape)))})
-
-(defn with-whiteboard-block-props
-  "Builds additional block attributes for a whiteboard block. Expects :block/properties
-   to be in file graph format"
-  [block page-id]
-  (let [shape? (shape-block? block)
-        shape (block->shape block)]
-    (merge (when shape?
-             (merge
-              {:block/uuid (uuid (:id shape))}
-              (with-whiteboard-block-refs shape)
-              (with-whiteboard-content shape)))
-           (when (nil? (:block/parent block)) {:block/parent page-id})
-           (when (nil? (:block/format block)) {:block/format :markdown}) ;; TODO: read from config
-           {:block/page page-id})))
-
-(defn shape->block [repo shape page-id]
-  (let [block-uuid (if (uuid? (:id shape)) (:id shape) (uuid (:id shape)))
-        properties {(db-property-util/get-pid repo :logseq.property/ls-type) :whiteboard-shape
-                    (db-property-util/get-pid repo :logseq.property.tldraw/shape) shape}
-        block {:block/uuid block-uuid
-               :block/title ""
-               :block/page page-id
-               :block/parent page-id}
-        block' (if (sqlite-util/db-based-graph? repo)
-                 (merge block properties)
-                 (assoc block :block/properties properties))
-        additional-props (with-whiteboard-block-props block' page-id)]
-    (merge block' additional-props)))

deps/graph-parser/test/logseq/graph_parser/block_test.cljs

@@ -1,11 +1,8 @@
 (ns logseq.graph-parser.block-test
-  (:require [logseq.graph-parser.block :as gp-block]
-            [logseq.graph-parser.mldoc :as gp-mldoc]
-            [logseq.graph-parser :as graph-parser]
-            [logseq.graph-parser.db :as gp-db]
-            [logseq.common.util.block-ref :as block-ref]
+  (:require [cljs.test :refer [deftest are testing is]]
             [datascript.core :as d]
-            [cljs.test :refer [deftest are testing is]]))
+            [logseq.graph-parser.block :as gp-block]
+            [logseq.graph-parser.mldoc :as gp-mldoc]))
 
 (defn- extract-properties
   [properties user-config]
@@ -19,11 +16,11 @@
 
 (deftest test-fix-duplicate-id
   (are [x y]
-      (let [result (gp-block/fix-duplicate-id (gp-block/block-keywordize x))]
-        (and (:block/uuid result)
-             (not= (:uuid x) (:block/uuid result))
-             (= (select-keys result
-                             [:block/properties :block/title :block/properties-text-values :block/properties-order]) (gp-block/block-keywordize y))))
+       (let [result (gp-block/fix-duplicate-id (gp-block/block-keywordize x))]
+         (and (:block/uuid result)
+              (not= (:uuid x) (:block/uuid result))
+              (= (select-keys result
+                              [:block/properties :block/title :block/properties-text-values :block/properties-order]) (gp-block/block-keywordize y))))
     {:properties {:id "63f199bc-c737-459f-983d-84acfcda14fe"}, :tags [], :format :markdown, :meta {:start_pos 51, :end_pos 101}, :macros [], :title "bar\nid:: 63f199bc-c737-459f-983d-84acfcda14fe", :properties-text-values {:id "63f199bc-c737-459f-983d-84acfcda14fe"}, :level 1, :uuid #uuid "63f199bc-c737-459f-983d-84acfcda14fe", :properties-order [:id]}
     {:properties {},
      :title "bar",
@@ -45,61 +42,60 @@
 (deftest test-extract-properties
   (are [x y] (= (:properties (extract-properties x {})) y)
        ;; Built-in properties
-       [["background-color" "#000000"]] {:background-color "#000000"}
-       [["alias" "[[name/with space]]"]] {:alias #{"name/with space"}}
-       [["tags" "[[foo]], [[bar]]"]] {:tags #{"foo" "bar"}}
-       [["tags" "[[foo]] [[bar]]"]] {:tags #{"foo" "bar"}}
-       [["tags" "bar"]] {:tags #{"bar"}}
-       [["file-path" "file:///home/x, y.pdf"]] {:file-path "file:///home/x, y.pdf"}
+    [["background-color" "#000000"]] {:background-color "#000000"}
+    [["alias" "[[name/with space]]"]] {:alias #{"name/with space"}}
+    [["tags" "[[foo]], [[bar]]"]] {:tags #{"foo" "bar"}}
+    [["tags" "[[foo]] [[bar]]"]] {:tags #{"foo" "bar"}}
+    [["tags" "bar"]] {:tags #{"bar"}}
+    [["file-path" "file:///home/x, y.pdf"]] {:file-path "file:///home/x, y.pdf"}
 
        ;; User properties
-       [["year" "1000"]] {:year 1000}
-       [["year" "\"1000\""]] {:year "\"1000\""}
-       [["year" "1000"] ["alias" "[[name/with space]]"]] {:year 1000, :alias #{"name/with space"}}
-       [["year" "1000"] ["tags" "[[name/with space]]"]] {:year 1000, :tags #{"name/with space"}}
-       [["year" "1000"] ["tags" "[[name/with space]], [[another]]"]] {:year 1000, :tags #{"name/with space" "another"}}
-       [["year" "1000"] ["alias" "[[name/with space]], [[another]]"]] {:year 1000, :alias #{"name/with space" "another"}}
-       [["year" "1000"] ["alias" "[[name/with space]], [[another [[nested]]]]"]] {:year 1000, :alias #{"name/with space" "another [[nested]]"}}
-       [["year" "1000"] ["alias" "[[name/with space]], [[[[nested]] another]]"]] {:year 1000, :alias #{"name/with space" "[[nested]] another"}}
-       [["foo" "bar"]] {:foo "bar"}
-       [["foo" "[[bar]], [[baz]]"]] {:foo #{"bar" "baz"}}
-       [["foo" "[[bar]], [[baz]]"]] {:foo #{"bar" "baz"}}
-       [["foo" "[[bar]], [[baz]]"]] {:foo #{"bar" "baz"}}
-       [["foo" "[[bar]], [[nested [[baz]]]]"]] {:foo #{"bar" "nested [[baz]]"}}
-       [["foo" "[[bar]], [[nested [[baz]]]]"]] {:foo #{"bar" "nested [[baz]]"}}
-       [["foo" "[[bar]], [[baz, test]]"]] {:foo #{"bar" "baz, test"}}
-       [["foo" "[[bar]], [[baz, test, [[nested]]]]"]] {:foo #{"bar" "baz, test, [[nested]]"}})
+    [["year" "1000"]] {:year 1000}
+    [["year" "\"1000\""]] {:year "\"1000\""}
+    [["year" "1000"] ["alias" "[[name/with space]]"]] {:year 1000, :alias #{"name/with space"}}
+    [["year" "1000"] ["tags" "[[name/with space]]"]] {:year 1000, :tags #{"name/with space"}}
+    [["year" "1000"] ["tags" "[[name/with space]], [[another]]"]] {:year 1000, :tags #{"name/with space" "another"}}
+    [["year" "1000"] ["alias" "[[name/with space]], [[another]]"]] {:year 1000, :alias #{"name/with space" "another"}}
+    [["year" "1000"] ["alias" "[[name/with space]], [[another [[nested]]]]"]] {:year 1000, :alias #{"name/with space" "another [[nested]]"}}
+    [["year" "1000"] ["alias" "[[name/with space]], [[[[nested]] another]]"]] {:year 1000, :alias #{"name/with space" "[[nested]] another"}}
+    [["foo" "bar"]] {:foo "bar"}
+    [["foo" "[[bar]], [[baz]]"]] {:foo #{"bar" "baz"}}
+    [["foo" "[[bar]], [[baz]]"]] {:foo #{"bar" "baz"}}
+    [["foo" "[[bar]], [[baz]]"]] {:foo #{"bar" "baz"}}
+    [["foo" "[[bar]], [[nested [[baz]]]]"]] {:foo #{"bar" "nested [[baz]]"}}
+    [["foo" "[[bar]], [[nested [[baz]]]]"]] {:foo #{"bar" "nested [[baz]]"}}
+    [["foo" "[[bar]], [[baz, test]]"]] {:foo #{"bar" "baz, test"}}
+    [["foo" "[[bar]], [[baz, test, [[nested]]]]"]] {:foo #{"bar" "baz, test, [[nested]]"}})
 
   (testing "page-refs"
     (are [x y] (= (vec (:page-refs
                         (extract-properties x {:property-pages/enabled? true}))) y)
-         [["year" "1000"]] ["year"]
-         [["year" "\"1000\""]] ["year"]
-         [["year" "1000"] ["month" "12"]] ["year" "month"]
-         [["foo" "[[bar]] test"]] ["bar" "foo"]
-         [["foo" "[[bar]] test [[baz]]"]] ["bar" "baz" "foo"]
-         [["foo" "[[bar]] test [[baz]] [[nested [[baz]]]]"]] ["bar" "baz" "nested [[baz]]" "foo"]
-         [["foo" "#bar, #baz"]] ["bar" "baz" "foo"]
-         [["foo" "[[nested [[page]]]], test"]] ["nested [[page]]" "foo"])
-
+      [["year" "1000"]] ["year"]
+      [["year" "\"1000\""]] ["year"]
+      [["year" "1000"] ["month" "12"]] ["year" "month"]
+      [["foo" "[[bar]] test"]] ["bar" "foo"]
+      [["foo" "[[bar]] test [[baz]]"]] ["bar" "baz" "foo"]
+      [["foo" "[[bar]] test [[baz]] [[nested [[baz]]]]"]] ["bar" "baz" "nested [[baz]]" "foo"]
+      [["foo" "#bar, #baz"]] ["bar" "baz" "foo"]
+      [["foo" "[[nested [[page]]]], test"]] ["nested [[page]]" "foo"])
 
     (are [x y] (= (vec (:page-refs
                         (extract-properties x {:property-pages/enabled? false}))) y)
-         [["year" "1000"]] []
-         [["year" "1000"] ["month" "12"]] []
-         [["foo" "[[bar]] test"]] ["bar"])
+      [["year" "1000"]] []
+      [["year" "1000"] ["month" "12"]] []
+      [["foo" "[[bar]] test"]] ["bar"])
 
     (is (= ["year"]
            (:page-refs
             (extract-properties [["year" "1000"] ["month" "12"]]
-                                         {:property-pages/enabled? true
-                                          :property-pages/excludelist #{:month :day}})))
+                                {:property-pages/enabled? true
+                                 :property-pages/excludelist #{:month :day}})))
         ":property-pages/exclude-list excludes specified properties")
 
     (is (= ["year"]
            (:page-refs
             (extract-properties [["year" "1000"]]
-                                         {})))
+                                {})))
         "Default to enabled when :property-pages/enabled? is not in config")
 
     (is (= ["foo" "bar" "tags"]
@@ -107,7 +103,7 @@
             (extract-properties
              ;; tags is linkable and background-color is not
              [["tags" "[[foo]], [[bar]]"] ["background-color" "#008000"]]
-                                         {:property-pages/enabled? true})))
+             {:property-pages/enabled? true})))
         "Only editable linkable built-in properties have page-refs in property values")))
 
 (defn find-block-for-content
@@ -118,50 +114,4 @@
             db
             content)
        (map first)
-       first))
-
-(defn- parse-file
-  [conn file-path file-content & [options]]
-  (graph-parser/parse-file conn file-path file-content (merge-with merge options {:extract-options {:verbose false}})))
-
-(deftest refs-from-block-refs
-  (let [conn (gp-db/start-conn)
-        id "63f528da-284a-45d1-ac9c-5d6a7435f6b4"
-        block (str "A block\nid:: " id)
-        block-ref-via-content (str "Link to " (block-ref/->block-ref id))
-        block-ref-via-block-properties (str "B block\nref:: " (block-ref/->block-ref id))
-        body (str "- " block "\n- " block-ref-via-content "\n- " block-ref-via-block-properties)]
-    (parse-file conn "foo.md" body {})
-
-    (testing "Block refs in blocks"
-      (is (= [{:block/uuid (uuid id)}]
-             (:block/refs (find-block-for-content @conn block-ref-via-content)))
-          "Block that links to a block via paragraph content has correct block ref")
-
-      (is (contains?
-           (set (:block/refs (find-block-for-content @conn block-ref-via-block-properties)))
-           {:block/uuid (uuid id)})
-          "Block that links to a block via block properties has correct block ref"))
-
-    (testing "Block refs in pre-block"
-      (let [block-ref-via-page-properties (str "page-ref:: " (block-ref/->block-ref id))]
-        (parse-file conn "foo2.md" block-ref-via-page-properties {})
-        (is (contains?
-             (set (:block/refs (find-block-for-content @conn block-ref-via-page-properties)))
-             {:block/uuid (uuid id)})
-            "Block that links to a block via page properties has correct block ref")))))
-
-(deftest timestamp-blocks
-  (let [conn (gp-db/start-conn)
-        deadline-block "do something\nDEADLINE: <2023-02-21 Tue>"
-        scheduled-block "do something else\nSCHEDULED: <2023-02-20 Mon>"
-        body (str "- " deadline-block "\n- " scheduled-block)]
-    (parse-file conn "foo.md" body {})
-
-    (is (= 20230220
-           (:block/scheduled (find-block-for-content @conn scheduled-block)))
-        "Scheduled block has correct block attribute and value")
-
-    (is (= 20230221
-           (:block/deadline (find-block-for-content @conn deadline-block)))
-        "Deadline block has correct block attribute and value")))
+       first))

deps/graph-parser/test/logseq/graph_parser/cli_test.cljs

@@ -1,31 +0,0 @@
-(ns ^:node-only logseq.graph-parser.cli-test
-  (:require [cljs.test :refer [deftest is testing]]
-            [clojure.string :as string]
-            [datascript.core :as d]
-            [logseq.graph-parser.cli :as gp-cli]
-            [logseq.graph-parser.test.docs-graph-helper :as docs-graph-helper]))
-
-;; Integration test that test parsing a large graph like docs
-(deftest ^:integration parse-graph
-  (let [graph-dir "test/resources/docs-0.10.12"
-        _ (docs-graph-helper/clone-docs-repo-if-not-exists graph-dir "v0.10.12")
-        {:keys [conn files asts]} (gp-cli/parse-graph graph-dir {:verbose false})]
-
-    (docs-graph-helper/docs-graph-assertions @conn graph-dir files)
-
-    (testing "Additional counts"
-      (is (= 58149 (count (d/datoms @conn :eavt))) "Correct datoms count"))
-
-    (testing "Asts"
-      (is (seq asts) "Asts returned are non-zero")
-      (is (= files (map :file asts))
-          "There's an ast returned for every file processed")
-      (is (empty? (remove #(or
-                            (seq (:ast %))
-                            ;; edn files don't have ast
-                            (string/ends-with? (:file %) ".edn")
-                            ;; logseq files don't have ast
-                            ;; could also used common-config but API isn't public yet
-                            (string/includes? (:file %) (str graph-dir "/logseq/")))
-                          asts))
-          "Parsed files shouldn't have empty asts"))))

deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs

@@ -236,7 +236,6 @@
                             :where [?b :block/tags :logseq.class/Tag] [?b :db/ident ?ident] (not [?b :logseq.property/built-in?])])
                      count))
           "Correct number of user classes")
-      (is (= 4 (count (d/datoms @conn :avet :block/tags :logseq.class/Whiteboard))))
       (is (= 0 (count @(:ignored-properties import-state))) "No ignored properties")
       (is (= 0 (count @(:ignored-assets import-state))) "No ignored assets")
       (is (= 1 (count @(:ignored-files import-state))) "Ignore .edn for now")
@@ -607,13 +606,7 @@
                   :block/refs
                   (map #(:db/ident (d/entity @conn (:db/id %))))
                   set))
-            "Block has correct task tag and property :block/refs")))
-
-    (testing "whiteboards"
-      (let [block-with-props (db-test/find-block-by-content @conn #"block with props")]
-        (is (= {:user.property/prop-num 10}
-               (db-test/readable-properties block-with-props)))
-        (is (= "block with props" (:block/title block-with-props)))))))
+            "Block has correct task tag and property :block/refs")))))
 
 (deftest-async export-basic-graph-with-convert-all-tags-option-disabled
   (p/let [file-graph-dir "test/resources/exporter-test-graph"

deps/graph-parser/test/logseq/graph_parser/extract_test.cljs

@@ -1,10 +1,9 @@
 (ns logseq.graph-parser.extract-test
   (:require [cljs.test :refer [deftest is are]]
             [datascript.core :as d]
-            [logseq.db.file-based.schema :as file-schema]
             [logseq.graph-parser.extract :as extract]))
 
-;; This is a copy of frontend.util.fs/multiplatform-reserved-chars for reserved chars testing
+;; This is a copy of frontend.components.repo/multiplatform-reserved-chars for reserved chars testing
 (def multiplatform-reserved-chars ":\\*\\?\"<>|\\#\\\\")
 
 ;; Stuffs should be parsable (don't crash) when users dump some random files
@@ -42,10 +41,15 @@
   (is (= "asldk lakls" (#'extract/path->file-body "file://data/app/asldk lakls.as")))
   (is (= "中文asldk lakls" (#'extract/path->file-body "file://中文data/app/中文asldk lakls.as"))))
 
+;; Bare minimum schema to test extract
+(def file-schema
+  {:block/uuid {:db/unique :db.unique/identity}
+   :block/name {:db/unique :db.unique/identity}})
+
 (defn- extract [file content & [options]]
   (extract/extract file
                    content
-                   (merge {:block-pattern "-" :db (d/empty-db file-schema/schema)
+                   (merge {:block-pattern "-" :db (d/empty-db file-schema)
                            :verbose false}
                           options)))
 
@@ -124,26 +128,4 @@
           "- line1
     - line2
       - line3
-     - line4"))))
-
-(def foo-edn
-  "Example exported whiteboard page as an edn exportable."
-  '{:blocks
-    ({:block/title "foo content a",
-      :block/format :markdown},
-     {:block/title "foo content b",
-      :block/format :markdown}),
-    :pages
-    ({:block/format :markdown,
-      :block/title "Foo"
-      :block/uuid #uuid "a846e3b4-c41d-4251-80e1-be6978c36d8c"
-      :block/properties {:title "my whiteboard foo"}})})
-
-(deftest test-extract-whiteboard-edn
-  (let [{:keys [pages blocks]} (extract/extract-whiteboard-edn "/whiteboards/foo.edn" (pr-str foo-edn) {})
-        page (first pages)]
-    (is (= (get-in page [:block/file :file/path]) "/whiteboards/foo.edn"))
-    (is (= (:block/name page) "foo"))
-    (is (= (:block/type page) "whiteboard"))
-    (is (= (:block/title page) "Foo"))
-    (is (every? #(= (:block/parent %) [:block/uuid #uuid "a846e3b4-c41d-4251-80e1-be6978c36d8c"]) blocks))))
+     - line4"))))

deps/graph-parser/test/logseq/graph_parser/mldoc_test.cljs

@@ -1,47 +1,8 @@
 (ns logseq.graph-parser.mldoc-test
   (:require [cljs.test :refer [testing deftest are is]]
-            [clojure.string :as string]
-            [logseq.graph-parser.cli :as gp-cli]
             [logseq.graph-parser.mldoc :as gp-mldoc]
-            [logseq.graph-parser.test.docs-graph-helper :as docs-graph-helper]
             [logseq.graph-parser.text :as text]))
 
-(deftest test-link
-  (testing "non-link"
-    (are [x y] (= (gp-mldoc/link? :markdown x) y)
-      "google.com" false))
-
-  (testing "plain links"
-    (are [x y] (= (gp-mldoc/link? :markdown x) y)
-      "http://www.google.com" true
-      "http://google.com" true))
-
-  (testing "org links with labels"
-    (are [x y] (= (gp-mldoc/link? :org x) y)
-      "[[http://www.google.com][google]]" true
-      "[[http://google.com][google]]" true
-      "[[https://www.google.com][google]]" true
-      "[[https://google.com][google]]" true))
-
-  (testing "org links without labels"
-    (are [x y] (= (gp-mldoc/link? :org x) y)
-      "[[http://www.google.com]]" true
-      "[[https://www.google.com]]" true
-      "[[draws/2022-03-06-15-00-28.excalidraw]]" true
-      "[[assets/2022-03-06-15-00-28.pdf]]" true))
-
-  (testing "markdown links"
-    (are [x y] (= (gp-mldoc/link? :markdown x) y)
-      "[google](http://www.google.com)" true
-      "[google](https://www.google.com)" true
-      "[[draws/2022-03-06-15-00-28.excalidraw]]" true
-      "![a pdf](assets/2022-03-06-15-00-28.pdf)" true))
-
-  ;; https://github.com/logseq/logseq/issues/4308
-  (testing "parsing links should be finished"
-    (are [x y] (= (gp-mldoc/link? :markdown x) y)
-      "[YouTube](https://www.youtube.com/watch?v=-8ym7pyUs9gL) - [Vimeo](https://vimeo.com/677920303) {{youtube https://www.youtube.com/watch?v=-8ym7pyUs9g}}" true)))
-
 (def md-config (gp-mldoc/default-config :markdown))
 
 (deftest src-test
@@ -134,34 +95,4 @@ line 4"]
 \t      line 2
 \t line 3
 \tline 4"]
-              (gp-mldoc/remove-indentation-spaces s 3 false))))))
-
-(deftest ^:integration test->edn
-  (let [graph-dir "test/resources/docs-0.10.12"
-        _ (docs-graph-helper/clone-docs-repo-if-not-exists graph-dir "v0.10.12")
-        files (#'gp-cli/build-graph-files graph-dir {})
-        asts-by-file (->> files
-                          (map (fn [{:file/keys [path content]}]
-                                 (let [format (if (string/ends-with? path ".org")
-                                                :org :markdown)]
-                                   [path
-                                    (gp-mldoc/->edn content
-                                                    (gp-mldoc/default-config format))])))
-                          (into {}))]
-    (is (= {"Custom" 62,
-            "Displayed_Math" 2,
-            "Drawer" 1,
-            "Example" 22,
-            "Footnote_Definition" 2,
-            "Heading" 6764,
-            "Hiccup" 9,
-            "List" 25,
-            "Paragraph" 629,
-            "Properties" 85,
-            "Property_Drawer" 510,
-            "Quote" 28,
-            "Raw_Html" 18,
-            "Src" 82,
-            "Table" 8}
-           (->> asts-by-file (mapcat val) (map ffirst) frequencies))
-        "AST node type counts")))
+              (gp-mldoc/remove-indentation-spaces s 3 false))))))

deps/graph-parser/test/logseq/graph_parser/property_test.cljs

@@ -27,86 +27,6 @@
     "hello\n:PROPERTIES:\n:foo: bar\n:nice\n:END:\nnice"
     "hello\nfoo:: bar\n:nice\nnice"))
 
-(deftest test-insert-property
-  (are [x y] (= x y)
-    (gp-property/insert-property test-db :org "hello" "a" "b")
-    "hello\n:PROPERTIES:\n:a: b\n:END:"
-
-    (gp-property/insert-property test-db :org "hello" "a" false)
-    "hello\n:PROPERTIES:\n:a: false\n:END:"
-
-    (gp-property/insert-property test-db :org "hello\n:PROPERTIES:\n:a: b\n:END:\n" "c" "d")
-    "hello\n:PROPERTIES:\n:a: b\n:c: d\n:END:"
-
-    (gp-property/insert-property test-db :org "hello\n:PROPERTIES:\n:a: b\n:END:\nworld\n" "c" "d")
-    "hello\n:PROPERTIES:\n:a: b\n:c: d\n:END:\nworld"
-
-    (gp-property/insert-property test-db :org "#+BEGIN_QUOTE
- hello world
-  #+END_QUOTE" "c" "d")
-    ":PROPERTIES:\n:c: d\n:END:\n#+BEGIN_QUOTE\n hello world\n  #+END_QUOTE"
-
-    (gp-property/insert-property test-db :org "hello
-DEADLINE: <2021-10-25 Mon>
-SCHEDULED: <2021-10-25 Mon>" "a" "b")
-    "hello\nSCHEDULED: <2021-10-25 Mon>\nDEADLINE: <2021-10-25 Mon>\n:PROPERTIES:\n:a: b\n:END:"
-
-    (gp-property/insert-property test-db :org "hello
-DEADLINE: <2021-10-25 Mon>
-SCHEDULED: <2021-10-25 Mon>\n:PROPERTIES:\n:a: b\n:END:\n" "c" "d")
-    "hello\nDEADLINE: <2021-10-25 Mon>\nSCHEDULED: <2021-10-25 Mon>\n:PROPERTIES:\n:a: b\n:c: d\n:END:"
-
-    (gp-property/insert-property test-db :org "hello
-DEADLINE: <2021-10-25 Mon>
-SCHEDULED: <2021-10-25 Mon>\n:PROPERTIES:\n:a: b\n:END:\nworld\n" "c" "d")
-    "hello\nDEADLINE: <2021-10-25 Mon>\nSCHEDULED: <2021-10-25 Mon>\n:PROPERTIES:\n:a: b\n:c: d\n:END:\nworld"
-
-    (gp-property/insert-property test-db :markdown "hello\na:: b\nworld\n" "c" "d")
-    "hello\na:: b\nc:: d\nworld"
-
-    (gp-property/insert-property test-db :markdown "> quote" "c" "d")
-    "c:: d\n> quote"
-
-    (gp-property/insert-property test-db :markdown "#+BEGIN_QUOTE
- hello world
-  #+END_QUOTE" "c" "d")
-    "c:: d\n#+BEGIN_QUOTE\n hello world\n  #+END_QUOTE"))
-
-(deftest test-insert-properties
-  (are [x y] (= x y)
-    (gp-property/insert-properties test-db :markdown "" {:foo "bar"})
-    "foo:: bar"
-
-    (gp-property/insert-properties test-db :markdown "" {"foo" "bar"})
-    "foo:: bar"
-
-    (gp-property/insert-properties test-db :markdown "" {"foo space" "bar"})
-    "foo-space:: bar"
-
-    (gp-property/insert-properties test-db :markdown "" {:foo #{"bar" "baz"}})
-    "foo:: [[bar]], [[baz]]"
-
-    (gp-property/insert-properties test-db :markdown "" {:foo ["bar" "bar" "baz"]})
-    "foo:: [[bar]], [[baz]]"
-
-    (gp-property/insert-properties test-db :markdown "a\nb\n" {:foo ["bar" "bar" "baz"]})
-    "a\nfoo:: [[bar]], [[baz]]\nb"
-
-    (gp-property/insert-properties test-db :markdown "" {:foo "\"bar, baz\""})
-    "foo:: \"bar, baz\""
-
-    (gp-property/insert-properties test-db :markdown "abcd\nempty::" {:id "123" :foo "bar"})
-    "abcd\nempty::\nid:: 123\nfoo:: bar"
-
-    (gp-property/insert-properties test-db :markdown "abcd\nempty:: " {:id "123" :foo "bar"})
-    "abcd\nempty:: \nid:: 123\nfoo:: bar"
-
-    (gp-property/insert-properties test-db :markdown "abcd\nempty::" {:id "123"})
-    "abcd\nempty::\nid:: 123"
-
-    (gp-property/insert-properties test-db :markdown "abcd\nempty::\nanother-empty::" {:id "123"})
-    "abcd\nempty::\nanother-empty::\nid:: 123"))
-
 (deftest test-remove-properties
   (testing "properties with non-blank lines"
     (are [x y] (= x y)