Merge remote-tracking branch 'origin/master' into enhance/rtc-migrate

.clj-kondo/config.edn

@@ -72,7 +72,6 @@
              frontend.commands commands
              frontend.common.file-based.db common-file-db
              frontend.common.date common-date
-             frontend.common.file.core common-file
              frontend.common.file.util wfu
              frontend.common.missionary-util c.m
              frontend.common.schema-register sr
@@ -167,6 +166,9 @@
              lambdaisland.glogi log
              logseq.cli.common.graph cli-common-graph
              logseq.cli.text-util cli-text-util
+             logseq.cli.common.export.common cli-export-common
+             logseq.cli.common.export.text cli-export-text
+             logseq.cli.common.file common-file
              logseq.common.config common-config
              logseq.common.date-time-util date-time-util
              logseq.common.graph common-graph

.github/workflows/build-android.yml

@@ -43,7 +43,7 @@ on:
 env:
   CLOJURE_VERSION: '1.11.1.1413'
   NODE_VERSION: '22'
-  JAVA_VERSION: '17'
+  JAVA_VERSION: '21'
 
 jobs:
   build-apk:
@@ -108,7 +108,7 @@ jobs:
           echo "ENABLE_FILE_SYNC_PRODUCTION=${{ inputs.enable-file-sync-production || github.event.inputs.enable-file-sync-production || inputs.build-target == '' }}" >> $GITHUB_ENV
 
       - name: Compile CLJS - app variant, use es6 instead of es-next
-        run: yarn install && yarn release-app
+        run: yarn install && yarn release-mobile
         env:
           LOGSEQ_SENTRY_DSN: ${{ secrets.LOGSEQ_SENTRY_DSN }}
           LOGSEQ_POSTHOG_TOKEN: ${{ secrets.LOGSEQ_POSTHOG_TOKEN }}
@@ -126,15 +126,6 @@ jobs:
           SENTRY_ORG: logseq
           SENTRY_PROJECT: logseq
 
-      - name: Prepare public Directory
-        run: |
-          cp -r static public/
-          rm -rvf public/js/publishing
-          rm -rvf public/js/*.js.map || true
-          rm -rvf public/*.*
-          rm -rvf public/ios
-          rm -rvf android/app/src/main/assets/public || true
-
       - name: Sync public to Android Project
         run: npx cap sync android
 

.github/workflows/build-desktop-release.yml

@@ -38,11 +38,11 @@ on:
         type: boolean
         required: true
         default: true
-      # build-android:
-      #   description: 'Build Android App'
-      #   type: boolean
-      #   required: true
-      #   default: true
+      build-android:
+        description: 'Build Android App'
+        type: boolean
+        required: true
+        default: true
   # schedule: # Every workday at the 2 P.M. (UTC) we run a scheduled nightly build
   #   - cron: '0 14 * * MON-FRI'
 
@@ -492,17 +492,17 @@ jobs:
           path: builds
 
   # reuse workflow via workflow_call
-  # build-android:
-  #   uses: ./.github/workflows/build-android.yml
-  #   if: ${{ github.event_name == 'schedule' || github.event.inputs.build-android == 'true' }}
-  #   with:
-  #     build-target: "${{ github.event.inputs.build-target }}"
-  #     # if scheduled, use production mode
-  #     enable-file-sync-production: "${{ github.event_name == 'schedule' || github.event.inputs.enable-file-sync-production == 'true' }}"
-  #   secrets:
-  #     ANDROID_KEYSTORE: "${{ secrets.ANDROID_KEYSTORE }}"
-  #     ANDROID_KEYSTORE_PASSWORD: "${{ secrets.ANDROID_KEYSTORE_PASSWORD }}"
-  #     SENTRY_AUTH_TOKEN: "${{ secrets.SENTRY_AUTH_TOKEN }}"
+  build-android:
+    uses: ./.github/workflows/build-android.yml
+    if: ${{ github.event_name == 'schedule' || github.event.inputs.build-android == 'true' }}
+    with:
+      build-target: "${{ github.event.inputs.build-target }}"
+      # if scheduled, use production mode
+      enable-file-sync-production: "${{ github.event_name == 'schedule' || github.event.inputs.enable-file-sync-production == 'true' }}"
+    secrets:
+      ANDROID_KEYSTORE: "${{ secrets.ANDROID_KEYSTORE }}"
+      ANDROID_KEYSTORE_PASSWORD: "${{ secrets.ANDROID_KEYSTORE_PASSWORD }}"
+      SENTRY_AUTH_TOKEN: "${{ secrets.SENTRY_AUTH_TOKEN }}"
 
   codesign-windows:
     if: ${{ github.event_name == 'schedule' || github.event.inputs.build-target == 'nightly' || github.event.inputs.build-target == 'beta' }}
@@ -572,11 +572,11 @@ jobs:
           name: logseq-win64-builds
           path: ./
 
-      # - name: Download Android Artifacts
-      #   uses: actions/download-artifact@v4
-      #   with:
-      #     name: logseq-android-builds
-      #     path: ./
+      - name: Download Android Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: logseq-android-builds
+          path: ./
 
       - name: Generate SHA256 checksums
         run: |
@@ -652,12 +652,12 @@ jobs:
           name: logseq-win64-builds
           path: ./
 
-      # - name: Download Android Artifacts
-      #   uses: actions/download-artifact@v4
-      #   if: ${{ github.event_name == 'schedule' || github.event.inputs.build-android == 'true' }}
-      #   with:
-      #     name: logseq-android-builds
-      #     path: ./
+      - name: Download Android Artifacts
+        uses: actions/download-artifact@v4
+        if: ${{ github.event_name == 'schedule' || github.event.inputs.build-android == 'true' }}
+        with:
+          name: logseq-android-builds
+          path: ./
 
       - name: List files
         run: ls -rl

.github/workflows/build-ios-release.yml

@@ -25,7 +25,7 @@ jobs:
           ref: ${{ github.event.inputs.git-ref }}
       - uses: maxim-lobanov/setup-xcode@v1
         with:
-          xcode-version: 16.1
+          xcode-version: latest-stable
       - name: Install Node.js, NPM and Yarn
         uses: actions/setup-node@v4
         with:

.github/workflows/clj-e2e.yml

@@ -78,7 +78,7 @@ jobs:
       # NOTE: require the app to be build with DEV-RELEASE flag
       - name: Prepare E2E test build
         run: |
-          yarn gulp:build && clojure -M:cljs release app workers --config-merge "{:closure-defines {frontend.config/DEV-RELEASE true}}" --debug && yarn webpack-app-build
+          yarn gulp:build && clojure -M:cljs release app db-worker inference-worker --config-merge "{:closure-defines {frontend.config/DEV-RELEASE true}}" --debug && yarn webpack-app-build
 
       - name: Run e2e tests
         run: cd clj-e2e && timeout 30m bb dev

.github/workflows/clj-rtc-e2e.yml

@@ -79,7 +79,7 @@ jobs:
       # NOTE: require the app to be build with DEV-RELEASE flag
       - name: Prepare E2E test build
         run: |
-          yarn gulp:build && clojure -M:cljs release app workers --config-merge "{:closure-defines {frontend.config/DEV-RELEASE true}}" --debug && yarn webpack-app-build
+          yarn gulp:build && clojure -M:cljs release app db-worker inference-worker --config-merge "{:closure-defines {frontend.config/DEV-RELEASE true}}" --debug && yarn webpack-app-build
           rsync -avz --exclude node_modules --exclude android --exclude ios ./static/ ./public/
           ls -lR ./public
 

.github/workflows/deploy-db-test-pages.yml

@@ -42,7 +42,7 @@ jobs:
 
       - name: Build Released-Web
         run: |
-          yarn gulp:build && clojure -M:cljs release app workers  --config-merge '{:compiler-options {:source-map true :source-map-include-sources-content true :source-map-detail-level :symbols}}' && yarn webpack-app-build
+          yarn gulp:build && clojure -M:cljs release app db-worker inference-worker  --config-merge '{:compiler-options {:source-map true :source-map-include-sources-content true :source-map-detail-level :symbols}}' && yarn webpack-app-build
           rsync -avz --exclude node_modules --exclude android --exclude ios --exclude mobile ./static/ ./public/
           ls -lR ./public && mkdir r2 && mv ./public/js/main.js.map ./r2/db-test.main.js.map
           sed -i 's/=main.js.map/=https:\/\/assets.logseq.io\/db-test.main.js.map/g' ./public/js/main.js

CONTRIBUTING.md

@@ -36,10 +36,7 @@ issues](https://github.com/logseq/logseq/issues?q=is%3Aopen+is%3Aissue+label%3A%
 to help you get started. These are issues that are beginner-friendly and do not
 require advanced knowledge of the codebase. We encourage new contributors to
 start with these issues and gradually work their way up to more challenging
-tasks. We also have a project board to keep track of community contributions
-[Logseq - Develop Together
-💪](https://github.com/orgs/logseq/projects/5?query=is%3Aopen+sort%3Aupdated-desc).
-Another way to help with coding is by extending Logseq with
+tasks. Another way to help with coding is by extending Logseq with
 [plugins](https://docs.logseq.com/#/page/Plugins) and submitting them to the [marketplace](https://github.com/logseq/marketplace) so that the
 whole community can benefit.
 
@@ -100,7 +97,8 @@ behavior and design you'd like to add.
 code without a signed CLA.
 
 After doing the above, you are ready to work on your PR! To create a PR, fork
-this repository and then create a branch for the fix. Once you push your code to
+this repository and then create a branch for the fix. If contributing to the [database version](/README.md#-database-version), create a branch from master. If contributing to the file (text-only) version e.g. any version up to `0.10.X`, create a branch from [version/file](https://github.com/logseq/logseq/tree/version/file/).
+Once you push your code to
 your fork, you'll be able to open a PR to the Logseq repository. For more info,
 you can follow this [GitHub
 guide](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request-from-a-fork).

README.md

@@ -68,7 +68,7 @@
 
 ## 🚀 Database Version
 
-The Database version (DB version) of Logseq introduces DB graphs while maintaining support for file graphs. [See this page](https://github.com/logseq/docs/blob/master/db-version.md) to get an overview of the main features for DB graphs. If you are an existing user, [see changes with the DB version](https://github.com/logseq/docs/blob/master/db-version-changes.md). The DB version has its own new mobile app! To participate in the mobile app alpha, [please complete this brief form](https://forms.gle/nfefJv51jUuULbFB9). The DB version also has a new sync approach, RTC (Real Time Collaboration)! You can use it to sync graphs between multiple devices or collaborate with others. To participate in the RTC alpha, [please fill out this form](https://forms.gle/YSyF4WfKPSDuwyjH6).
+The Database version (DB version) of Logseq introduces DB graphs while maintaining support for file graphs. [See this page](https://github.com/logseq/docs/blob/master/db-version.md) to get an overview of the main features for DB graphs. If you are an existing user, [see changes with the DB version](https://github.com/logseq/docs/blob/master/db-version-changes.md). The DB version has its own new mobile app (on iOS, with Android coming soon)! To participate in the mobile app alpha, [please complete this brief form](https://forms.gle/nfefJv51jUuULbFB9). The DB version also has a new sync approach, RTC (Real Time Collaboration)! You can use it to sync graphs between multiple devices or collaborate with others. To participate in the RTC alpha, [please fill out this form](https://forms.gle/YSyF4WfKPSDuwyjH6).
 
 The DB version is in beta status while the new mobile app and RTC is in alpha. This means that **data loss is possible** so we recommend [automated backups](https://github.com/logseq/docs/blob/master/db-version.md#automated-backup) or [regular SQLite DB backups](https://github.com/logseq/docs/blob/master/db-version.md#graph-export). When using DB graphs, we recommend you create a dedicated test graph and choose one project that’s not crucial for you. When using file graphs, **data corruption is possible** as some file content can be duplicated. We only recommend using it with file graphs if you make regular backups with git.
 
@@ -111,7 +111,6 @@ That's it! You can now enjoy the benefits of using Logseq to streamline your wor
   * FAQ page: [Logseq Docs:  FAQ](https://docs.logseq.com/#/page/faq)
 * Blog: [blog.logseq.com](https://blog.logseq.com)
   * Please visit our [About page](https://blog.logseq.com/about) for the latest updates.
-* Logseq Hub: [hub.logseq.com](https://hub.logseq.com)
 * Forum: [discuss.logseq.com](https://discuss.logseq.com) - Where we answer questions, discuss workflows, and share tips
   * FAQ forum section: [Logseq Forum: FAQ](https://discuss.logseq.com/c/faq/6)
 * [Awesome Logseq](https://github.com/logseq/awesome-logseq) - Awesome Logseq extensions and resources created by the community <3

android/app/capacitor.build.gradle

@@ -1,15 +1,15 @@
 // DO NOT EDIT THIS FILE! IT IS GENERATED EACH TIME "capacitor update" IS RUN
 
 android {
-    compileOptions {
-        sourceCompatibility JavaVersion.VERSION_21
-        targetCompatibility JavaVersion.VERSION_21
-    }
+  compileOptions {
+      sourceCompatibility JavaVersion.VERSION_21
+      targetCompatibility JavaVersion.VERSION_21
+  }
 }
 
 apply from: "../capacitor-cordova-android-plugins/cordova.variables.gradle"
 dependencies {
-    implementation 'androidx.core:core-ktx:1.16.0'
+    implementation project(':capacitor-community-safe-area')
     implementation project(':capacitor-action-sheet')
     implementation project(':capacitor-app')
     implementation project(':capacitor-camera')
@@ -17,11 +17,10 @@ dependencies {
     implementation project(':capacitor-filesystem')
     implementation project(':capacitor-haptics')
     implementation project(':capacitor-keyboard')
+    implementation project(':capacitor-network')
     implementation project(':capacitor-share')
     implementation project(':capacitor-splash-screen')
     implementation project(':capacitor-status-bar')
-    implementation project(':capawesome-capacitor-background-task')
-    implementation project(':capgo-capacitor-navigation-bar')
     implementation project(':capacitor-voice-recorder')
     implementation project(':send-intent')
     implementation project(':jcesarmobile-ssl-skip')
@@ -30,5 +29,5 @@ dependencies {
 
 
 if (hasProperty('postBuildExtras')) {
-    postBuildExtras()
+  postBuildExtras()
 }

android/app/src/main/AndroidManifest.xml

@@ -11,21 +11,22 @@
     <uses-permission android:name="android.permission.RECORD_AUDIO" />
 
     <application
-        android:networkSecurityConfig="@xml/network_security_config"
         android:allowBackup="true"
         android:icon="@mipmap/ic_launcher"
         android:label="@string/app_name"
+        android:networkSecurityConfig="@xml/network_security_config"
+        android:requestLegacyExternalStorage="true"
         android:roundIcon="@mipmap/ic_launcher_round"
         android:supportsRtl="true"
-        android:requestLegacyExternalStorage="true"
         android:theme="@style/AppTheme">
         <activity
-            android:exported="true"
-            android:configChanges="orientation|keyboardHidden|keyboard|screenSize|locale|smallestScreenSize|screenLayout|uiMode|navigation"
             android:name="com.logseq.app.MainActivity"
+            android:configChanges="orientation|keyboardHidden|keyboard|screenSize|locale|smallestScreenSize|screenLayout|uiMode|navigation"
+            android:exported="true"
+            android:windowSoftInputMode="adjustNothing"
             android:label="@string/title_activity_main"
-            android:theme="@style/AppTheme.NoActionBarLaunch"
-            android:launchMode="singleTask">
+            android:launchMode="singleTask"
+            android:theme="@style/AppTheme.NoActionBarLaunch">
             <intent-filter>
                 <action android:name="android.intent.action.MAIN" />
                 <category android:name="android.intent.category.LAUNCHER" />
@@ -34,6 +35,7 @@
             <intent-filter>
                 <action android:name="android.intent.action.SEND" />
                 <category android:name="android.intent.category.DEFAULT" />
+
                 <data android:mimeType="text/plain" />
                 <data android:mimeType="image/*" />
                 <data android:mimeType="application/*" />
@@ -42,8 +44,10 @@
 
             <intent-filter>
                 <action android:name="android.intent.action.VIEW" />
+
                 <category android:name="android.intent.category.DEFAULT" />
                 <category android:name="android.intent.category.BROWSABLE" />
+
                 <data android:scheme="logseq" />
             </intent-filter>
         </activity>
@@ -53,7 +57,9 @@
             android:authorities="${applicationId}.fileprovider"
             android:exported="false"
             android:grantUriPermissions="true">
-            <meta-data android:name="android.support.FILE_PROVIDER_PATHS" android:resource="@xml/file_paths" />
+            <meta-data
+                android:name="android.support.FILE_PROVIDER_PATHS"
+                android:resource="@xml/file_paths" />
         </provider>
     </application>
 </manifest>

android/app/src/main/assets/capacitor.plugins.json

@@ -1,4 +1,8 @@
 [
+	{
+		"pkg": "@capacitor-community/safe-area",
+		"classpath": "com.getcapacitor.community.safearea.SafeAreaPlugin"
+	},
 	{
 		"pkg": "@capacitor/action-sheet",
 		"classpath": "com.capacitorjs.plugins.actionsheet.ActionSheetPlugin"
@@ -27,6 +31,10 @@
 		"pkg": "@capacitor/keyboard",
 		"classpath": "com.capacitorjs.plugins.keyboard.KeyboardPlugin"
 	},
+	{
+		"pkg": "@capacitor/network",
+		"classpath": "com.capacitorjs.plugins.network.NetworkPlugin"
+	},
 	{
 		"pkg": "@capacitor/share",
 		"classpath": "com.capacitorjs.plugins.share.SharePlugin"
@@ -39,14 +47,6 @@
 		"pkg": "@capacitor/status-bar",
 		"classpath": "com.capacitorjs.plugins.statusbar.StatusBarPlugin"
 	},
-	{
-		"pkg": "@capawesome/capacitor-background-task",
-		"classpath": "io.capawesome.capacitorjs.plugins.backgroundtask.BackgroundTaskPlugin"
-	},
-	{
-		"pkg": "@capgo/capacitor-navigation-bar",
-		"classpath": "ee.forgr.capacitor_navigation_bar.NavigationBarPlugin"
-	},
 	{
 		"pkg": "capacitor-voice-recorder",
 		"classpath": "com.tchvu3.capacitorvoicerecorder.VoiceRecorder"

android/app/src/main/java/com/logseq/app/MainActivity.java

@@ -1,8 +1,13 @@
 package com.logseq.app;
 
 import android.content.Intent;
+import android.content.res.Configuration;
+import android.os.Build;
 import android.os.Bundle;
+import android.view.View;
+import android.view.Window;
 import android.webkit.ValueCallback;
+import android.webkit.WebView;
 
 import com.getcapacitor.BridgeActivity;
 
@@ -16,6 +21,12 @@ public class MainActivity extends BridgeActivity {
         registerPlugin(UILocal.class);
 
         super.onCreate(savedInstanceState);
+        WebView webView = getBridge().getWebView();
+        webView.setOverScrollMode(WebView.OVER_SCROLL_NEVER);
+        webView.getSettings().setUseWideViewPort(true);
+        webView.getSettings().setLoadWithOverviewMode(true);
+
+        setNavigationBarColorBasedOnTheme();
 
         new Timer().schedule(new TimerTask() {
             @Override
@@ -51,4 +62,22 @@ public class MainActivity extends BridgeActivity {
             });
         }
     }
+
+    private void setNavigationBarColorBasedOnTheme() {
+        Window window = getWindow();
+        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { // API 26及以上支持
+            // 根据主题选择颜色
+            int navBarColor = isDarkMode()
+                    ? getResources().getColor(R.color.colorPrimaryDark, getTheme())
+                    : getResources().getColor(R.color.colorPrimary, getTheme());
+
+            // 设置导航栏颜色
+            window.setNavigationBarColor(navBarColor);
+        }
+    }
+
+    private boolean isDarkMode() {
+        int nightModeFlags = getResources().getConfiguration().uiMode & Configuration.UI_MODE_NIGHT_MASK;
+        return nightModeFlags == Configuration.UI_MODE_NIGHT_YES;
+    }
 }

android/app/src/main/res/values/colors.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="utf-8"?>
 <resources>
-    <color name="logoPrimary">#002b36</color>
-    <color name="colorPrimary">#ffffff</color>
-    <color name="colorPrimaryDark">#002b36</color>
+    <color name="logoPrimary">#0f262e</color>
+    <color name="colorPrimary">#f7f7f7</color>
+    <color name="colorPrimaryDark">#0f262e</color>
 </resources>

android/capacitor.settings.gradle

@@ -2,6 +2,9 @@
 include ':capacitor-android'
 project(':capacitor-android').projectDir = new File('../node_modules/@capacitor/android/capacitor')
 
+include ':capacitor-community-safe-area'
+project(':capacitor-community-safe-area').projectDir = new File('../node_modules/@capacitor-community/safe-area/android')
+
 include ':capacitor-action-sheet'
 project(':capacitor-action-sheet').projectDir = new File('../node_modules/@capacitor/action-sheet/android')
 
@@ -23,6 +26,9 @@ project(':capacitor-haptics').projectDir = new File('../node_modules/@capacitor/
 include ':capacitor-keyboard'
 project(':capacitor-keyboard').projectDir = new File('../node_modules/@capacitor/keyboard/android')
 
+include ':capacitor-network'
+project(':capacitor-network').projectDir = new File('../node_modules/@capacitor/network/android')
+
 include ':capacitor-share'
 project(':capacitor-share').projectDir = new File('../node_modules/@capacitor/share/android')
 
@@ -32,12 +38,6 @@ project(':capacitor-splash-screen').projectDir = new File('../node_modules/@capa
 include ':capacitor-status-bar'
 project(':capacitor-status-bar').projectDir = new File('../node_modules/@capacitor/status-bar/android')
 
-include ':capawesome-capacitor-background-task'
-project(':capawesome-capacitor-background-task').projectDir = new File('../node_modules/@capawesome/capacitor-background-task/android')
-
-include ':capgo-capacitor-navigation-bar'
-project(':capgo-capacitor-navigation-bar').projectDir = new File('../node_modules/@capgo/capacitor-navigation-bar/android')
-
 include ':capacitor-voice-recorder'
 project(':capacitor-voice-recorder').projectDir = new File('../node_modules/capacitor-voice-recorder/android')
 

capacitor.config.ts

@@ -9,7 +9,7 @@ const config: CapacitorConfig = {
   webDir: 'static/mobile',
   loggingBehavior: 'debug',
   server: {
-      androidScheme: 'http',
+    androidScheme: 'http',
   },
   plugins: {
     StatusBar: {
@@ -28,6 +28,16 @@ const config: CapacitorConfig = {
 
     Keyboard: {
       resize: 'none'
+    },
+
+    SafeArea: {
+      enabled: true,
+      customColorsForSystemBars: true,
+      statusBarColor: '#000000',
+      statusBarContent: 'light',
+      navigationBarColor: '#000000',
+      navigationBarContent: 'light',
+      offset: 0
     }
   },
   android: {

clj-e2e/.clj-kondo/config.edn

@@ -1,3 +1,4 @@
 {:linters
- {:unresolved-var {:exclude [wally.main/click
+ {:redundant-let {:level :off}
+  :unresolved-var {:exclude [wally.main/click
                              wally.main/fill]}}}

clj-e2e/test/logseq/e2e/editor_basic_test.clj

@@ -31,8 +31,9 @@
     (b/select-blocks 3)
     (b/toggle-property "Tags" "Page")
     (assert/assert-is-visible ".ls-page-blocks .ls-block .ls-icon-file")
-    (w/wait-for (format "#ac-0.menu-link:has-text('%s')" "Page"))
-    (k/enter)
+    (w/wait-for (format ".menu-link:has-text('%s')" "Page"))
+    (k/esc)
+    (b/toggle-property "Tags" "Page")
     (w/wait-for-not-visible ".ls-page-blocks .ls-block .ls-icon-file")))
 
 (deftest disallow-adding-page-tag-to-normal-pages
@@ -66,6 +67,7 @@
     (w/fill "input[placeholder=\"Move blocks to\"]" "Target page 2")
     (w/wait-for (w/get-by-test-id "Target page 2"))
     (.focus (w/-query ".cp__cmdk-search-input"))
+    (k/arrow-down)
     (k/enter)
     (assert/assert-have-count ".ls-page-blocks .page-blocks-inner .ls-block" 0)))
 
@@ -75,6 +77,8 @@
     (p/new-page "test page")
     (b/new-blocks ["block1" "block2" "block3"])
     (b/select-blocks 3)
+    (b/toggle-property "Tags" "Page")
+    (assert/assert-is-visible ".ls-page-blocks .ls-block .ls-icon-file")
     (k/press "ControlOrMeta+Shift+m")
     (w/fill "input[placeholder=\"Move blocks to\"]" "Library")
     (w/wait-for (w/get-by-test-id "Library"))
@@ -86,6 +90,8 @@
     (p/goto-page "test page")
     (b/new-blocks ["block4" "block5"])
     (b/select-blocks 2)
+    (b/toggle-property "Tags" "Page")
+    (assert/assert-is-visible ".ls-page-blocks .ls-block .ls-icon-file")
     (k/press "ControlOrMeta+Shift+m")
     (w/fill "input[placeholder=\"Move blocks to\"]" "Library")
     (w/wait-for (w/get-by-test-id "Library"))

clj-e2e/test/logseq/e2e/plugins_basic_test.clj

@@ -61,6 +61,8 @@
     (page/new-page "test-block-apis")
     (ls-api-call! :ui.showMsg "hello world" "info")
     (let [ret (ls-api-call! :editor.appendBlockInPage "test-block-apis" "append-block-in-page-0")
+          ret1 (ls-api-call! :editor.appendBlockInPage "append-block-in-current-page-0")
+          _ (assert-api-ls-block! ret1)
           uuid' (assert-api-ls-block! ret)]
       (-> (ls-api-call! :editor.insertBlock uuid' "insert-0")
           (assert-api-ls-block!))
@@ -75,11 +77,13 @@
     (let [ret (ls-api-call! :editor.appendBlockInPage "test-block-properties-apis" "block-in-page-0" {:properties {:p1 1}})
           uuid' (assert-api-ls-block! ret)
           prop1 (ls-api-call! :editor.getBlockProperty uuid' "p1")
-          props1 (ls-api-call! :editor.getBlockProperties uuid' "p1")]
+          props1 (ls-api-call! :editor.getBlockProperties uuid' "p1")
+          props2 (ls-api-call! :editor.getPageProperties "test-block-properties-apis")]
       (w/wait-for ".property-k:text('p1')")
       (is (= 1 (get prop1 "value")))
       (is (= (get prop1 "ident") ":plugin.property._api/p1"))
       (is (= 1 (get props1 ":plugin.property._api/p1")))
+      (is (= ["Page"] (get props2 ":block/tags")))
       (ls-api-call! :editor.upsertBlockProperty uuid' "p2" "p2")
       (ls-api-call! :editor.upsertBlockProperty uuid' "p3" true)
       (ls-api-call! :editor.upsertBlockProperty uuid' "p4" {:a 1, :b [2, 3]})

deps.edn

@@ -5,7 +5,7 @@
                                          :sha     "5d672bf84ed944414b9f61eeb83808ead7be9127"}
 
   datascript/datascript                 {:git/url "https://github.com/logseq/datascript" ;; fork
-                                         :sha     "b28f6574b9447bba9ccaa5d2b0cfd79308acf0e3"}
+                                         :sha     "45f6721bf2038c24eb9fe3afb422322ab3f473b5"}
 
   datascript-transit/datascript-transit {:mvn/version "0.3.0"}
   borkdude/rewrite-edn                  {:mvn/version "0.4.9"}

deps/cli/.carve/ignore

@@ -4,3 +4,5 @@ logseq.cli.commands.graph/show-graph
 logseq.cli.commands.graph/list-graphs
 logseq.cli.commands.query/query
 logseq.cli.commands.search/search
+logseq.cli.commands.export/export
+logseq.cli.commands.append/append

deps/cli/.clj-kondo/config.edn

@@ -2,7 +2,8 @@
  {:aliased-namespace-symbol {:level :warning}
   :namespace-name-mismatch {:level :warning}
   :used-underscored-binding {:level :warning}
-  :shadowed-var {:level :warning}
+  :shadowed-var {:level :warning
+                 :exclude [meta time min meta name type]}
 
   :consistent-alias
   {:aliases {"fs" fs
@@ -12,6 +13,9 @@
              datascript.core d
              logseq.cli.commands.graph cli-graph
              logseq.cli.common.graph cli-common-graph
+             logseq.cli.common.export.text cli-export-text
+             logseq.cli.common.export.common cli-export-common
+             logseq.cli.common.file common-file
              logseq.cli.util cli-util
              logseq.cli.text-util cli-text-util
              logseq.common.config common-config

deps/cli/CHANGELOG.md

@@ -1,6 +1,14 @@
+## 0.2.0
+* Add export command to export graph as markdown
+* Add append command to add text to current page
+* Change export-edn command to default to writing to a file, like the export command
+* Rename --api-query-token options to --api-server-token
+* API related commands can also authenticate with $LOGSEQ_API_SERVER_TOKEN
+* Add descriptions for most commands to explain in-depth usage
+
 ## 0.1.0
 
 * Initial release!
 * Provides commands: list, show, search, query, export-edn and help
-* All commands except search work offline. search and query have options for calling HTTP Server of
+* All commands work offline. search and query have options for calling HTTP API Server of
   open desktop Logseq app

deps/cli/README.md

@@ -12,7 +12,7 @@ This section assumes you have installed the CLI from npm or via the [dev
 setup](#setup). If you haven't, substitute `node cli.mjs` for `logseq` e.g.
 `node.cli.mjs -h`.
 
-All commands can be used offline or on CI. The `search` command and any command that has an api-query-token option require the [HTTP API Server](https://docs.logseq.com/#/page/local%20http%20server) to be turned on.
+All commands except for `append` can be used offline or on CI. The `search` command and any command that has an api-server-token option require the [HTTP API Server](https://docs.logseq.com/#/page/local%20http%20server) to be turned on.
 
 Now let's use it!
 
@@ -26,12 +26,13 @@ Options:
 Commands:
 list                 List graphs
 show                 Show DB graph(s) info
-search [options]     Search current DB graph
+search [options]     Search DB graph
 query [options]      Query DB graph(s)
+export [options]     Export DB graph as Markdown
 export-edn [options] Export DB graph as EDN
+append [options]     Appends text to current page
 help                 Print a command's help
 
-
 $ logseq list
 DB Graphs:
 db-test
@@ -60,6 +61,9 @@ Search found 100 results:
 dev:db-export woot woot.edn && dev:db-create woot2 woot.edn
 dev:db-diff woot woot2
 ...
+# Can also authenticate api with $LOGSEQ_API_SERVER_TOKEN
+$ LOGSEQ_API_SERVER_TOKEN=my-token logseq search woot
+...
 
 # Search a local graph
 $ logseq search woot page
@@ -110,9 +114,17 @@ $ logseq query '(task DOING)' -a my-token
   :uuid "68795144-e5f6-48e8-849d-79cd6473b952"}
   ...
 
-# Export your DB graph as EDN
+# Export DB graph as markdown
+$ logseq export yep
+Exported 41 pages to yep_markdown_1756128259.zip
+
+# Export DB graph as EDN
 $ logseq export-edn woot -f woot.edn
 Exported 16 properties, 16 classes and 36 pages
+
+# Append text to current page
+$ logseq append add this text -a my-token
+Success!
 ```
 
 ## API

deps/cli/bb.edn

@@ -40,5 +40,5 @@
 
  :tasks/config
  {:large-vars
-  {:max-lines-count 30
+  {:max-lines-count 45
    :metadata-exceptions #{:large-vars/cleanup-todo}}}}

deps/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@logseq/cli",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "Logseq CLI",
   "bin": {
     "logseq": "cli.mjs"
@@ -10,9 +10,11 @@
   },
   "license": "MIT",
   "dependencies": {
-    "@logseq/nbb-logseq": "github:logseq/nbb-logseq#feat-db-v26",
+    "@logseq/nbb-logseq": "github:logseq/nbb-logseq#feat-db-v28",
     "better-sqlite3": "~11.10.0",
-    "fs-extra": "^11.3.0"
+    "fs-extra": "^11.3.0",
+    "jszip": "3.8.0",
+    "mldoc": "^1.5.9"
   },
   "repository": {
     "type": "git",

deps/cli/src/logseq/cli.cljs

@@ -6,6 +6,7 @@
             [clojure.string :as string]
             [logseq.cli.common.graph :as cli-common-graph]
             [logseq.cli.spec :as cli-spec]
+            [logseq.cli.text-util :as cli-text-util]
             [nbb.error]
             [promesa.core :as p]))
 
@@ -38,15 +39,20 @@
                      (aget "version")))))
     (help m)))
 
-(defn- command-help [{{:keys [command]} :opts}]
+(defn- print-command-help [command cmd-map]
+  (println (str "Usage: logseq " command
+                (when (:args->opts cmd-map)
+                  (str " " (string/join " "
+                                        (map #(str "[" (name %) "]") (:args->opts cmd-map)))))
+                (when (:spec cmd-map)
+                  (str " [options]\n\nOptions:\n"
+                       (cli/format-opts {:spec (:spec cmd-map)})))
+                (when (:description cmd-map)
+                  (str "\n\nDescription:\n" (cli-text-util/wrap-text (:description cmd-map) 80))))))
+
+(defn- help-command [{{:keys [command]} :opts}]
   (if-let [cmd-map (and command (some #(when (= command (first (:cmds %))) %) table))]
-    (println (str "Usage: logseq " command
-                  (when (:args->opts cmd-map)
-                    (str " " (string/join " "
-                                          (map #(str "[" (name %) "]") (:args->opts cmd-map)))))
-                  (when (:spec cmd-map)
-                    (str " [options]\n\nOptions:\n"
-                         (cli/format-opts {:spec (:spec cmd-map)})))))
+    (print-command-help command cmd-map)
     (println "Command" (pr-str command) "does not exist")))
 
 (defn- lazy-load-fn
@@ -65,22 +71,35 @@
   [{:cmds ["list"] :desc "List graphs"
     :fn (lazy-load-fn 'logseq.cli.commands.graph/list-graphs)}
    {:cmds ["show"] :desc "Show DB graph(s) info"
+    :description "For each graph, prints information related to a graph's creation and anything that is helpful for debugging."
     :fn (lazy-load-fn 'logseq.cli.commands.graph/show-graph)
     :args->opts [:graphs] :coerce {:graphs []} :require [:graphs]}
    {:cmds ["search"]
     :fn (lazy-load-fn 'logseq.cli.commands.search/search)
     :desc "Search DB graph"
+    :description "Search a local graph or the current in-app graph if --api-server-token is given. For a local graph it only searches the :block/title of blocks."
     :args->opts [:graph :search-terms] :coerce {:search-terms []} :require [:graph]
     :spec cli-spec/search}
    {:cmds ["query"] :desc "Query DB graph(s)"
+    :description "Query a local graph or the current in-app graph if --api-server-token is given. For a local graph, queries are a datalog query or an entity query. A datalog query can use built-in rules. An entity query consists of one or more integers, uuids or :db/ident keywords. For an in-app query, queries can be an advanced or simple query."
     :fn (lazy-load-fn 'logseq.cli.commands.query/query)
     :args->opts [:graph :args] :coerce {:args []} :no-keyword-opts true :require [:graph]
     :spec cli-spec/query}
+   {:cmds ["export"] :desc "Export DB graph as Markdown"
+    :description "Export a graph to Markdown like the in-app graph export."
+    :fn (lazy-load-fn 'logseq.cli.commands.export/export)
+    :args->opts [:graph] :require [:graph]
+    :spec cli-spec/export}
    {:cmds ["export-edn"] :desc "Export DB graph as EDN"
+    :description "Export a graph to EDN like the in-app graph EDN export. See https://github.com/logseq/docs/blob/master/db-version.md#edn-data-export for more about this export type."
     :fn (lazy-load-fn 'logseq.cli.commands.export-edn/export)
     :args->opts [:graph] :require [:graph]
     :spec cli-spec/export-edn}
-   {:cmds ["help"] :fn command-help :desc "Print a command's help"
+   {:cmds ["append"] :desc "Appends text to current page"
+    :fn (lazy-load-fn 'logseq.cli.commands.append/append)
+    :args->opts [:args] :require [:args] :coerce {:args []}
+    :spec cli-spec/append}
+   {:cmds ["help"] :fn help-command :desc "Print a command's help"
     :args->opts [:command] :require [:command]}
    {:cmds []
     :spec default-spec
@@ -101,12 +120,18 @@
                   {:error-fn (fn [{:keys [cause msg option] type' :type :as data}]
                                (if (and (= :org.babashka/cli type')
                                         (= :require cause))
-                                 (println "Error: Command missing required"
-                                          (if (get-in data [:spec option]) "option" "argument")
-                                          option)
+                                 (do
+                                   (println "Error: Command missing required"
+                                            (if (get-in data [:spec option]) "option" "argument")
+                                            (pr-str (name option)))
+                                   (when-let [cmd-m (some #(when (= {:spec (:spec %)
+                                                                     :require (:require %)}
+                                                                    (select-keys data [:spec :require])) %) table)]
+                                     (print-command-help (-> cmd-m :cmds first) cmd-m)))
                                  (throw (ex-info msg data)))
                                (js/process.exit 1))})
     (catch ^:sci/error js/Error e
-      (nbb.error/print-error-report e))))
+      (nbb.error/print-error-report e)
+      (js/process.exit 1))))
 
 #js {:main -main}

deps/cli/src/logseq/cli/commands/append.cljs

@@ -0,0 +1,14 @@
+(ns logseq.cli.commands.append
+  "Command to append to a page"
+  (:require [clojure.string :as string]
+            [logseq.cli.util :as cli-util]
+            [promesa.core :as p]))
+
+(defn append
+  [{{:keys [api-server-token args]} :opts}]
+  (let [text (string/join " " args)]
+    (-> (p/let [resp (cli-util/api-fetch api-server-token "logseq.app.append_block_in_page" [text nil nil])]
+          (if (= 200 (.-status resp))
+            (println "Success!")
+            (cli-util/api-handle-error-response resp)))
+        (p/catch cli-util/command-catch-handler))))

deps/cli/src/logseq/cli/commands/export.cljs

@@ -0,0 +1,80 @@
+(ns logseq.cli.commands.export
+  "Export MD command"
+  (:require ["fs" :as fs]
+            [cljs.pprint]
+            [clojure.string :as string]
+            [datascript.core :as d]
+            [logseq.cli.common.export.common :as cli-export-common]
+            [logseq.cli.common.export.text :as cli-export-text]
+            [logseq.cli.common.file :as common-file]
+            [logseq.cli.common.util :as cli-common-util]
+            [logseq.cli.util :as cli-util]
+            [logseq.common.config :as common-config]
+            [logseq.common.util :as common-util]
+            [logseq.db.common.sqlite-cli :as sqlite-cli]))
+
+(defn- get-content-config [db]
+  (let [repo-config (-> (d/q '[:find ?content
+                               :where [?b :file/path "logseq/config.edn"] [?b :file/content ?content]]
+                             db)
+                        ffirst
+                        common-util/safe-read-map-string)
+        indent
+        ;; Copy of state/get-export-bullet-indentation
+        (case (get repo-config :export/bullet-indentation :tab)
+          :eight-spaces
+          "        "
+          :four-spaces
+          "    "
+          :two-spaces
+          "  "
+          :tab
+          "\t")]
+    {:export-bullet-indentation indent}))
+
+(defn- get-file-contents
+  "Modified version of export.common/<get-file-contents which doesn't have to deal with worker threads"
+  [repo db content-config suffix]
+  (let [page->content (common-file/get-all-page->content repo db content-config)]
+    (map (fn [[page-title content]]
+           {:path (str page-title "." suffix)
+            :content content
+            :title page-title
+            :format :markdown})
+         page->content)))
+
+(defn- export-files-as-markdown
+  "Modified version of handler.export.text/export-files-as-markdown for the CLI"
+  [repo files options]
+  (mapv
+   (fn [{:keys [path title content]}]
+     [(or path title) (cli-export-text/export-helper repo content :markdown options)])
+   files))
+
+(defn- export-repo-as-markdown!
+  "Modified version of handler.export.text/export-repo-as-markdown for the CLI"
+  [repo db {:keys [file]}]
+  (let [content-config (get-content-config db)
+        files* (get-file-contents repo db content-config "md")]
+    (when (seq files*)
+      (let [files (binding [cli-export-common/*current-db* db
+                            cli-export-common/*current-repo* repo
+                            cli-export-common/*content-config* content-config]
+                    (export-files-as-markdown repo files* nil))
+            repo' (string/replace repo common-config/db-version-prefix "")
+            zip-file-name (if file
+                            (string/replace-first file #"(?i)\.zip$" "")
+                            (str repo' "_markdown_" (quot (common-util/time-ms) 1000)))
+            file-name (or file (str zip-file-name ".zip"))
+            zip (cli-common-util/make-export-zip zip-file-name files)
+            ;; matches behavior in make-export-zip
+            exported-files (remove #(string/blank? (second %)) files)]
+        (-> (.generateNodeStream zip #js {:streamFiles true :type "nodebuffer"})
+            (.pipe (fs/createWriteStream file-name)))
+        (println "Exported" (count exported-files) "pages to" file-name)))))
+
+(defn export [{{:keys [graph] :as opts} :opts}]
+  (if (fs/existsSync (cli-util/get-graph-dir graph))
+    (let [conn (apply sqlite-cli/open-db! (cli-util/->open-db-args graph))]
+      (export-repo-as-markdown! (str common-config/db-version-prefix graph) @conn opts))
+    (cli-util/error "Graph" (pr-str graph) "does not exist")))

deps/cli/src/logseq/cli/commands/export_edn.cljs

@@ -4,21 +4,20 @@
             [clojure.pprint :as pprint]
             [logseq.db.common.sqlite-cli :as sqlite-cli]
             [logseq.db.sqlite.export :as sqlite-export]
+            [logseq.common.util :as common-util]
             [logseq.cli.util :as cli-util]))
 
 (defn export [{{:keys [graph] :as options} :opts}]
   (if (fs/existsSync (cli-util/get-graph-dir graph))
-   (let [conn (apply sqlite-cli/open-db! (cli-util/->open-db-args graph))
-         export-map (sqlite-export/build-export @conn
-                                                (cond-> {:export-type (:export-type options)}
-                                                  (= :graph (:export-type options))
-                                                  (assoc :graph-options (dissoc options :file :export-type :graph))))]
-     (if (:file options)
-       (do
-         (println "Exported" (count (:properties export-map)) "properties,"
-                  (count (:properties export-map)) "classes and"
-                  (count (:pages-and-blocks export-map)) "pages")
-         (fs/writeFileSync (:file options)
-                           (with-out-str (pprint/pprint export-map))))
-       (pprint/pprint export-map)))
+    (let [conn (apply sqlite-cli/open-db! (cli-util/->open-db-args graph))
+          export-map (sqlite-export/build-export @conn
+                                                 (cond-> {:export-type (:export-type options)}
+                                                   (= :graph (:export-type options))
+                                                   (assoc :graph-options (dissoc options :file :export-type :graph))))
+          file (or (:file options) (str graph "_" (quot (common-util/time-ms) 1000) ".edn"))]
+      (println "Exported" (count (:properties export-map)) "properties,"
+               (count (:properties export-map)) "classes and"
+               (count (:pages-and-blocks export-map)) "pages to" file)
+      (fs/writeFileSync file
+                        (with-out-str (pprint/pprint export-map))))
     (cli-util/error "Graph" (pr-str graph) "does not exist")))

deps/cli/src/logseq/cli/commands/query.cljs

@@ -99,8 +99,8 @@
         (cli-util/error "Graph" (pr-str graph') "does not exist")))))
 
 (defn query
-  [{{:keys [graph args api-query-token]} :opts :as m}]
-  (if api-query-token
+  [{{:keys [graph args api-server-token]} :opts :as m}]
+  (if api-server-token
     ;; graph can be query since it's not used for api-query
-    (api-query (or graph (first args)) api-query-token)
+    (api-query (or graph (first args)) api-server-token)
     (local-query m)))

deps/cli/src/logseq/cli/commands/search.cljs

@@ -41,8 +41,8 @@
                                 (map highlight-fn)))))))
 
 (defn- api-search
-  [search-term {{:keys [api-query-token raw limit]} :opts}]
-  (-> (p/let [resp (cli-util/api-fetch api-query-token "logseq.app.search" [search-term {:limit limit}])]
+  [search-term {{:keys [api-server-token raw limit]} :opts}]
+  (-> (p/let [resp (cli-util/api-fetch api-server-token "logseq.app.search" [search-term {:limit limit}])]
         (if (= 200 (.-status resp))
           (p/let [body (.json resp)]
             (let [{:keys [blocks]} (js->clj body :keywordize-keys true)]
@@ -61,7 +61,7 @@
       (format-results nodes search-term {:raw raw}))
     (cli-util/error "Graph" (pr-str graph) "does not exist")))
 
-(defn search [{{:keys [graph search-terms api-query-token]} :opts :as m}]
-  (if api-query-token
+(defn search [{{:keys [graph search-terms api-server-token]} :opts :as m}]
+  (if api-server-token
     (api-search (string/join " " (into [graph] search-terms)) m)
     (local-search (string/join " " search-terms) m)))

deps/cli/src/logseq/cli/common/export/common.cljs

@@ -0,0 +1,808 @@
+(ns logseq.cli.common.export.common
+  "common fns for exporting.
+  exclude some fns which produce lazy-seq, which can cause strange behaviors
+  when use together with dynamic var."
+  (:refer-clojure :exclude [map filter])
+  (:require [cljs.core.match :refer [match]]
+            [clojure.string :as string]
+            [datascript.core :as d]
+            [logseq.cli.common.file :as common-file]
+            [logseq.cli.common.util :as cli-common-util :refer-macros [removev concatv mapcatv]]
+            [logseq.common.util :as common-util]
+            [logseq.db :as ldb]
+            [logseq.graph-parser.mldoc :as gp-mldoc]
+            [logseq.outliner.tree :as otree]
+            [malli.core :as m]
+            [malli.util :as mu]))
+
+;;; TODO: split frontend.handler.export.text related states
+(def ^:dynamic *state*
+  "dynamic var, state used for exporting"
+  {;; current level of Heading, start from 1(same as mldoc), use when `block-ast->simple-ast`
+   :current-level 1
+   ;; emphasis symbol (use when `block-ast->simple-ast`)
+   :outside-em-symbol nil
+   ;; (use when `block-ast->simple-ast`)
+   :indent-after-break-line? false
+   ;; TODO: :last-empty-heading? false
+   ;;       current:  |  want:
+   ;;       -         |  - xxx
+   ;;         xxx     |    yyy
+   ;;         yyy     |
+
+   ;; this submap is used when replace block-reference, block-embed, page-embed
+   :replace-ref-embed
+   {;; start from 1
+    :current-level 1
+    :block-ref-replaced? false
+    :block&page-embed-replaced? false}
+
+   ;; submap for :newline-after-block internal state
+   :newline-after-block
+   {:current-block-is-first-heading-block? true}
+
+   ;; export-options submap
+   :export-options
+   {;; dashes, spaces, no-indent
+    :indent-style "dashes"
+    :remove-page-ref-brackets? false
+    :remove-emphasis? false
+    :remove-tags? false
+    :remove-properties? true
+    :keep-only-level<=N :all
+    :newline-after-block false}})
+
+;; Global vars that are not explicitly passed in all fns
+;; These vars must be bound in order to use most fns in this namespace
+(def ^:dynamic *current-db* nil)
+(def ^:dynamic *current-repo* nil)
+;; Config used by logseq.cli.common.file fns
+(def ^:dynamic *content-config* nil)
+
+;;; internal utils
+(defn ^:api get-blocks-contents
+  [repo root-block-uuid & {:keys [init-level]
+                           :or {init-level 1}}]
+  (let [block (d/entity *current-db* [:block/uuid root-block-uuid])
+        link (:block/link block)
+        block' (or link block)
+        root-id (:block/uuid block')
+        blocks (ldb/get-block-and-children *current-db* root-id)
+        tree (otree/blocks->vec-tree repo *current-db* blocks root-id {:link link})]
+    (common-file/tree->file-content *current-repo* *current-db* tree
+                                    {:init-level init-level :link link}
+                                    *content-config*)))
+
+(declare remove-block-ast-pos Properties-block-ast?)
+
+(defn- block-uuid->ast
+  [block-uuid]
+  (let [block (into {} (d/entity *current-db* [:block/uuid block-uuid]))
+        content (common-file/tree->file-content *current-repo* *current-db* [block] {:init-level 1} *content-config*)
+        format :markdown]
+    (when content
+      (removev Properties-block-ast?
+               (mapv remove-block-ast-pos
+                     (gp-mldoc/->edn *current-repo* content format))))))
+
+(defn- block-uuid->ast-with-children
+  [block-uuid]
+  (let [content (get-blocks-contents *current-repo* block-uuid)
+        format :markdown]
+    (when content
+      (removev Properties-block-ast?
+               (mapv remove-block-ast-pos
+                     (gp-mldoc/->edn *current-repo* content format))))))
+
+(defn ^:api get-page-content
+  [page-uuid]
+  (common-file/block->content *current-repo* *current-db* page-uuid nil *content-config*))
+
+(defn- page-name->ast
+  [page-name]
+  (let [page (ldb/get-page *current-db* page-name)]
+    (when-let [content (get-page-content (:block/uuid page))]
+      (when content
+        (let [format :markdown]
+          (removev Properties-block-ast?
+                   (mapv remove-block-ast-pos
+                         (gp-mldoc/->edn *current-repo* content format))))))))
+
+(defn- update-level-in-block-ast-coll
+  [block-ast-coll origin-level]
+  (mapv
+   (fn [block-ast]
+     (let [[ast-type ast-content] block-ast]
+       (if (= ast-type "Heading")
+         [ast-type (update ast-content :level #(+ (dec %) origin-level))]
+         block-ast)))
+   block-ast-coll))
+
+(defn- plain-indent-inline-ast
+  [level & {:keys [spaces] :or {spaces "  "}}]
+  ["Plain" (str (reduce str (repeat (dec level) "\t")) spaces)])
+
+(defn- mk-paragraph-ast
+  [inline-coll meta]
+  (with-meta ["Paragraph" inline-coll] meta))
+
+;;; internal utils (ends)
+
+;;; utils
+
+(defn priority->string
+  [priority]
+  (str "[#" priority "]"))
+
+(defn- repetition-to-string
+  [[[kind] [duration] n]]
+  (let [kind (case kind
+               "Dotted" "."
+               "Plus" "+"
+               "DoublePlus" "++")]
+    (str kind n (string/lower-case (str (first duration))))))
+
+(defn timestamp-to-string
+  [{:keys [date time repetition wday active]}]
+  (let [{:keys [year month day]} date
+        {:keys [hour min]} time
+        [open close] (if active ["<" ">"] ["[" "]"])
+        repetition (if repetition
+                     (str " " (repetition-to-string repetition))
+                     "")
+        hour (when hour (common-util/zero-pad hour))
+        min  (when min (common-util/zero-pad min))
+        time (cond
+               (and hour min)
+               (common-util/format " %s:%s" hour min)
+               hour
+               (common-util/format " %s" hour)
+               :else
+               "")]
+    (common-util/format "%s%s-%s-%s %s%s%s%s"
+                        open
+                        (str year)
+                        (common-util/zero-pad month)
+                        (common-util/zero-pad day)
+                        wday
+                        time
+                        repetition
+                        close)))
+
+(defn hashtag-value->string
+  [inline-coll]
+  (reduce str
+          (mapv
+           (fn [inline]
+             (let [[ast-type ast-content] inline]
+               (case ast-type
+                 "Nested_link"
+                 (:content ast-content)
+                 "Link"
+                 (:full_text ast-content)
+                 "Plain"
+                 ast-content)))
+           inline-coll)))
+
+;;; utils (ends)
+
+;;; replace block-ref, block-embed, page-embed
+
+(defn- replace-block-reference-in-heading
+  [{:keys [title] :as ast-content}]
+  (let [inline-coll  title
+        inline-coll*
+        (mapcatv
+         #(match [%]
+            [["Link" {:url ["Block_ref" block-uuid]}]]
+            (let [[[_ {title-inline-coll :title}]]
+                  (block-uuid->ast (uuid block-uuid))]
+              (set! *state* (assoc-in *state* [:replace-ref-embed :block-ref-replaced?] true))
+              title-inline-coll)
+
+            :else [%])
+         inline-coll)]
+    (assoc ast-content :title inline-coll*)))
+
+(defn- replace-block-reference-in-paragraph
+  [inline-coll]
+  (mapcatv
+   #(match [%]
+      [["Link" {:url ["Block_ref" block-uuid]}]]
+      (let [[[_ {title-inline-coll :title}]]
+            (block-uuid->ast (uuid block-uuid))]
+        (set! *state* (assoc-in *state* [:replace-ref-embed :block-ref-replaced?] true))
+        title-inline-coll)
+      :else [%])
+   inline-coll))
+
+(declare replace-block-references)
+
+(defn- replace-block-reference-in-list
+  [list-items]
+  (mapv
+   (fn [{block-ast-coll :content sub-items :items :as item}]
+     (assoc item
+            :content (mapv replace-block-references block-ast-coll)
+            :items (replace-block-reference-in-list sub-items)))
+   list-items))
+
+(defn- replace-block-reference-in-quote
+  [block-ast-coll]
+  (mapv replace-block-references block-ast-coll))
+
+(defn- replace-block-reference-in-table
+  [{:keys [header groups] :as table}]
+  (let [header*
+        (mapv
+         (fn [col]
+           (mapcatv
+            #(match [%]
+               [["Link" {:url ["Block_ref" block-uuid]}]]
+               (let [[[_ {title-inline-coll :title}]]
+                     (block-uuid->ast (uuid block-uuid))]
+                 (set! *state* (assoc-in *state* [:replace-ref-embed :block-ref-replaced?] true))
+                 title-inline-coll)
+               :else [%])
+            col))
+         header)
+        groups*
+        (mapv
+         (fn [group]
+           (mapv
+            (fn [row]
+              (mapv
+               (fn [col]
+                 (mapcatv
+                  #(match [%]
+                     [["Link" {:url ["Block_ref" block-uuid]}]]
+                     (let [[[_ {title-inline-coll :title}]]
+                           (block-uuid->ast (uuid block-uuid))]
+                       (set! *state* (assoc-in *state* [:replace-ref-embed :block-ref-replaced?] true))
+                       title-inline-coll)
+                     :else [%])
+                  col))
+               row))
+            group))
+         groups)]
+    (assoc table :header header* :groups groups*)))
+
+(defn- replace-block-references
+  [block-ast]
+  (let [[ast-type ast-content] block-ast]
+    (case ast-type
+      "Heading"
+      [ast-type (replace-block-reference-in-heading ast-content)]
+
+      "Paragraph"
+      (mk-paragraph-ast (replace-block-reference-in-paragraph ast-content) (meta block-ast))
+
+      "List"
+      [ast-type (replace-block-reference-in-list ast-content)]
+
+      "Quote"
+      [ast-type (replace-block-reference-in-quote ast-content)]
+
+      "Table"
+      [ast-type (replace-block-reference-in-table ast-content)]
+      ;; else
+      block-ast)))
+
+(defn- replace-block-references-until-stable
+  [block-ast]
+  (binding [*state* *state*]
+    (loop [block-ast block-ast]
+      (let [block-ast* (replace-block-references block-ast)]
+        (if (get-in *state* [:replace-ref-embed :block-ref-replaced?])
+          (do (set! *state* (assoc-in *state* [:replace-ref-embed :block-ref-replaced?] false))
+              (recur block-ast*))
+          block-ast*)))))
+
+(defn- replace-block-embeds-helper
+  [current-paragraph-inlines block-uuid blocks-tcoll level]
+  (let [block-uuid* (subs block-uuid 2 (- (count block-uuid) 2))
+        ast-coll (update-level-in-block-ast-coll
+                  (block-uuid->ast-with-children (uuid block-uuid*))
+                  level)]
+    (cond-> blocks-tcoll
+      (seq current-paragraph-inlines)
+      (conj! ["Paragraph" current-paragraph-inlines])
+      true
+      (#(reduce conj! % ast-coll)))))
+
+(defn- replace-page-embeds-helper
+  [current-paragraph-inlines page-name blocks-tcoll level]
+  (let [page-name* (subs page-name 2 (- (count page-name) 2))
+        ast-coll (update-level-in-block-ast-coll
+                  (page-name->ast page-name*)
+                  level)]
+    (cond-> blocks-tcoll
+      (seq current-paragraph-inlines)
+      (conj! ["Paragraph" current-paragraph-inlines])
+      true
+      (#(reduce conj! % ast-coll)))))
+
+(defn- replace-block&page-embeds-in-heading
+  [{inline-coll :title origin-level :level :as ast-content}]
+  (set! *state* (assoc-in *state* [:replace-ref-embed :current-level] origin-level))
+  (if (empty? inline-coll)
+    ;; it's just a empty Heading, return itself
+    [["Heading" ast-content]]
+    (loop [[inline & other-inlines] inline-coll
+           heading-exist? false
+           current-paragraph-inlines []
+           r (transient [])]
+      (if-not inline
+        (persistent!
+         (if (seq current-paragraph-inlines)
+           (conj! r (if heading-exist?
+                      ["Paragraph" current-paragraph-inlines]
+                      ["Heading" (assoc ast-content :title current-paragraph-inlines)]))
+           r))
+        (match [inline]
+          [["Macro" {:name "embed" :arguments [block-uuid-or-page-name]}]]
+          (cond
+            (and (string/starts-with? block-uuid-or-page-name "((")
+                 (string/ends-with? block-uuid-or-page-name "))"))
+            (do (set! *state* (assoc-in *state* [:replace-ref-embed :block&page-embed-replaced?] true))
+                (recur other-inlines true []
+                       (replace-block-embeds-helper
+                        current-paragraph-inlines block-uuid-or-page-name r origin-level)))
+            (and (string/starts-with? block-uuid-or-page-name "[[")
+                 (string/ends-with? block-uuid-or-page-name "]]"))
+            (do (set! *state* (assoc-in *state* [:replace-ref-embed :block&page-embed-replaced?] true))
+                (recur other-inlines true []
+                       (replace-page-embeds-helper
+                        current-paragraph-inlines block-uuid-or-page-name r origin-level)))
+            :else ;; not ((block-uuid)) or [[page-name]], just drop the original ast
+            (recur other-inlines heading-exist? current-paragraph-inlines r))
+
+          :else
+          (let [current-paragraph-inlines*
+                (if (and (empty? current-paragraph-inlines)
+                         heading-exist?)
+                  (conj current-paragraph-inlines (plain-indent-inline-ast origin-level))
+                  current-paragraph-inlines)]
+            (recur other-inlines heading-exist? (conj current-paragraph-inlines* inline) r)))))))
+
+(defn- replace-block&page-embeds-in-paragraph
+  [inline-coll meta]
+  (let [current-level (get-in *state* [:replace-ref-embed :current-level])]
+    (loop [[inline & other-inlines] inline-coll
+           current-paragraph-inlines []
+           just-after-embed? false
+           blocks (transient [])]
+      (if-not inline
+        (let [[first-block & other-blocks] (persistent!
+                                            (if (seq current-paragraph-inlines)
+                                              (conj! blocks ["Paragraph" current-paragraph-inlines])
+                                              blocks))]
+          (if first-block
+            (apply vector (with-meta first-block meta) other-blocks)
+            []))
+        (match [inline]
+          [["Macro" {:name "embed" :arguments [block-uuid-or-page-name]}]]
+          (cond
+            (and (string/starts-with? block-uuid-or-page-name "((")
+                 (string/ends-with? block-uuid-or-page-name "))"))
+            (do (set! *state* (assoc-in *state* [:replace-ref-embed :block&page-embed-replaced?] true))
+                (recur other-inlines [] true
+                       (replace-block-embeds-helper
+                        current-paragraph-inlines block-uuid-or-page-name blocks current-level)))
+            (and (string/starts-with? block-uuid-or-page-name "[[")
+                 (string/ends-with? block-uuid-or-page-name "]]"))
+            (do (set! *state* (assoc-in *state* [:replace-ref-embed :block&page-embed-replaced?] true))
+                (recur other-inlines [] true
+                       (replace-page-embeds-helper
+                        current-paragraph-inlines block-uuid-or-page-name blocks current-level)))
+            :else ;; not ((block-uuid)) or [[page-name]], just drop the original ast
+            (recur other-inlines current-paragraph-inlines false blocks))
+
+          :else
+          (let [current-paragraph-inlines*
+                (if just-after-embed?
+                  (conj current-paragraph-inlines (plain-indent-inline-ast current-level))
+                  current-paragraph-inlines)]
+            (recur other-inlines (conj current-paragraph-inlines* inline) false blocks)))))))
+
+(declare replace-block&page-embeds)
+
+(defn- replace-block&page-embeds-in-list-helper
+  [list-items]
+  (binding [*state* (update-in *state* [:replace-ref-embed :current-level] inc)]
+    (mapv
+     (fn [{block-ast-coll :content sub-items :items :as item}]
+       (assoc item
+              :content (mapcatv replace-block&page-embeds block-ast-coll)
+              :items (replace-block&page-embeds-in-list-helper sub-items)))
+     list-items)))
+
+(defn- replace-block&page-embeds-in-list
+  [list-items]
+  [["List" (replace-block&page-embeds-in-list-helper list-items)]])
+
+(defn- replace-block&page-embeds-in-quote
+  [block-ast-coll]
+  (->> block-ast-coll
+       (mapcatv replace-block&page-embeds)
+       (vector "Quote")
+       vector))
+
+(defn- replace-block&page-embeds
+  [block-ast]
+  (let [[ast-type ast-content] block-ast]
+    (case ast-type
+      "Heading"
+      (replace-block&page-embeds-in-heading ast-content)
+      "Paragraph"
+      (replace-block&page-embeds-in-paragraph ast-content (meta block-ast))
+      "List"
+      (replace-block&page-embeds-in-list ast-content)
+      "Quote"
+      (replace-block&page-embeds-in-quote ast-content)
+      "Table"
+      ;; TODO: block&page embeds in table are not replaced yet
+      [block-ast]
+      ;; else
+      [block-ast])))
+
+(defn replace-block&page-reference&embed
+  "add meta :embed-depth to the embed replaced block-ast,
+  to avoid too deep block-ref&embed (or maybe it's a cycle)"
+  [block-ast-coll]
+  (loop [block-ast-coll block-ast-coll
+         result-block-ast-tcoll (transient [])
+         block-ast-coll-to-replace-references []
+         block-ast-coll-to-replace-embeds []]
+    (cond
+      (seq block-ast-coll-to-replace-references)
+      (let [[block-ast-to-replace-ref & other-block-asts-to-replace-ref]
+            block-ast-coll-to-replace-references
+            embed-depth (:embed-depth (meta block-ast-to-replace-ref) 0)
+            block-ast-replaced (-> (replace-block-references-until-stable block-ast-to-replace-ref)
+                                   (with-meta {:embed-depth embed-depth}))]
+        (if (>= embed-depth 5)
+          ;; if :embed-depth >= 5, dont replace embed for this block anymore
+          ;; there is too deep, or maybe it just a ref/embed cycle
+          (recur block-ast-coll (conj! result-block-ast-tcoll block-ast-replaced)
+                 (vec other-block-asts-to-replace-ref) block-ast-coll-to-replace-embeds)
+          (recur block-ast-coll result-block-ast-tcoll (vec other-block-asts-to-replace-ref)
+                 (conj block-ast-coll-to-replace-embeds block-ast-replaced))))
+
+      (seq block-ast-coll-to-replace-embeds)
+      (let [[block-ast-to-replace-embed & other-block-asts-to-replace-embed]
+            block-ast-coll-to-replace-embeds
+            embed-depth (:embed-depth (meta block-ast-to-replace-embed) 0)
+            block-ast-coll-replaced (->> (replace-block&page-embeds block-ast-to-replace-embed)
+                                         (mapv #(with-meta % {:embed-depth (inc embed-depth)})))]
+        (if (get-in *state* [:replace-ref-embed :block&page-embed-replaced?])
+          (do (set! *state* (assoc-in *state* [:replace-ref-embed :block&page-embed-replaced?] false))
+              (recur block-ast-coll result-block-ast-tcoll
+                     (concatv block-ast-coll-to-replace-references block-ast-coll-replaced)
+                     (vec other-block-asts-to-replace-embed)))
+          (recur block-ast-coll (reduce conj! result-block-ast-tcoll block-ast-coll-replaced)
+                 (vec block-ast-coll-to-replace-references) (vec other-block-asts-to-replace-embed))))
+
+      :else
+      (let [[block-ast & other-block-ast] block-ast-coll]
+        (if-not block-ast
+          (persistent! result-block-ast-tcoll)
+          (recur other-block-ast result-block-ast-tcoll
+                 (conj block-ast-coll-to-replace-references block-ast)
+                 (vec block-ast-coll-to-replace-embeds)))))))
+
+;;; replace block-ref, block-embed, page-embed (ends)
+
+(def remove-block-ast-pos
+  "[[ast-type ast-content] _pos] -> [ast-type ast-content]"
+  first)
+
+(defn Properties-block-ast?
+  [[tp _]]
+  (= tp "Properties"))
+
+(defn replace-Heading-with-Paragraph
+  "works on block-ast
+  replace all heading with paragraph when indent-style is no-indent"
+  [heading-ast]
+  (let [[heading-type {:keys [title marker priority size]}] heading-ast]
+    (if (= heading-type "Heading")
+      (let [inline-coll
+            (cond->> title
+              priority (cons ["Plain" (str (priority->string priority) " ")])
+              marker (cons ["Plain" (str marker " ")])
+              size (cons ["Plain" (str (reduce str (repeat size "#")) " ")])
+              true vec)]
+        (mk-paragraph-ast inline-coll {:origin-ast heading-ast}))
+      heading-ast)))
+
+(defn keep-only-level<=n
+  [block-ast-coll n]
+  (-> (reduce
+       (fn [{:keys [result-ast-tcoll accepted-heading] :as r} ast]
+         (let [[heading-type {level :level}] ast
+               is-heading?                   (= heading-type "Heading")]
+           (cond
+             (and (not is-heading?) accepted-heading)
+             {:result-ast-tcoll (conj! result-ast-tcoll ast) :accepted-heading accepted-heading}
+
+             (and (not is-heading?) (not accepted-heading))
+             r
+
+             (and is-heading? (<= level n))
+             {:result-ast-tcoll (conj! result-ast-tcoll ast) :accepted-heading true}
+
+             (and is-heading? (> level n))
+             {:result-ast-tcoll result-ast-tcoll :accepted-heading false})))
+       {:result-ast-tcoll  (transient []) :accepted-heading false}
+       block-ast-coll)
+      :result-ast-tcoll
+      persistent!))
+
+;;; inline transformers
+
+(defn remove-emphasis
+  ":mapcat-fns-on-inline-ast"
+  [inline-ast]
+  (let [[ast-type ast-content] inline-ast]
+    (case ast-type
+      "Emphasis"
+      (let [[_ inline-coll] ast-content]
+        inline-coll)
+      ;; else
+      [inline-ast])))
+
+(defn remove-page-ref-brackets
+  ":map-fns-on-inline-ast"
+  [inline-ast]
+  (let [[ast-type ast-content] inline-ast]
+    (case ast-type
+      "Link"
+      (let [{:keys [url label]} ast-content]
+        (if (and (= "Page_ref" (first url))
+                 (or (empty? label)
+                     (= label [["Plain" ""]])))
+          ["Plain" (second url)]
+          inline-ast))
+      ;; else
+      inline-ast)))
+
+(defn remove-tags
+  ":mapcat-fns-on-inline-ast"
+  [inline-ast]
+  (let [[ast-type _ast-content] inline-ast]
+    (case ast-type
+      "Tag"
+      []
+      ;; else
+      [inline-ast])))
+
+(defn remove-prefix-spaces-in-Plain
+  [inline-coll]
+  (:r
+   (reduce
+    (fn [{:keys [r after-break-line?]} ast]
+      (let [[ast-type ast-content] ast]
+        (case ast-type
+          "Plain"
+          (let [trimmed-content (string/triml ast-content)]
+            (if after-break-line?
+              (if (empty? trimmed-content)
+                {:r r :after-break-line? false}
+                {:r (conj r ["Plain" trimmed-content]) :after-break-line? false})
+              {:r (conj r ast) :after-break-line? false}))
+          ("Break_Line" "Hard_Break_Line")
+          {:r (conj r ast) :after-break-line? true}
+        ;; else
+          {:r (conj r ast) :after-break-line? false})))
+    {:r [] :after-break-line? true}
+    inline-coll)))
+
+;;; inline transformers (ends)
+
+;;; walk on block-ast, apply inline transformers
+
+(defn- walk-block-ast-helper
+  [inline-coll map-fns-on-inline-ast mapcat-fns-on-inline-ast fns-on-inline-coll]
+  (->>
+   (reduce (fn [inline-coll f] (f inline-coll)) inline-coll fns-on-inline-coll)
+   (mapv #(reduce (fn [inline-ast f] (f inline-ast)) % map-fns-on-inline-ast))
+   (mapcatv #(reduce
+              (fn [inline-ast-coll f] (mapcatv f inline-ast-coll)) [%] mapcat-fns-on-inline-ast))))
+
+(declare walk-block-ast)
+
+(defn- walk-block-ast-for-list
+  [list-items map-fns-on-inline-ast mapcat-fns-on-inline-ast]
+  (mapv
+   (fn [{block-ast-coll :content sub-items :items :as item}]
+     (assoc item
+            :content
+            (mapv
+             (partial walk-block-ast
+                      {:map-fns-on-inline-ast map-fns-on-inline-ast
+                       :mapcat-fns-on-inline-ast mapcat-fns-on-inline-ast})
+             block-ast-coll)
+            :items
+            (walk-block-ast-for-list sub-items map-fns-on-inline-ast mapcat-fns-on-inline-ast)))
+   list-items))
+
+(defn walk-block-ast
+  [{:keys [map-fns-on-inline-ast mapcat-fns-on-inline-ast fns-on-inline-coll] :as fns}
+   block-ast]
+  (let [[ast-type ast-content] block-ast]
+    (case ast-type
+      "Paragraph"
+      (mk-paragraph-ast
+       (walk-block-ast-helper ast-content map-fns-on-inline-ast mapcat-fns-on-inline-ast fns-on-inline-coll)
+       (meta block-ast))
+      "Heading"
+      (let [{:keys [title]} ast-content]
+        ["Heading"
+         (assoc ast-content
+                :title
+                (walk-block-ast-helper title map-fns-on-inline-ast mapcat-fns-on-inline-ast fns-on-inline-coll))])
+      "List"
+      ["List" (walk-block-ast-for-list ast-content map-fns-on-inline-ast mapcat-fns-on-inline-ast)]
+      "Quote"
+      ["Quote" (mapv (partial walk-block-ast fns) ast-content)]
+      "Footnote_Definition"
+      (let [[name contents] (rest block-ast)]
+        ["Footnote_Definition"
+         name (walk-block-ast-helper contents map-fns-on-inline-ast mapcat-fns-on-inline-ast fns-on-inline-coll)])
+      "Table"
+      (let [{:keys [header groups]} ast-content
+            header* (mapv
+                     #(walk-block-ast-helper % map-fns-on-inline-ast mapcat-fns-on-inline-ast fns-on-inline-coll)
+                     header)
+            groups* (mapv
+                     (fn [group]
+                       (mapv
+                        (fn [row]
+                          (mapv
+                           (fn [col]
+                             (walk-block-ast-helper col map-fns-on-inline-ast mapcat-fns-on-inline-ast fns-on-inline-coll))
+                           row))
+                        group))
+                     groups)]
+        ["Table" (assoc ast-content :header header* :groups groups*)])
+
+       ;; else
+      block-ast)))
+
+;;; walk on block-ast, apply inline transformers (ends)
+
+;;; simple ast
+(def simple-ast-malli-schema
+  (mu/closed-schema
+   [:or
+    [:map
+     [:type [:= :raw-text]]
+     [:content :string]]
+    [:map
+     [:type [:= :space]]]
+    [:map
+     [:type [:= :newline]]
+     [:line-count :int]]
+    [:map
+     [:type [:= :indent]]
+     [:level :int]
+     [:extra-space-count :int]]]))
+
+(defn raw-text [& contents]
+  {:type :raw-text :content (reduce str contents)})
+(def space {:type :space})
+(defn newline* [line-count]
+  {:type :newline :line-count line-count})
+(defn indent [level extra-space-count]
+  {:type :indent :level level :extra-space-count extra-space-count})
+
+(defn- simple-ast->string
+  [simple-ast]
+  {:pre [(m/validate simple-ast-malli-schema simple-ast)]}
+  (case (:type simple-ast)
+    :raw-text (:content simple-ast)
+    :space " "
+    :newline (reduce str (repeat (:line-count simple-ast) "\n"))
+    :indent (reduce str (concatv (repeat (:level simple-ast) "\t")
+                                 (repeat (:extra-space-count simple-ast) " ")))))
+
+(defn- ^:large-vars/cleanup-todo merge-adjacent-spaces&newlines
+  [simple-ast-coll]
+  (loop [r                             (transient [])
+         last-ast                      nil
+         last-raw-text-space-suffix?   false
+         last-raw-text-newline-suffix? false
+         [simple-ast & other-ast-coll] simple-ast-coll]
+    (if (nil? simple-ast)
+      (persistent! (if last-ast (conj! r last-ast) r))
+      (let [tp            (:type simple-ast)
+            last-ast-type (:type last-ast)]
+        (case tp
+          :space
+          (if (or (contains? #{:space :newline :indent} last-ast-type)
+                  last-raw-text-space-suffix?
+                  last-raw-text-newline-suffix?)
+            ;; drop this :space
+            (recur r last-ast last-raw-text-space-suffix? last-raw-text-newline-suffix? other-ast-coll)
+            (recur (if last-ast (conj! r last-ast) r) simple-ast false false other-ast-coll))
+
+          :newline
+          (case last-ast-type
+            (:space :indent) ;; drop last-ast
+            (recur r simple-ast false false other-ast-coll)
+            :newline
+            (let [last-newline-count (:line-count last-ast)
+                  current-newline-count (:line-count simple-ast)
+                  kept-ast (if (> last-newline-count current-newline-count) last-ast simple-ast)]
+              (recur r kept-ast false false other-ast-coll))
+            :raw-text
+            (if last-raw-text-newline-suffix?
+              (recur r last-ast last-raw-text-space-suffix? last-raw-text-newline-suffix? other-ast-coll)
+              (recur (if last-ast (conj! r last-ast) r) simple-ast false false other-ast-coll))
+            ;; no-last-ast
+            (recur r simple-ast false false other-ast-coll))
+
+          :indent
+          (case last-ast-type
+            (:space :indent)            ; drop last-ast
+            (recur r simple-ast false false other-ast-coll)
+            :newline
+            (recur (if last-ast (conj! r last-ast) r) simple-ast false false other-ast-coll)
+            :raw-text
+            (if last-raw-text-space-suffix?
+              ;; drop this :indent
+              (recur r last-ast last-raw-text-space-suffix? last-raw-text-newline-suffix? other-ast-coll)
+              (recur (if last-ast (conj! r last-ast) r) simple-ast false false other-ast-coll))
+            ;; no-last-ast
+            (recur r simple-ast false false other-ast-coll))
+
+          :raw-text
+          (let [content         (:content simple-ast)
+                empty-content?  (empty? content)
+                first-ch        (first content)
+                last-ch         (let [num (count content)]
+                                  (when (pos? num)
+                                    (nth content (dec num))))
+                newline-prefix? (some-> first-ch #{"\r" "\n"} boolean)
+                newline-suffix? (some-> last-ch #{"\n"} boolean)
+                space-prefix?   (some-> first-ch #{" "} boolean)
+                space-suffix?   (some-> last-ch #{" "} boolean)]
+            (cond
+              empty-content?            ;drop this raw-text
+              (recur r last-ast last-raw-text-space-suffix? last-raw-text-newline-suffix? other-ast-coll)
+              newline-prefix?
+              (case last-ast-type
+                (:space :indent :newline) ;drop last-ast
+                (recur r simple-ast space-suffix? newline-suffix? other-ast-coll)
+                :raw-text
+                (recur (if last-ast (conj! r last-ast) r) simple-ast space-suffix? newline-suffix? other-ast-coll)
+                ;; no-last-ast
+                (recur r simple-ast space-suffix? newline-suffix? other-ast-coll))
+              space-prefix?
+              (case last-ast-type
+                (:space :indent)        ;drop last-ast
+                (recur r simple-ast space-suffix? newline-suffix? other-ast-coll)
+                (:newline :raw-text)
+                (recur (if last-ast (conj! r last-ast) r) simple-ast space-suffix? newline-suffix? other-ast-coll)
+                ;; no-last-ast
+                (recur r simple-ast space-suffix? newline-suffix? other-ast-coll))
+              :else
+              (recur (if last-ast (conj! r last-ast) r) simple-ast space-suffix? newline-suffix? other-ast-coll))))))))
+
+(defn simple-asts->string
+  [simple-ast-coll]
+  (->> simple-ast-coll
+       merge-adjacent-spaces&newlines
+       merge-adjacent-spaces&newlines
+       (mapv simple-ast->string)
+       string/join))
+
+;;; simple ast (ends)
+
+;;; TODO: walk the hiccup tree,
+;;; and call escape-html on all its contents
+;;;
+
+;;; walk the hiccup tree,
+;;; and call escape-html on all its contents (ends)

deps/cli/src/logseq/cli/common/export/text.cljs

@@ -0,0 +1,493 @@
+(ns logseq.cli.common.export.text
+  "Common fns between frontend and CLI for exporting as markdown"
+  (:require [clojure.string :as string]
+            [logseq.cli.common.export.common :as cli-export-common :refer
+             [*state* newline* indent raw-text space simple-asts->string]]
+            [logseq.cli.common.util :refer-macros [removev concatv mapcatv]]
+            [logseq.graph-parser.mldoc :as gp-mldoc]
+            [logseq.graph-parser.schema.mldoc :as mldoc-schema]))
+
+;;; block-ast, inline-ast -> simple-ast
+
+(defn ^:api indent-with-2-spaces
+  "also consider (get-in *state* [:export-options :indent-style])"
+  [level]
+  (let [indent-style (get-in *state* [:export-options :indent-style])]
+    (case indent-style
+      "dashes"               (indent level 2)
+      ("spaces" "no-indent") (indent level 0)
+      (assert false (print-str "unknown indent-style:" indent-style)))))
+
+(declare inline-ast->simple-ast
+         block-ast->simple-ast)
+
+(defn- block-heading
+  [{:keys [title _tags marker level _numbering priority _anchor _meta _unordered size]}]
+  (let [indent-style (get-in *state* [:export-options :indent-style])
+        priority* (and priority (raw-text (cli-export-common/priority->string priority)))
+        heading* (if (= indent-style "dashes")
+                   [(indent (dec level) 0) (raw-text "-")]
+                   [(indent (dec level) 0)])
+        size* (and size [space (raw-text (reduce str (repeat size "#")))])
+        marker* (and marker (raw-text marker))]
+    (set! *state* (assoc *state* :current-level level))
+    (let [simple-asts
+          (removev nil? (concatv
+                         (when (and (get-in *state* [:export-options :newline-after-block])
+                                    (not (get-in *state* [:newline-after-block :current-block-is-first-heading-block?])))
+                           [(newline* 2)])
+                         heading* size*
+                         [space marker* space priority* space]
+                         (mapcatv inline-ast->simple-ast title)
+                         [(newline* 1)]))]
+      (set! *state* (assoc-in *state* [:newline-after-block :current-block-is-first-heading-block?] false))
+      simple-asts)))
+
+(declare block-list)
+(defn- block-list-item
+  [{:keys [content items number _name checkbox]}]
+  (let [content* (mapcatv block-ast->simple-ast content)
+        number* (raw-text
+                 (if number
+                   (str number ". ")
+                   "* "))
+        checkbox* (raw-text
+                   (if (some? checkbox)
+                     (if (boolean checkbox)
+                       "[X]" "[ ]")
+                     ""))
+        current-level (get *state* :current-level 1)
+        indent' (when (> current-level 1)
+                  (indent (dec current-level) 0))
+        items* (block-list items :in-list? true)]
+    (concatv [indent' number* checkbox* space]
+             content*
+             [(newline* 1)]
+             items*
+             [(newline* 1)])))
+
+(defn- block-list
+  [l & {:keys [in-list?]}]
+  (binding [*state* (update *state* :current-level inc)]
+    (concatv (mapcatv block-list-item l)
+             (when (and (pos? (count l))
+                        (not in-list?))
+               [(newline* 2)]))))
+
+(defn- block-property-drawer
+  [properties]
+  (when-not (get-in *state* [:export-options :remove-properties?])
+    (let [level (dec (get *state* :current-level 1))
+          indent' (indent-with-2-spaces level)]
+      (reduce
+       (fn [r [k v]]
+         (conj r indent' (raw-text k "::") space (raw-text v) (newline* 1)))
+       [] properties))))
+
+(defn- block-example
+  [l]
+  (let [level (dec (get *state* :current-level 1))]
+    (mapcatv
+     (fn [line]
+       [(indent-with-2-spaces level)
+        (raw-text "    ")
+        (raw-text line)
+        (newline* 1)])
+     l)))
+
+(defn- remove-max-prefix-spaces
+  [lines]
+  (let [common-prefix-spaces
+        (reduce
+         (fn [r line]
+           (if (string/blank? line)
+             r
+             (let [leading-spaces (re-find #"^\s+" line)]
+               (if (nil? r)
+                 leading-spaces
+                 (if (string/starts-with? r leading-spaces)
+                   leading-spaces
+                   r)))))
+         nil
+         lines)
+        pattern (re-pattern (str "^" common-prefix-spaces))]
+    (mapv (fn [line] (string/replace-first line pattern "")) lines)))
+
+(defn- block-src
+  [{:keys [lines language]}]
+  (let [level (dec (get *state* :current-level 1))
+        lines* (if (= "no-indent" (get-in *state* [:export-options :indent-style]))
+                 (remove-max-prefix-spaces lines)
+                 lines)]
+    (concatv
+     [(indent-with-2-spaces level) (raw-text "```")]
+     (when language [(raw-text language)])
+     [(newline* 1)]
+     (mapv raw-text lines*)
+     [(indent-with-2-spaces level) (raw-text "```") (newline* 1)])))
+
+(defn- block-quote
+  [block-coll]
+  (let [level (dec (get *state* :current-level 1))]
+    (binding [*state* (assoc *state* :indent-after-break-line? true)]
+      (concatv (mapcatv (fn [block]
+                          (let [block-simple-ast (block-ast->simple-ast block)]
+                            (when (seq block-simple-ast)
+                              (concatv [(indent-with-2-spaces level) (raw-text ">") space]
+                                       block-simple-ast))))
+                        block-coll)
+               [(newline* 2)]))))
+
+(declare inline-latex-fragment)
+(defn- block-latex-fragment
+  [ast-content]
+  (inline-latex-fragment ast-content))
+
+(defn- block-latex-env
+  [[name options content]]
+  (let [level (dec (get *state* :current-level 1))]
+    [(indent-with-2-spaces level) (raw-text "\\begin{" name "}" options)
+     (newline* 1)
+     (indent-with-2-spaces level) (raw-text content)
+     (newline* 1)
+     (indent-with-2-spaces level) (raw-text "\\end{" name "}")
+     (newline* 1)]))
+
+(defn- block-displayed-math
+  [ast-content]
+  [space (raw-text "$$" ast-content "$$") space])
+
+(defn- block-drawer
+  [[name lines]]
+  (let [level (dec (get *state* :current-level))]
+    (concatv
+     [(raw-text ":" name ":")
+      (newline* 1)]
+     (mapcatv (fn [line] [(indent-with-2-spaces level) (raw-text line)]) lines)
+     [(newline* 1) (raw-text ":END:") (newline* 1)])))
+
+(defn- block-footnote-definition
+  [[name content]]
+  (concatv
+   [(raw-text "[^" name "]:") space]
+   (mapcatv inline-ast->simple-ast content)
+   [(newline* 1)]))
+
+(def ^:private block-horizontal-rule [(newline* 1) (raw-text "---") (newline* 1)])
+
+(defn- block-table
+  [{:keys [header groups]}]
+  (let [level    (dec (get *state* :current-level 1))
+        sep-line (raw-text "|" (string/join "|" (repeat (count header) "---")) "|")
+        header-line
+        (concatv (mapcatv
+                  (fn [h] (concatv [space (raw-text "|") space] (mapcatv inline-ast->simple-ast h)))
+                  header)
+                 [space (raw-text "|")])
+        group-lines
+        (mapcatv
+         (fn [group]
+           (mapcatv
+            (fn [row]
+              (concatv [(indent-with-2-spaces level)]
+                       (mapcatv
+                        (fn [col]
+                          (concatv [(raw-text "|") space]
+                                   (mapcatv inline-ast->simple-ast col)
+                                   [space]))
+                        row)
+                       [(raw-text "|") (newline* 1)]))
+            group))
+         groups)]
+    (concatv [(newline* 1) (indent-with-2-spaces level)]
+             (when (seq header) header-line)
+             (when (seq header) [(newline* 1) (indent-with-2-spaces level) sep-line (newline* 1)])
+             group-lines)))
+
+(defn- block-comment
+  [s]
+  (let [level (dec (get *state* :current-level 1))]
+    [(indent-with-2-spaces level) (raw-text "<!---") (newline* 1)
+     (indent-with-2-spaces level) (raw-text s) (newline* 1)
+     (indent-with-2-spaces level) (raw-text "-->") (newline* 1)]))
+
+(defn- block-raw-html
+  [s]
+  (let [level (dec (get *state* :current-level 1))]
+    [(indent-with-2-spaces level) (raw-text s) (newline* 1)]))
+
+(defn- block-hiccup
+  [s]
+  (let [level (dec (get *state* :current-level 1))]
+    [(indent-with-2-spaces level) (raw-text s) space]))
+
+(defn- inline-link
+  [{full-text :full_text}]
+  [(raw-text full-text)])
+
+(defn- inline-nested-link
+  [{content :content}]
+  [(raw-text content)])
+
+(defn- inline-subscript
+  [inline-coll]
+  (concatv [(raw-text "_{")]
+           (mapcatv (fn [inline] (cons space (inline-ast->simple-ast inline))) inline-coll)
+           [(raw-text "}")]))
+
+(defn- inline-superscript
+  [inline-coll]
+  (concatv [(raw-text "^{")]
+           (mapcatv (fn [inline] (cons space (inline-ast->simple-ast inline))) inline-coll)
+           [(raw-text "}")]))
+
+(defn- inline-footnote-reference
+  [{name :name}]
+  [(raw-text  "[" name "]")])
+
+(defn- inline-cookie
+  [ast-content]
+  [(raw-text
+    (case (first ast-content)
+      "Absolute"
+      (let [[_ current total] ast-content]
+        (str "[" current "/" total "]"))
+      "Percent"
+      (str "[" (second ast-content) "%]")))])
+
+(defn- inline-latex-fragment
+  [ast-content]
+  (let [[type content] ast-content
+        wrapper (case type
+                  "Inline" "$"
+                  "Displayed" "$$")]
+    [space (raw-text (str wrapper content wrapper)) space]))
+
+(defn- inline-macro
+  [{:keys [name arguments]}]
+  (->
+   (if (= name "cloze")
+     (string/join "," arguments)
+     (let [l (cond-> ["{{" name]
+               (pos? (count arguments)) (conj "(" (string/join "," arguments) ")")
+               true (conj "}}"))]
+       (string/join l)))
+   raw-text
+   vector))
+
+(defn- inline-entity
+  [{unicode :unicode}]
+  [(raw-text unicode)])
+
+(defn- inline-timestamp
+  [ast-content]
+  (let [[type timestamp-content] ast-content]
+    (-> (case type
+          "Scheduled" ["SCHEDULED: " (cli-export-common/timestamp-to-string timestamp-content)]
+          "Deadline" ["DEADLINE: " (cli-export-common/timestamp-to-string timestamp-content)]
+          "Date" [(cli-export-common/timestamp-to-string timestamp-content)]
+          "Closed" ["CLOSED: " (cli-export-common/timestamp-to-string timestamp-content)]
+          "Clock" ["CLOCK: " (cli-export-common/timestamp-to-string (second timestamp-content))]
+          "Range" (let [{:keys [start stop]} timestamp-content]
+                    [(str (cli-export-common/timestamp-to-string start) "--" (cli-export-common/timestamp-to-string stop))]))
+        string/join
+        raw-text
+        vector)))
+
+(defn- inline-email
+  [{:keys [local_part domain]}]
+  [(raw-text (str "<" local_part "@" domain ">"))])
+
+(defn- emphasis-wrap-with
+  [inline-coll em-symbol]
+  (binding [*state* (assoc *state* :outside-em-symbol (first em-symbol))]
+    (concatv [(raw-text em-symbol)]
+             (mapcatv inline-ast->simple-ast inline-coll)
+             [(raw-text em-symbol)])))
+
+(defn- inline-emphasis
+  [emphasis]
+  (let [[[type] inline-coll] emphasis
+        outside-em-symbol (:outside-em-symbol *state*)]
+    (case type
+      "Bold"
+      (emphasis-wrap-with inline-coll (if (= outside-em-symbol "*") "__" "**"))
+      "Italic"
+      (emphasis-wrap-with inline-coll (if (= outside-em-symbol "*") "_" "*"))
+      "Underline"
+      (binding [*state* (assoc *state* :outside-em-symbol outside-em-symbol)]
+        (mapcatv (fn [inline] (cons space (inline-ast->simple-ast inline))) inline-coll))
+      "Strike_through"
+      (emphasis-wrap-with inline-coll "~~")
+      "Highlight"
+      (emphasis-wrap-with inline-coll "^^")
+      ;; else
+      (assert false (print-str :inline-emphasis emphasis "is invalid")))))
+
+(defn- inline-break-line
+  []
+  [(if (= "no-indent" (get-in *state* [:export-options :indent-style]))
+     (raw-text "\n")
+     (raw-text "  \n"))
+   (when (:indent-after-break-line? *state*)
+     (let [current-level (get *state* :current-level 1)]
+       (when (> current-level 1)
+         (indent-with-2-spaces (dec current-level)))))])
+
+;; {:malli/schema ...} only works on public vars so make this public
+(defn ^:large-vars/cleanup-todo ^:api block-ast->simple-ast
+  {:malli/schema [:=> [:cat mldoc-schema/block-ast-schema] [:sequential cli-export-common/simple-ast-malli-schema]]}
+  [block]
+  (let [newline-after-block? (get-in *state* [:export-options :newline-after-block])]
+    (removev
+     nil?
+     (let [[ast-type ast-content] block]
+       (case ast-type
+         "Paragraph"
+         (let [{:keys [origin-ast]} (meta block)
+               current-block-is-first-heading-block? (get-in *state* [:newline-after-block :current-block-is-first-heading-block?])]
+           (set! *state* (assoc-in *state* [:newline-after-block :current-block-is-first-heading-block?] false))
+           (concatv
+            (when (and origin-ast newline-after-block? (not current-block-is-first-heading-block?))
+              [(newline* 2)])
+            (mapcatv inline-ast->simple-ast ast-content)
+            (let [last-element (last ast-content)
+                  [last-element-type] last-element]
+              (when (and newline-after-block? (= "Break_Line" last-element-type))
+                (inline-break-line)))
+            [(newline* 1)]))
+         "Paragraph_line"
+         (assert false "Paragraph_line is mldoc internal ast")
+         "Paragraph_Sep"
+         [(newline* ast-content)]
+         "Heading"
+         (block-heading ast-content)
+         "List"
+         (block-list ast-content)
+         ("Directive" "Results" "Export" "CommentBlock" "Custom")
+         nil
+         "Example"
+         (block-example ast-content)
+         "Src"
+         (block-src ast-content)
+         "Quote"
+         (block-quote ast-content)
+         "Latex_Fragment"
+         (block-latex-fragment ast-content)
+         "Latex_Environment"
+         (block-latex-env (rest block))
+         "Displayed_Math"
+         (block-displayed-math ast-content)
+         "Drawer"
+         (block-drawer (rest block))
+         "Property_Drawer"
+         (block-property-drawer ast-content)
+         "Footnote_Definition"
+         (block-footnote-definition (rest block))
+         "Horizontal_Rule"
+         block-horizontal-rule
+         "Table"
+         (block-table ast-content)
+         "Comment"
+         (block-comment ast-content)
+         "Raw_Html"
+         (block-raw-html ast-content)
+         "Hiccup"
+         (block-hiccup ast-content)
+         (assert false (print-str :block-ast->simple-ast ast-type "not implemented yet")))))))
+
+(defn- ^:large-vars/cleanup-todo inline-ast->simple-ast
+  [inline]
+  (let [[ast-type ast-content] inline]
+    (case ast-type
+      "Emphasis"
+      (inline-emphasis ast-content)
+      ("Break_Line" "Hard_Break_Line")
+      (inline-break-line)
+      "Verbatim"
+      [(raw-text ast-content)]
+      "Code"
+      [(raw-text "`" ast-content "`")]
+      "Tag"
+      [(raw-text (str "#" (cli-export-common/hashtag-value->string ast-content)))]
+      "Spaces"                          ; what's this ast-type for ?
+      nil
+      "Plain"
+      [(raw-text ast-content)]
+      "Link"
+      (inline-link ast-content)
+      "Nested_link"
+      (inline-nested-link ast-content)
+      "Target"
+      [(raw-text (str "<<" ast-content ">>"))]
+      "Subscript"
+      (inline-subscript ast-content)
+      "Superscript"
+      (inline-superscript ast-content)
+      "Footnote_Reference"
+      (inline-footnote-reference ast-content)
+      "Cookie"
+      (inline-cookie ast-content)
+      "Latex_Fragment"
+      (inline-latex-fragment ast-content)
+      "Macro"
+      (inline-macro ast-content)
+      "Entity"
+      (inline-entity ast-content)
+      "Timestamp"
+      (inline-timestamp ast-content)
+      "Radio_Target"
+      [(raw-text (str "<<<" ast-content ">>>"))]
+      "Email"
+      (inline-email ast-content)
+      "Inline_Hiccup"
+      [(raw-text ast-content)]
+      "Inline_Html"
+      [(raw-text ast-content)]
+      ("Export_Snippet" "Inline_Source_Block")
+      nil
+      (assert false (print-str :inline-ast->simple-ast ast-type "not implemented yet")))))
+
+;;; block-ast, inline-ast -> simple-ast (ends)
+
+(defn ^:large-vars/cleanup-todo export-helper
+  [repo content format options]
+  (let [remove-options (set (:remove-options options))
+        other-options (:other-options options)]
+    (binding [*state* (merge *state*
+                             {:export-options
+                              {:indent-style (or (:indent-style options) "dashes")
+                               :remove-emphasis? (contains? remove-options :emphasis)
+                               :remove-page-ref-brackets? (contains? remove-options :page-ref)
+                               :remove-tags? (contains? remove-options :tag)
+                               :remove-properties? (contains? remove-options :property)
+                               :keep-only-level<=N (:keep-only-level<=N other-options)
+                               :newline-after-block (:newline-after-block other-options)}})]
+      (let [ast (gp-mldoc/->edn repo content format)
+            ast (mapv cli-export-common/remove-block-ast-pos ast)
+            ast (removev cli-export-common/Properties-block-ast? ast)
+            ast* (cli-export-common/replace-block&page-reference&embed ast)
+            keep-level<=n (get-in *state* [:export-options :keep-only-level<=N])
+            ast* (if (pos? keep-level<=n)
+                   (cli-export-common/keep-only-level<=n ast* keep-level<=n)
+                   ast*)
+            ast** (if (= "no-indent" (get-in *state* [:export-options :indent-style]))
+                    (mapv cli-export-common/replace-Heading-with-Paragraph ast*)
+                    ast*)
+            config-for-walk-block-ast (cond-> {}
+                                        (get-in *state* [:export-options :remove-emphasis?])
+                                        (update :mapcat-fns-on-inline-ast conj cli-export-common/remove-emphasis)
+
+                                        (get-in *state* [:export-options :remove-page-ref-brackets?])
+                                        (update :map-fns-on-inline-ast conj cli-export-common/remove-page-ref-brackets)
+
+                                        (get-in *state* [:export-options :remove-tags?])
+                                        (update :mapcat-fns-on-inline-ast conj cli-export-common/remove-tags)
+
+                                        (= "no-indent" (get-in *state* [:export-options :indent-style]))
+                                        (update :fns-on-inline-coll conj cli-export-common/remove-prefix-spaces-in-Plain))
+            ast*** (if-not (empty? config-for-walk-block-ast)
+                     (mapv (partial cli-export-common/walk-block-ast config-for-walk-block-ast) ast**)
+                     ast**)
+            simple-asts (mapcatv block-ast->simple-ast ast***)]
+        (simple-asts->string simple-asts)))))

src/main/frontend/common/file/core.cljs → deps/cli/src/logseq/cli/common/file.cljs

@@ -1,11 +1,12 @@
-(ns frontend.common.file.core
-  "Convert blocks to file content. Used for exports and saving file to disk. Shared
-  by worker and frontend namespaces"
+(ns logseq.cli.common.file
+  "Convert blocks to file content for file and DB graphs. Used for exports and
+  saving file to disk. Shared by CLI, worker and frontend namespaces"
   (:require [clojure.string :as string]
             [datascript.core :as d]
             [logseq.db :as ldb]
             [logseq.db.common.entity-plus :as entity-plus]
             [logseq.db.frontend.content :as db-content]
+            [logseq.db.sqlite.create-graph :as sqlite-create-graph]
             [logseq.db.sqlite.util :as sqlite-util]
             [logseq.graph-parser.property :as gp-property]
             [logseq.outliner.tree :as otree]))
@@ -29,7 +30,7 @@
     :else
     content))
 
-(defn- transform-content
+(defn- ^:large-vars/cleanup-todo transform-content
   [repo db {:block/keys [collapsed? format pre-block? properties] :as b} level {:keys [heading-to-list?]} context {:keys [db-based?]}]
   (let [title (or (:block/raw-title b) (:block/title b))
         block-ref-not-saved? (and (not db-based?)
@@ -91,6 +92,7 @@
               (if-let [children (seq (:block/children f))]
                 (cons content (tree->file-content-aux repo db children {:init-level (inc level)} context))
                 [content])]
+          #_:clj-kondo/ignore
           (conj! block-contents new-content)
           (recur r level))))))
 
@@ -120,3 +122,18 @@
     (tree->file-content repo db tree
                         (assoc tree->file-opts :init-level init-level)
                         context)))
+
+(defn get-all-page->content
+  "Exports a graph's pages as tuples of page name and page content"
+  [repo db options]
+  (let [filter-fn (if (ldb/db-based-graph? db)
+                    (fn [ent]
+                      (or (not (:logseq.property/built-in? ent))
+                          (contains? sqlite-create-graph/built-in-pages-names (:block/title ent))))
+                    (constantly true))]
+    (->> (d/datoms db :avet :block/name)
+         (map #(d/entity db (:e %)))
+         (filter filter-fn)
+         (map (fn [e]
+                [(:block/title e)
+                 (block->content repo db (:block/uuid e) {} options)])))))

deps/cli/src/logseq/cli/common/graph.cljs

@@ -7,7 +7,7 @@
             [logseq.common.config :as common-config]
             [logseq.common.graph :as common-graph]))
 
-(defn- graph-name->path
+(defn ^:api graph-name->path
   [graph-name]
   (when graph-name
     (-> graph-name

deps/cli/src/logseq/cli/common/util.cljc

@@ -0,0 +1,36 @@
+(ns logseq.cli.common.util
+  "Common util fns between CLI and frontend"
+  (:require #?(:org.babashka/nbb ["jszip$default" :as JSZip]
+               :cljs ["jszip" :as JSZip])
+            #_:clj-kondo/ignore
+            [clojure.string :as string]))
+
+#?(:cljs
+   (defn make-export-zip
+     "Makes a zipfile for an exported version of graph. Removes files with blank content"
+     [zip-filename file-name-content]
+     (let [zip (JSZip.)
+           folder (.folder zip zip-filename)]
+       (doseq [[file-name content] file-name-content]
+         (when-not (string/blank? content)
+           (.file folder (-> file-name
+                             (string/replace #"^/+" ""))
+                  content)))
+       zip)))
+
+;; Macros are defined at top-level for frontend and nbb
+
+(defmacro concatv
+  "Vector version of concat. non-lazy"
+  [& args]
+  `(vec (concat ~@args)))
+
+(defmacro mapcatv
+  "Vector version of mapcat. non-lazy"
+  [f coll & colls]
+  `(vec (mapcat ~f ~coll ~@colls)))
+
+(defmacro removev
+  "Vector version of remove. non-lazy"
+  [pred coll]
+  `(vec (remove ~pred ~coll)))

deps/cli/src/logseq/cli/spec.cljs

@@ -2,11 +2,15 @@
   "Babashka.cli specs for commands. Normally these would live alongside their
   commands but are separate because command namespaces are lazy loaded")
 
+(def export
+  {:file {:alias :f
+          :desc "File to save export"}})
+
 (def export-edn
   {:include-timestamps? {:alias :T
                          :desc "Include timestamps in export"}
    :file {:alias :f
-          :desc "Saves edn to file"}
+          :desc "File to save export"}
    :catch-validation-errors? {:alias :c
                               :desc "Catch validation errors for dev"}
    :exclude-namespaces {:alias :e
@@ -24,20 +28,24 @@
 (def query
   {:graphs {:alias :g
             :coerce []
-            :desc "Additional graphs to query"}
+            :desc "Additional graphs to local query"}
    :properties-readable {:alias :p
                          :coerce :boolean
-                         :desc "Make properties on entity queries show property values instead of ids"}
+                         :desc "Make properties on local, entity queries show property values instead of ids"}
    :title-query {:alias :t
-                 :desc "Invokes local query on :block/title"}
-   :api-query-token {:alias :a
-                     :desc "Query current graph with api server token"}})
+                 :desc "Invoke local query on :block/title"}
+   :api-server-token {:alias :a
+                      :desc "API server token to query current graph"}})
 
 (def search
-  {:api-query-token {:alias :a
-                     :desc "Api server token"}
+  {:api-server-token {:alias :a
+                      :desc "API server token to search current graph"}
    :raw {:alias :r
          :desc "Print raw response"}
    :limit {:alias :l
            :default 100
-           :desc "Limit max number of results"}})
+           :desc "Limit max number of results"}})
+
+(def append
+  {:api-server-token {:alias :a
+                      :desc "API server token to modify current graph"}})

deps/cli/src/logseq/cli/text_util.cljs

@@ -20,4 +20,24 @@
                 m-cut (subs m-cut 0 e-pos)]
             [b-cut m-cut e-cut])
           [b-cut m-cut nil]))
-      [value nil nil])))
+      [value nil nil])))
+
+(defn wrap-text
+  "Wraps a string to a given width without breaking words. Returns a single string with newlines."
+  [s width]
+  (->> (loop [remaining (string/trim s)
+              acc []]
+         (if (empty? remaining)
+           acc
+           (if (<= (count remaining) width)
+             (conj acc remaining)
+             (let [substring (subs remaining 0 width)
+                   split-idx (or (string/last-index-of substring \space)
+                                 (string/last-index-of substring \tab)
+                                 ;; fallback: hard split
+                                 80)
+                   line (subs remaining 0 split-idx)
+                   rest' (subs remaining split-idx)]
+               (recur (string/triml rest')
+                      (conj acc line))))))
+       (string/join "\n")))

deps/cli/src/logseq/cli/util.cljs

@@ -1,9 +1,10 @@
 (ns ^:node-only logseq.cli.util
-  "Util fns"
+  "CLI only util fns"
   (:require ["path" :as node-path]
             [clojure.string :as string]
             [logseq.cli.common.graph :as cli-common-graph]
             [logseq.db.common.sqlite :as common-sqlite]
+            [promesa.core :as p]
             [nbb.error]))
 
 (defn get-graph-dir
@@ -18,18 +19,25 @@
 (defn api-fetch [token method args]
   (js/fetch "http://127.0.0.1:12315/api"
             (clj->js {:method "POST"
-                      :headers {"Authorization" (str "Bearer " token)
+                      :headers {"Authorization"
+                                (str "Bearer " (or token js/process.env.LOGSEQ_API_SERVER_TOKEN))
                                 "Content-Type" "application/json"}
                       :body (js/JSON.stringify
                              (clj->js {:method method
                                        :args args}))})))
 
 (defn api-handle-error-response
-  "Handles a non 200 response"
+  "Handles a non 200 response. For 500 return full response to provide more detail"
   [resp]
-  (js/console.error "Error: API Server responded with status" (.-status resp)
-                    (when (.-statusText resp) (str "and body " (pr-str (.-statusText resp)))))
-  (js/process.exit 1))
+  (if (= 500 (.-status resp))
+    (p/let [body (.text resp)]
+      (js/console.error "Error: API Server responded with status" (.-status resp)
+                        "\nAPI Response:" (pr-str body))
+      (js/process.exit 1))
+    (do
+      (js/console.error "Error: API Server responded with status" (.-status resp)
+                        (when (.-statusText resp) (str "and body " (pr-str (.-statusText resp)))))
+      (js/process.exit 1))))
 
 (defn command-catch-handler
   "Default p/catch handler for commands which handles sci errors and HTTP API Server connections gracefully"

deps/cli/yarn.lock

@@ -2,12 +2,22 @@
 # yarn lockfile v1
 
 
-"@logseq/nbb-logseq@github:logseq/nbb-logseq#feat-db-v26":
-  version "1.2.173-feat-db-v25"
-  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/b26d290944234b20762ff109e5328b87ea240692"
+"@logseq/nbb-logseq@github:logseq/nbb-logseq#feat-db-v28":
+  version "1.2.173-feat-db-v28"
+  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/c0410ff81a8b0d510705581cccd39788d862dc91"
   dependencies:
     import-meta-resolve "^4.1.0"
 
+ansi-regex@^2.0.0:
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-2.1.1.tgz#c3b33ab5ee360d86e0e628f0468ae7ef27d654df"
+  integrity sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==
+
+ansi-regex@^3.0.0:
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-3.0.1.tgz#123d6479e92ad45ad897d4054e3c7ca7db4944e1"
+  integrity sha512-+O9Jct8wf++lXxxFc4hc8LsjaSq0HFzzL7cVsw8pRDIPdjKD2mT4ytDZlLuSBZ4cLKZFXIrMGO7DbQCtMJJMKw==
+
 base64-js@^1.3.1:
   version "1.5.1"
   resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a"
@@ -45,11 +55,51 @@ buffer@^5.5.0:
     base64-js "^1.3.1"
     ieee754 "^1.1.13"
 
+camelcase@^5.0.0:
+  version "5.3.1"
+  resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-5.3.1.tgz#e3c9b31569e106811df242f715725a1f4c494320"
+  integrity sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==
+
 chownr@^1.1.1:
   version "1.1.4"
   resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.1.4.tgz#6fc9d7b42d32a583596337666e7d08084da2cc6b"
   integrity sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==
 
+cliui@^4.0.0:
+  version "4.1.0"
+  resolved "https://registry.yarnpkg.com/cliui/-/cliui-4.1.0.tgz#348422dbe82d800b3022eef4f6ac10bf2e4d1b49"
+  integrity sha512-4FG+RSG9DL7uEwRUZXZn3SS34DiDPfzP0VOiEwtUWlE+AR2EIg+hSyvrIgUUfhdgR/UkAeW2QHgeP+hWrXs7jQ==
+  dependencies:
+    string-width "^2.1.1"
+    strip-ansi "^4.0.0"
+    wrap-ansi "^2.0.0"
+
+code-point-at@^1.0.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/code-point-at/-/code-point-at-1.1.0.tgz#0d070b4d043a5bea33a2f1a40e2edb3d9a4ccf77"
+  integrity sha512-RpAVKQA5T63xEj6/giIbUEtZwJ4UFIc3ZtvEkiaUERylqe8xb5IvqcgOurZLahv93CLKfxcw5YI+DZcUBRyLXA==
+
+core-util-is@~1.0.0:
+  version "1.0.3"
+  resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.3.tgz#a6042d3634c2b27e9328f837b965fac83808db85"
+  integrity sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==
+
+cross-spawn@^6.0.0:
+  version "6.0.6"
+  resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-6.0.6.tgz#30d0efa0712ddb7eb5a76e1e8721bffafa6b5d57"
+  integrity sha512-VqCUuhcd1iB+dsv8gxPttb5iZh/D0iubSP21g36KXdEuf6I5JiioesUVjpCdHV9MZRUfVFlvwtIUyPfxo5trtw==
+  dependencies:
+    nice-try "^1.0.4"
+    path-key "^2.0.1"
+    semver "^5.5.0"
+    shebang-command "^1.2.0"
+    which "^1.2.9"
+
+decamelize@^1.2.0:
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290"
+  integrity sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==
+
 decompress-response@^6.0.0:
   version "6.0.0"
   resolved "https://registry.yarnpkg.com/decompress-response/-/decompress-response-6.0.0.tgz#ca387612ddb7e104bd16d85aab00d5ecf09c66fc"
@@ -74,6 +124,19 @@ end-of-stream@^1.1.0, end-of-stream@^1.4.1:
   dependencies:
     once "^1.4.0"
 
+execa@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/execa/-/execa-1.0.0.tgz#c6236a5bb4df6d6f15e88e7f017798216749ddd8"
+  integrity sha512-adbxcyWV46qiHyvSp50TKt05tB4tK3HcmF7/nxfAdhnox83seTDbwnaqKO4sXRy7roHAIFqJP/Rw/AuEbX61LA==
+  dependencies:
+    cross-spawn "^6.0.0"
+    get-stream "^4.0.0"
+    is-stream "^1.1.0"
+    npm-run-path "^2.0.0"
+    p-finally "^1.0.0"
+    signal-exit "^3.0.0"
+    strip-eof "^1.0.0"
+
 expand-template@^2.0.3:
   version "2.0.3"
   resolved "https://registry.yarnpkg.com/expand-template/-/expand-template-2.0.3.tgz#6e14b3fcee0f3a6340ecb57d2e8918692052a47c"
@@ -84,6 +147,13 @@ [email protected]:
   resolved "https://registry.yarnpkg.com/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz#553a7b8446ff6f684359c445f1e37a05dacc33dd"
   integrity sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==
 
+find-up@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/find-up/-/find-up-3.0.0.tgz#49169f1d7993430646da61ecc5ae355c21c97b73"
+  integrity sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==
+  dependencies:
+    locate-path "^3.0.0"
+
 fs-constants@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/fs-constants/-/fs-constants-1.0.0.tgz#6be0de9be998ce16af8afc24497b9ee9b7ccd9ad"
@@ -98,6 +168,18 @@ fs-extra@^11.3.0:
     jsonfile "^6.0.1"
     universalify "^2.0.0"
 
+get-caller-file@^1.0.1:
+  version "1.0.3"
+  resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-1.0.3.tgz#f978fa4c90d1dfe7ff2d6beda2a515e713bdcf4a"
+  integrity sha512-3t6rVToeoZfYSGd8YoLFR2DJkiQrIiUrGcjvFX2mDw3bn6k2OtwHN0TNCLbBO+w8qTvimhDkv+LSscbJY1vE6w==
+
+get-stream@^4.0.0:
+  version "4.1.0"
+  resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-4.1.0.tgz#c1b255575f3dc21d59bfc79cd3d2b46b1c3a54b5"
+  integrity sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==
+  dependencies:
+    pump "^3.0.0"
+
 [email protected]:
   version "0.0.0"
   resolved "https://registry.yarnpkg.com/github-from-package/-/github-from-package-0.0.0.tgz#97fb5d96bfde8973313f20e8288ef9a167fa64ce"
@@ -113,12 +195,17 @@ ieee754@^1.1.13:
   resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352"
   integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==
 
+immediate@~3.0.5:
+  version "3.0.6"
+  resolved "https://registry.yarnpkg.com/immediate/-/immediate-3.0.6.tgz#9db1dbd0faf8de6fbe0f5dd5e56bb606280de69b"
+  integrity sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==
+
 import-meta-resolve@^4.1.0:
   version "4.1.0"
   resolved "https://registry.yarnpkg.com/import-meta-resolve/-/import-meta-resolve-4.1.0.tgz#f9db8bead9fafa61adb811db77a2bf22c5399706"
   integrity sha512-I6fiaX09Xivtk+THaMfAwnA3MVA5Big1WHF1Dfx9hFuvNIWpXnorlkzhcQf6ehrqQiiZECRt1poOAkPmer3ruw==
 
-inherits@^2.0.3, inherits@^2.0.4:
+inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.3:
   version "2.0.4"
   resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c"
   integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==
@@ -128,6 +215,38 @@ ini@~1.3.0:
   resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.8.tgz#a29da425b48806f34767a4efce397269af28432c"
   integrity sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==
 
+invert-kv@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/invert-kv/-/invert-kv-2.0.0.tgz#7393f5afa59ec9ff5f67a27620d11c226e3eec02"
+  integrity sha512-wPVv/y/QQ/Uiirj/vh3oP+1Ww+AWehmi1g5fFWGPF6IpCBCDVrhgHRMvrLfdYcwDh3QJbGXDW4JAuzxElLSqKA==
+
+is-fullwidth-code-point@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz#ef9e31386f031a7f0d643af82fde50c457ef00cb"
+  integrity sha512-1pqUqRjkhPJ9miNq9SwMfdvi6lBJcd6eFxvfaivQhaH3SgisfiuudvFntdKOmxuee/77l+FPjKrQjWvmPjWrRw==
+  dependencies:
+    number-is-nan "^1.0.0"
+
+is-fullwidth-code-point@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz#a3b30a5c4f199183167aaab93beefae3ddfb654f"
+  integrity sha512-VHskAKYM8RfSFXwee5t5cbN5PZeq1Wrh6qd5bkyiXIf6UQcN6w/A0eXM9r6t8d+GYOh+o6ZhiEnb88LN/Y8m2w==
+
+is-stream@^1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-1.1.0.tgz#12d4a3dd4e68e0b79ceb8dbc84173ae80d91ca44"
+  integrity sha512-uQPm8kcs47jx38atAcWTVxyltQYoPT68y9aWYdV6yWXSyW8mzSat0TL6CiWdZeCdF3KrAvpVtnHbTv4RN+rqdQ==
+
+isarray@~1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11"
+  integrity sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==
+
+isexe@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10"
+  integrity sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==
+
 jsonfile@^6.0.1:
   version "6.1.0"
   resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-6.1.0.tgz#bc55b2634793c679ec6403094eb13698a6ec0aae"
@@ -137,6 +256,59 @@ jsonfile@^6.0.1:
   optionalDependencies:
     graceful-fs "^4.1.6"
 
[email protected]:
+  version "3.8.0"
+  resolved "https://registry.yarnpkg.com/jszip/-/jszip-3.8.0.tgz#a2ac3c33fe96a76489765168213655850254d51b"
+  integrity sha512-cnpQrXvFSLdsR9KR5/x7zdf6c3m8IhZfZzSblFEHSqBaVwD2nvJ4CuCKLyvKvwBgZm08CgfSoiTBQLm5WW9hGw==
+  dependencies:
+    lie "~3.3.0"
+    pako "~1.0.2"
+    readable-stream "~2.3.6"
+    set-immediate-shim "~1.0.1"
+
+lcid@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/lcid/-/lcid-2.0.0.tgz#6ef5d2df60e52f82eb228a4c373e8d1f397253cf"
+  integrity sha512-avPEb8P8EGnwXKClwsNUgryVjllcRqtMYa49NTsbQagYuT1DcXnl1915oxWjoyGrXR6zH/Y0Zc96xWsPcoDKeA==
+  dependencies:
+    invert-kv "^2.0.0"
+
+lie@~3.3.0:
+  version "3.3.0"
+  resolved "https://registry.yarnpkg.com/lie/-/lie-3.3.0.tgz#dcf82dee545f46074daf200c7c1c5a08e0f40f6a"
+  integrity sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==
+  dependencies:
+    immediate "~3.0.5"
+
+locate-path@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-3.0.0.tgz#dbec3b3ab759758071b58fe59fc41871af21400e"
+  integrity sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==
+  dependencies:
+    p-locate "^3.0.0"
+    path-exists "^3.0.0"
+
+map-age-cleaner@^0.1.1:
+  version "0.1.3"
+  resolved "https://registry.yarnpkg.com/map-age-cleaner/-/map-age-cleaner-0.1.3.tgz#7d583a7306434c055fe474b0f45078e6e1b4b92a"
+  integrity sha512-bJzx6nMoP6PDLPBFmg7+xRKeFZvFboMrGlxmNj9ClvX53KrmvM5bXFXEWjbz4cz1AFn+jWJ9z/DJSz7hrs0w3w==
+  dependencies:
+    p-defer "^1.0.0"
+
+mem@^4.0.0:
+  version "4.3.0"
+  resolved "https://registry.yarnpkg.com/mem/-/mem-4.3.0.tgz#461af497bc4ae09608cdb2e60eefb69bff744178"
+  integrity sha512-qX2bG48pTqYRVmDB37rn/6PT7LcR8T7oAX3bf99u1Tt1nzxYfxkgqDwUwolPlXweM0XzBOBFzSx4kfp7KP1s/w==
+  dependencies:
+    map-age-cleaner "^0.1.1"
+    mimic-fn "^2.0.0"
+    p-is-promise "^2.0.0"
+
+mimic-fn@^2.0.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-2.1.0.tgz#7ed2c2ccccaf84d3ffcb7a69b57711fc2083401b"
+  integrity sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==
+
 mimic-response@^3.1.0:
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-3.1.0.tgz#2d1d59af9c1b129815accc2c46a022a5ce1fa3c9"
@@ -152,11 +324,23 @@ mkdirp-classic@^0.5.2, mkdirp-classic@^0.5.3:
   resolved "https://registry.yarnpkg.com/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz#fa10c9115cc6d8865be221ba47ee9bed78601113"
   integrity sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==
 
+mldoc@^1.5.9:
+  version "1.5.9"
+  resolved "https://registry.yarnpkg.com/mldoc/-/mldoc-1.5.9.tgz#43d740351c64285f0f4988ac9497922d54ae66fc"
+  integrity sha512-87FQ7hseS87tsk+VdpIigpu8LH+GwmbbFgpxgFwvnbH5oOjmIrc47laH4Dyggzqiy8/vMjDHkl7vsId0eXhCDQ==
+  dependencies:
+    yargs "^12.0.2"
+
 napi-build-utils@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/napi-build-utils/-/napi-build-utils-2.0.0.tgz#13c22c0187fcfccce1461844136372a47ddc027e"
   integrity sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==
 
+nice-try@^1.0.4:
+  version "1.0.5"
+  resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.5.tgz#a3378a7696ce7d223e88fc9b764bd7ef1089e366"
+  integrity sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==
+
 node-abi@^3.3.0:
   version "3.75.0"
   resolved "https://registry.yarnpkg.com/node-abi/-/node-abi-3.75.0.tgz#2f929a91a90a0d02b325c43731314802357ed764"
@@ -164,6 +348,18 @@ node-abi@^3.3.0:
   dependencies:
     semver "^7.3.5"
 
+npm-run-path@^2.0.0:
+  version "2.0.2"
+  resolved "https://registry.yarnpkg.com/npm-run-path/-/npm-run-path-2.0.2.tgz#35a9232dfa35d7067b4cb2ddf2357b1871536c5f"
+  integrity sha512-lJxZYlT4DW/bRUtFh1MQIWqmLwQfAxnqWG4HhEdjMlkrJYnJn0Jrr2u3mgxqaWsdiBc76TYkTG/mhrnYTuzfHw==
+  dependencies:
+    path-key "^2.0.0"
+
+number-is-nan@^1.0.0:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/number-is-nan/-/number-is-nan-1.0.1.tgz#097b602b53422a522c1afb8790318336941a011d"
+  integrity sha512-4jbtZXNAsfZbAHiiqjLPBiCl16dES1zI4Hpzzxw61Tk+loF+sBDBKx1ICKKKwIqQ7M0mFn1TmkN7euSncWgHiQ==
+
 once@^1.3.1, once@^1.4.0:
   version "1.4.0"
   resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1"
@@ -171,6 +367,64 @@ once@^1.3.1, once@^1.4.0:
   dependencies:
     wrappy "1"
 
+os-locale@^3.0.0:
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/os-locale/-/os-locale-3.1.0.tgz#a802a6ee17f24c10483ab9935719cef4ed16bf1a"
+  integrity sha512-Z8l3R4wYWM40/52Z+S265okfFj8Kt2cC2MKY+xNi3kFs+XGI7WXu/I309QQQYbRW4ijiZ+yxs9pqEhJh0DqW3Q==
+  dependencies:
+    execa "^1.0.0"
+    lcid "^2.0.0"
+    mem "^4.0.0"
+
+p-defer@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/p-defer/-/p-defer-1.0.0.tgz#9f6eb182f6c9aa8cd743004a7d4f96b196b0fb0c"
+  integrity sha512-wB3wfAxZpk2AzOfUMJNL+d36xothRSyj8EXOa4f6GMqYDN9BJaaSISbsk+wS9abmnebVw95C2Kb5t85UmpCxuw==
+
+p-finally@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/p-finally/-/p-finally-1.0.0.tgz#3fbcfb15b899a44123b34b6dcc18b724336a2cae"
+  integrity sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==
+
+p-is-promise@^2.0.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/p-is-promise/-/p-is-promise-2.1.0.tgz#918cebaea248a62cf7ffab8e3bca8c5f882fc42e"
+  integrity sha512-Y3W0wlRPK8ZMRbNq97l4M5otioeA5lm1z7bkNkxCka8HSPjR0xRWmpCmc9utiaLP9Jb1eD8BgeIxTW4AIF45Pg==
+
+p-limit@^2.0.0:
+  version "2.3.0"
+  resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-2.3.0.tgz#3dd33c647a214fdfffd835933eb086da0dc21db1"
+  integrity sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==
+  dependencies:
+    p-try "^2.0.0"
+
+p-locate@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-3.0.0.tgz#322d69a05c0264b25997d9f40cd8a891ab0064a4"
+  integrity sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==
+  dependencies:
+    p-limit "^2.0.0"
+
+p-try@^2.0.0:
+  version "2.2.0"
+  resolved "https://registry.yarnpkg.com/p-try/-/p-try-2.2.0.tgz#cb2868540e313d61de58fafbe35ce9004d5540e6"
+  integrity sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==
+
+pako@~1.0.2:
+  version "1.0.11"
+  resolved "https://registry.yarnpkg.com/pako/-/pako-1.0.11.tgz#6c9599d340d54dfd3946380252a35705a6b992bf"
+  integrity sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==
+
+path-exists@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-3.0.0.tgz#ce0ebeaa5f78cb18925ea7d810d7b59b010fd515"
+  integrity sha512-bpC7GYwiDYQ4wYLe+FA8lhRjhQCMcQGuSgGGqDkg/QerRWw9CmGRT0iSOVRSZJ29NMLZgIzqaljJ63oaL4NIJQ==
+
+path-key@^2.0.0, path-key@^2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/path-key/-/path-key-2.0.1.tgz#411cadb574c5a140d3a4b1910d40d80cc9f40b40"
+  integrity sha512-fEHGKCSmUSDPv4uoj8AlD+joPlq3peND+HRYyxFz4KPw4z926S/b8rIuFs2FYJg3BwsxJf6A9/3eIdLaYC+9Dw==
+
 prebuild-install@^7.1.1:
   version "7.1.3"
   resolved "https://registry.yarnpkg.com/prebuild-install/-/prebuild-install-7.1.3.tgz#d630abad2b147443f20a212917beae68b8092eec"
@@ -189,6 +443,11 @@ prebuild-install@^7.1.1:
     tar-fs "^2.0.0"
     tunnel-agent "^0.6.0"
 
+process-nextick-args@~2.0.0:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2"
+  integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==
+
 pump@^3.0.0:
   version "3.0.3"
   resolved "https://registry.yarnpkg.com/pump/-/pump-3.0.3.tgz#151d979f1a29668dc0025ec589a455b53282268d"
@@ -216,16 +475,76 @@ readable-stream@^3.1.1, readable-stream@^3.4.0:
     string_decoder "^1.1.1"
     util-deprecate "^1.0.1"
 
+readable-stream@~2.3.6:
+  version "2.3.8"
+  resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.8.tgz#91125e8042bba1b9887f49345f6277027ce8be9b"
+  integrity sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==
+  dependencies:
+    core-util-is "~1.0.0"
+    inherits "~2.0.3"
+    isarray "~1.0.0"
+    process-nextick-args "~2.0.0"
+    safe-buffer "~5.1.1"
+    string_decoder "~1.1.1"
+    util-deprecate "~1.0.1"
+
+require-directory@^2.1.1:
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42"
+  integrity sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==
+
+require-main-filename@^1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/require-main-filename/-/require-main-filename-1.0.1.tgz#97f717b69d48784f5f526a6c5aa8ffdda055a4d1"
+  integrity sha512-IqSUtOVP4ksd1C/ej5zeEh/BIP2ajqpn8c5x+q99gvcIG/Qf0cud5raVnE/Dwd0ua9TXYDoDc0RE5hBSdz22Ug==
+
 safe-buffer@^5.0.1, safe-buffer@~5.2.0:
   version "5.2.1"
   resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6"
   integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==
 
+safe-buffer@~5.1.0, safe-buffer@~5.1.1:
+  version "5.1.2"
+  resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d"
+  integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==
+
+semver@^5.5.0:
+  version "5.7.2"
+  resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.2.tgz#48d55db737c3287cd4835e17fa13feace1c41ef8"
+  integrity sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==
+
 semver@^7.3.5:
   version "7.7.2"
   resolved "https://registry.yarnpkg.com/semver/-/semver-7.7.2.tgz#67d99fdcd35cec21e6f8b87a7fd515a33f982b58"
   integrity sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA==
 
+set-blocking@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7"
+  integrity sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==
+
+set-immediate-shim@~1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/set-immediate-shim/-/set-immediate-shim-1.0.1.tgz#4b2b1b27eb808a9f8dcc481a58e5e56f599f3f61"
+  integrity sha512-Li5AOqrZWCVA2n5kryzEmqai6bKSIvpz5oUJHPVj6+dsbD3X1ixtsY5tEnsaNpH3pFAHmG8eIHUrtEtohrg+UQ==
+
+shebang-command@^1.2.0:
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/shebang-command/-/shebang-command-1.2.0.tgz#44aac65b695b03398968c39f363fee5deafdf1ea"
+  integrity sha512-EV3L1+UQWGor21OmnvojK36mhg+TyIKDh3iFBKBohr5xeXIhNBcx8oWdgkTEEQ+BEFFYdLRuqMfd5L84N1V5Vg==
+  dependencies:
+    shebang-regex "^1.0.0"
+
+shebang-regex@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-1.0.0.tgz#da42f49740c0b42db2ca9728571cb190c98efea3"
+  integrity sha512-wpoSFAxys6b2a2wHZ1XpDSgD7N9iVjg29Ph9uV/uaP9Ex/KXlkTZTeddxDPSYQpgvzKLGJke2UU0AzoGCjNIvQ==
+
+signal-exit@^3.0.0:
+  version "3.0.7"
+  resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.7.tgz#a9a1767f8af84155114eaabd73f99273c8f59ad9"
+  integrity sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==
+
 simple-concat@^1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/simple-concat/-/simple-concat-1.0.1.tgz#f46976082ba35c2263f1c8ab5edfe26c41c9552f"
@@ -240,6 +559,23 @@ simple-get@^4.0.0:
     once "^1.3.1"
     simple-concat "^1.0.0"
 
+string-width@^1.0.1:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/string-width/-/string-width-1.0.2.tgz#118bdf5b8cdc51a2a7e70d211e07e2b0b9b107d3"
+  integrity sha512-0XsVpQLnVCXHJfyEs8tC0zpTVIr5PKKsQtkT29IwupnPTjtPmQ3xT/4yCREF9hYkV/3M3kzcUTSAZT6a6h81tw==
+  dependencies:
+    code-point-at "^1.0.0"
+    is-fullwidth-code-point "^1.0.0"
+    strip-ansi "^3.0.0"
+
+string-width@^2.0.0, string-width@^2.1.1:
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/string-width/-/string-width-2.1.1.tgz#ab93f27a8dc13d28cac815c462143a6d9012ae9e"
+  integrity sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==
+  dependencies:
+    is-fullwidth-code-point "^2.0.0"
+    strip-ansi "^4.0.0"
+
 string_decoder@^1.1.1:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.3.0.tgz#42f114594a46cf1a8e30b0a84f56c78c3edac21e"
@@ -247,6 +583,32 @@ string_decoder@^1.1.1:
   dependencies:
     safe-buffer "~5.2.0"
 
+string_decoder@~1.1.1:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8"
+  integrity sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==
+  dependencies:
+    safe-buffer "~5.1.0"
+
+strip-ansi@^3.0.0, strip-ansi@^3.0.1:
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-3.0.1.tgz#6a385fb8853d952d5ff05d0e8aaf94278dc63dcf"
+  integrity sha512-VhumSSbBqDTP8p2ZLKj40UjBCV4+v8bUSEpUb4KjRgWk9pbqGF4REFj6KEagidb2f/M6AzC0EmFyDNGaw9OCzg==
+  dependencies:
+    ansi-regex "^2.0.0"
+
+strip-ansi@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-4.0.0.tgz#a8479022eb1ac368a871389b635262c505ee368f"
+  integrity sha512-4XaJ2zQdCzROZDivEVIDPkcQn8LMFSa8kj8Gxb/Lnwzv9A8VctNZ+lfivC/sV3ivW8ElJTERXZoPBRrZKkNKow==
+  dependencies:
+    ansi-regex "^3.0.0"
+
+strip-eof@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/strip-eof/-/strip-eof-1.0.0.tgz#bb43ff5598a6eb05d89b59fcd129c983313606bf"
+  integrity sha512-7FCwGGmx8mD5xQd3RPUvnSpUXHM3BWuzjtpD4TXsfcZ9EL4azvVVUscFYwD9nx8Kh+uCBC00XBtAykoMHwTh8Q==
+
 strip-json-comments@~2.0.1:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-2.0.1.tgz#3c531942e908c2697c0ec344858c286c7ca0a60a"
@@ -285,12 +647,63 @@ universalify@^2.0.0:
   resolved "https://registry.yarnpkg.com/universalify/-/universalify-2.0.1.tgz#168efc2180964e6386d061e094df61afe239b18d"
   integrity sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==
 
-util-deprecate@^1.0.1:
+util-deprecate@^1.0.1, util-deprecate@~1.0.1:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"
   integrity sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==
 
+which-module@^2.0.0:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/which-module/-/which-module-2.0.1.tgz#776b1fe35d90aebe99e8ac15eb24093389a4a409"
+  integrity sha512-iBdZ57RDvnOR9AGBhML2vFZf7h8vmBjhoaZqODJBFWHVtKkDmKuHai3cx5PgVMrX5YDNp27AofYbAwctSS+vhQ==
+
+which@^1.2.9:
+  version "1.3.1"
+  resolved "https://registry.yarnpkg.com/which/-/which-1.3.1.tgz#a45043d54f5805316da8d62f9f50918d3da70b0a"
+  integrity sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==
+  dependencies:
+    isexe "^2.0.0"
+
+wrap-ansi@^2.0.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-2.1.0.tgz#d8fc3d284dd05794fe84973caecdd1cf824fdd85"
+  integrity sha512-vAaEaDM946gbNpH5pLVNR+vX2ht6n0Bt3GXwVB1AuAqZosOvHNF3P7wDnh8KLkSqgUh0uh77le7Owgoz+Z9XBw==
+  dependencies:
+    string-width "^1.0.1"
+    strip-ansi "^3.0.1"
+
 wrappy@1:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f"
   integrity sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==
+
+"y18n@^3.2.1 || ^4.0.0":
+  version "4.0.3"
+  resolved "https://registry.yarnpkg.com/y18n/-/y18n-4.0.3.tgz#b5f259c82cd6e336921efd7bfd8bf560de9eeedf"
+  integrity sha512-JKhqTOwSrqNA1NY5lSztJ1GrBiUodLMmIZuLiDaMRJ+itFd+ABVE8XBjOvIWL+rSqNDC74LCSFmlb/U4UZ4hJQ==
+
+yargs-parser@^11.1.1:
+  version "11.1.1"
+  resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-11.1.1.tgz#879a0865973bca9f6bab5cbdf3b1c67ec7d3bcf4"
+  integrity sha512-C6kB/WJDiaxONLJQnF8ccx9SEeoTTLek8RVbaOIsrAUS8VrBEXfmeSnCZxygc+XC2sNMBIwOOnfcxiynjHsVSQ==
+  dependencies:
+    camelcase "^5.0.0"
+    decamelize "^1.2.0"
+
+yargs@^12.0.2:
+  version "12.0.5"
+  resolved "https://registry.yarnpkg.com/yargs/-/yargs-12.0.5.tgz#05f5997b609647b64f66b81e3b4b10a368e7ad13"
+  integrity sha512-Lhz8TLaYnxq/2ObqHDql8dX8CJi97oHxrjUcYtzKbbykPtVW9WB+poxI+NM2UIzsMgNCZTIf0AQwsjK5yMAqZw==
+  dependencies:
+    cliui "^4.0.0"
+    decamelize "^1.2.0"
+    find-up "^3.0.0"
+    get-caller-file "^1.0.1"
+    os-locale "^3.0.0"
+    require-directory "^2.1.1"
+    require-main-filename "^1.0.1"
+    set-blocking "^2.0.0"
+    string-width "^2.0.0"
+    which-module "^2.0.0"
+    y18n "^3.2.1 || ^4.0.0"
+    yargs-parser "^11.1.1"

deps/common/package.json

@@ -3,7 +3,7 @@
   "version": "1.0.0",
   "private": true,
   "devDependencies": {
-    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v26"
+    "@logseq/nbb-logseq": "github:logseq/nbb-logseq#feat-db-v28"
   },
   "scripts": {
     "test": "yarn nbb-logseq -cp test -m nextjournal.test-runner"

deps/common/src/logseq/common/defkeywords.cljc

@@ -3,7 +3,6 @@
   #?(:cljs (:require-macros [logseq.common.defkeywords])))
 
 (def ^:private *defined-kws (volatile! {}))
-(def ^:private *defined-kw->config (volatile! {}))
 
 #_:clj-kondo/ignore
 (defmacro defkeyword
@@ -25,16 +24,8 @@
           (vswap! *defined-kws assoc kw current-meta)
           (throw (ex-info "keyword already defined somewhere else" {:kw kw :info info}))))
       (vswap! *defined-kws assoc kw current-meta))
-    (let [kw->config (partition 2 keyvals)]
-      (doseq [[kw config] kw->config]
-        (vswap! *defined-kw->config assoc kw config))))
+    ;; (let [kw->config (partition 2 keyvals)]
+    ;;   (doseq [[kw config] kw->config]
+    ;;     (vswap! *defined-kw->config assoc kw config)))
+    )
   `(vector ~@keyvals))
-
-(defmacro get-all-defined-kw->config
-  []
-  `'~(deref *defined-kw->config))
-
-(comment
-  "update anything here to trigger this ns to be recompiled,
-so macro get-all-defined-kw->config's result will be updated."
-  1)

deps/common/src/logseq/common/util.cljs

@@ -113,7 +113,6 @@
       (js->clj :keywordize-keys true)))
 
 (defn zero-pad
-  "Copy of frontend.util/zero-pad. Too basic to couple to main app"
   [n]
   (if (< n 10)
     (str "0" n)

deps/common/src/logseq/common/util/date_time.cljs

@@ -6,6 +6,8 @@
             [clojure.string :as string]
             [logseq.common.util :as common-util]))
 
+(def ^:private yyyyMMdd-formatter (tf/formatter "yyyyMMdd"))
+
 ;; (tf/parse (tf/formatter "dd.MM.yyyy") "2021Q4") => 20040120T000000
 (defn safe-journal-title-formatters
   [date-formatter]
@@ -32,7 +34,7 @@
   (when journal-title
     (let [journal-title (common-util/capitalize-all journal-title)]
       (journal-title-> journal-title
-                       #(parse-long (tf/unparse (tf/formatter "yyyyMMdd") %))
+                       #(parse-long (tf/unparse yyyyMMdd-formatter %))
                        formatters))))
 
 (defn format
@@ -51,7 +53,7 @@
 (defn int->journal-title
   [day date-formatter]
   (when day
-    (format (tf/parse (tf/formatter "yyyyMMdd") (str day)) date-formatter)))
+    (format (tf/parse yyyyMMdd-formatter (str day)) date-formatter)))
 
 (defn- get-weekday
   [date]
@@ -94,7 +96,7 @@
   "Converts a journal's :block/journal-day integer into milliseconds"
   [day]
   (when day
-    (-> (tf/parse (tf/formatter "yyyyMMdd") (str day))
+    (-> (tf/parse yyyyMMdd-formatter (str day))
         (tc/to-long))))
 
 (defn ms->journal-day
@@ -103,5 +105,5 @@
   (some->> ms
            tc/from-long
            t/to-default-time-zone
-           (tf/unparse (tf/formatter "yyyyMMdd"))
-           parse-long))
+           (tf/unparse yyyyMMdd-formatter)
+           parse-long))

deps/common/yarn.lock

@@ -2,9 +2,9 @@
 # yarn lockfile v1
 
 
-"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v26":
-  version "1.2.173-feat-db-v25"
-  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/b26d290944234b20762ff109e5328b87ea240692"
+"@logseq/nbb-logseq@github:logseq/nbb-logseq#feat-db-v28":
+  version "1.2.173-feat-db-v28"
+  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/c0410ff81a8b0d510705581cccd39788d862dc91"
   dependencies:
     import-meta-resolve "^4.1.0"
 

deps/db/.carve/ignore

@@ -44,3 +44,5 @@ logseq.db.sqlite.gc/gc-kvs-table!
 logseq.db.sqlite.gc/gc-kvs-table-node-version!
 ;; API
 logseq.db.sqlite.gc/ensure-no-garbage
+;; documenting keywords
+logseq.db.frontend.kv-entity/kv-entities

deps/db/bb.edn

@@ -29,7 +29,7 @@
    :doc "Lint datalog rules for parsability and unbound variables"
    :task (datalog/lint-rules
           (set
-           (concat (mapcat val (merge file-rules/rules rules/rules))
+           (concat (mapcat val (merge file-rules/rules (dissoc rules/rules :self-ref)))
                    ;; TODO: Update linter to handle false positive on ?str-val for :property
                    (rules/extract-rules (dissoc file-rules/query-dsl-rules :property))
                    ;; TODO: Update linter to handle false positive on :task, :priority, :*property* rules

deps/db/deps.edn

@@ -1,7 +1,7 @@
 {:deps
  ;; These nbb-logseq deps are kept in sync with https://github.com/logseq/nbb-logseq/blob/main/bb.edn
  {datascript/datascript {:git/url "https://github.com/logseq/datascript" ;; fork
-                         :sha     "b28f6574b9447bba9ccaa5d2b0cfd79308acf0e3"}
+                         :sha     "45f6721bf2038c24eb9fe3afb422322ab3f473b5"}
   datascript-transit/datascript-transit {:mvn/version "0.3.0"
                                          :exclusions [datascript/datascript]}
   cljs-bean/cljs-bean         {:mvn/version "1.5.0"}

deps/db/package.json

@@ -3,7 +3,7 @@
   "version": "1.0.0",
   "private": true,
   "devDependencies": {
-    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v26"
+    "@logseq/nbb-logseq": "github:logseq/nbb-logseq#feat-db-v28"
   },
   "dependencies": {
     "better-sqlite3": "11.10.0"

deps/db/src/logseq/db.cljs

@@ -136,8 +136,8 @@
              :as opts}]
   (if-let [children (sort-by-order
                      (if include-property-block?
-                       (:block/_raw-parent entity)
-                       (:block/_parent entity)))]
+                       (entity-plus/lookup-kv-then-entity entity :block/_raw-parent)
+                       (entity-plus/lookup-kv-then-entity entity :block/_parent)))]
     (cons entity (mapcat #(get-block-and-children-aux % opts) children))
     [entity]))
 
@@ -218,7 +218,7 @@
 (defn page-exists?
   "Returns truthy value if page exists.
    For db graphs, returns all page db ids that given title and one of the given `tags`.
-   For file graphs, returns page entity if it exists"
+   For file graphs, returns page db/id vector if it exists"
   [db page-name tags]
   (when page-name
     (if (db-based-graph? db)
@@ -249,7 +249,8 @@
             db
             (common-util/page-name-sanity-lc page-name)
             tags'))))
-      (d/entity db [:block/name (common-util/page-name-sanity-lc page-name)]))))
+      (when-let [id (:db/id (d/entity db [:block/name (common-util/page-name-sanity-lc page-name)]))]
+        [id]))))
 
 (defn get-page
   "Get a page given its unsanitized name or uuid"

deps/db/src/logseq/db/common/delete_blocks.cljs

@@ -39,8 +39,7 @@
              tx (cond->
                  (mapcat
                   (fn [block]
-                    [[:db/retract (:db/id ref) :block/refs (:db/id block)]
-                     [:db/retract (:db/id ref) :block/path-refs (:db/id block)]]) retracted-blocks)
+                    [[:db/retract (:db/id ref) :block/refs (:db/id block)]]) retracted-blocks)
                   replaced-title
                   (conj [:db/add id :block/title replaced-title]))]
          tx))

deps/db/src/logseq/db/common/entity_plus.cljc

@@ -33,7 +33,7 @@
   it means `(db/entity :block/title)` always return same result"
   #{:block/link :block/updated-at :block/refs :block/closed-value-property
     :block/created-at :block/collapsed? :block/tags :block/title
-    :block/path-refs :block/parent :block/order :block/page
+    :block/parent :block/order :block/page
 
     :logseq.property/created-from-property
     :logseq.property/icon

deps/db/src/logseq/db/common/initial_data.cljs

@@ -141,8 +141,7 @@
             identity
             (fn [e]
               (keep (fn [[k v]]
-                      (when (and (not (contains? #{:block/path-refs} k))
-                                 (or (empty? properties) (properties k)))
+                      (when (or (empty? properties) (properties k))
                         (let [v' (cond
                                    (= k :block/parent)
                                    (:db/id v)
@@ -216,7 +215,8 @@
                   nil))
         block-refs-count? (some #{:block.temp/refs-count} properties)]
     (when block
-      ;; (prn :debug :get-block (:db/id block) (:block/title block) :children? children?)
+      ;; (prn :debug :get-block (:db/id block) (:block/title block) :children? children?
+      ;;      :include-collapsed-children? include-collapsed-children?)
       (let [children (when children?
                        (let [children-blocks (get-block-children db (:block/uuid block) {:include-collapsed-children? include-collapsed-children?})
                              large-page? (>= (count children-blocks) 100)
@@ -243,7 +243,13 @@
                      block-refs-count?
                      (assoc :block.temp/refs-count (get-block-refs-count db (:db/id block)))
                      true
-                     (assoc :block.temp/load-status (if (and children? (empty? properties)) :full :self)))]
+                     (assoc :block.temp/load-status (cond
+                                                      (and children? include-collapsed-children? (empty? properties))
+                                                      :full
+                                                      (and children? (empty? properties))
+                                                      :children
+                                                      :else
+                                                      :self)))]
         (cond->
          {:block block'}
           children?

deps/db/src/logseq/db/common/reference.cljs

@@ -8,7 +8,8 @@
             [logseq.db :as ldb]
             [logseq.db.common.entity-plus :as entity-plus]
             [logseq.db.common.initial-data :as common-initial-data]
-            [logseq.db.frontend.class :as db-class]))
+            [logseq.db.frontend.class :as db-class]
+            [logseq.db.frontend.rules :as rules]))
 
 (defn get-filters
   [db page]
@@ -36,32 +37,39 @@
                (log/error :syntax/filters e)))))))
 
 (defn- build-include-exclude-query
-  [variable includes excludes]
+  [includes excludes]
   (concat
    (for [include includes]
-     [variable :block/path-refs include])
+     (list 'has-ref '?b include))
    (for [exclude excludes]
-     (list 'not [variable :block/path-refs exclude]))))
+     (list 'not (list 'has-ref '?b exclude)))))
 
 (defn- filter-refs-query
-  [attribute includes excludes class-ids]
+  [includes excludes class-ids]
   (let [clauses (concat
-                 (build-include-exclude-query '?b includes excludes)
+                 (build-include-exclude-query includes excludes)
                  (for [class-id class-ids]
                    (list 'not ['?b :block/tags class-id])))]
     (into [:find '[?b ...]
-           :in '$ '[?id ...]
+           :in '$ '% '[?id ...]
            :where
-           ['?b attribute '?id]]
+           (list 'has-ref '?b '?id)]
           clauses)))
 
+(defn- get-path-refs
+  [db entity]
+  (let [refs (mapcat :block/refs (ldb/get-block-parents db (:block/uuid entity)))
+        block-page (:block/page entity)]
+    (->> (cond->> refs (some? block-page) (cons block-page))
+         distinct)))
+
 (defn- get-ref-pages-count
   [db id ref-blocks children-ids]
   (when (seq ref-blocks)
     (let [children (->> children-ids
                         (map (fn [id] (d/entity db id))))]
-      (->> (concat (mapcat :block/path-refs ref-blocks)
-                   (mapcat :block/refs children))
+      (->> (concat (mapcat #(get-path-refs db %) ref-blocks)
+                   (mapcat :block/refs (concat ref-blocks children)))
            frequencies
            (keep (fn [[ref size]]
                    (when (and (ldb/page? ref)
@@ -99,7 +107,12 @@
                       (set (conj class-children id))))
         full-ref-block-ids (->> (mapcat (fn [id] (map :db/id (:block/_refs (d/entity db id)))) ids)
                                 set)
-        matched-ref-block-ids (set (d/q (filter-refs-query :block/path-refs includes excludes class-ids) db ids))
+        matched-ref-block-ids (set (d/q (filter-refs-query includes excludes class-ids)
+                                        db
+                                        (rules/extract-rules rules/db-query-dsl-rules
+                                                             [:has-ref]
+                                                             {:deps rules/rules-dependencies})
+                                        ids))
         matched-refs-with-children-ids (let [*result (atom #{})]
                                          (doseq [ref-id matched-ref-block-ids]
                                            (get-block-parents-until-top-ref db id ref-id full-ref-block-ids *result))

deps/db/src/logseq/db/common/view.cljs

@@ -7,6 +7,7 @@
             [logseq.common.util :as common-util]
             [logseq.db :as ldb]
             [logseq.db.common.entity-plus :as entity-plus]
+            [logseq.db.common.initial-data :as common-initial-data]
             [logseq.db.common.reference :as db-reference]
             [logseq.db.frontend.class :as db-class]
             [logseq.db.frontend.entity-util :as entity-util]
@@ -301,7 +302,7 @@
                               (entity-util/built-in? e)))
                 (cond-> e
                   refs-count?
-                  (assoc :block.temp/refs-count (count (:block/_refs e)))))))
+                  (assoc :block.temp/refs-count (common-initial-data/get-block-refs-count db (:e d)))))))
           (d/datoms db :avet property-ident))))
 
 (defn- get-entities
@@ -450,8 +451,13 @@
           filters (or (:logseq.property.table/filters view) filters)
           feat-type (or view-feature-type (:logseq.property.view/feature-type view))
           query? (= feat-type :query-result)
+          query-entity-ids (when (seq query-entity-ids) (set query-entity-ids))
           entities-result (if query?
-                            (keep #(d/entity db %) query-entity-ids)
+                            (keep (fn [id]
+                                    (let [e (d/entity db id)]
+                                      (when-not (= :logseq.property/query (:db/ident (:logseq.property/created-from-property e)))
+                                        e)))
+                                  query-entity-ids)
                             (get-view-entities db view-id opts))
           entities (if (= feat-type :linked-references)
                      (:ref-blocks entities-result)

deps/db/src/logseq/db/file_based/rules.cljc

@@ -14,13 +14,7 @@
   "Rules used by frontend.db.query-dsl for file graphs. The symbols ?b and ?p
   respectively refer to block and page. Do not alter them as they are
   programmatically built by the query-dsl ns"
-  {:block-parent
-   '[[(block-parent ?p ?c)
-      [?c :block/parent ?p]]
-     [(block-parent ?p ?c)
-      [?t :block/parent ?p]
-      (block-parent ?t ?c)]]
-   :page-property
+  {:page-property
    '[(page-property ?p ?key ?val)
      [?p :block/name]
      [?p :block/properties ?prop]
@@ -95,5 +89,5 @@
 
    :page-ref
    '[(page-ref ?b ?page-name)
-     [?b :block/path-refs ?br]
-     [?br :block/name ?page-name]]})
+     [?br :block/name ?page-name]
+     (has-ref ?b ?br)]})

deps/db/src/logseq/db/file_based/schema.cljs

@@ -28,10 +28,6 @@
    ;; reference blocks
    :block/refs {:db/valueType :db.type/ref
                 :db/cardinality :db.cardinality/many}
-   ;; referenced pages inherited from the parents
-   :block/path-refs {:db/valueType   :db.type/ref
-                     :db/cardinality :db.cardinality/many}
-
    :block/tags {:db/valueType :db.type/ref
                 :db/cardinality :db.cardinality/many}
 

deps/db/src/logseq/db/frontend/class.cljs

@@ -136,9 +136,9 @@
 
 (defn get-class-extends
   "Returns all extends of a class"
-  [node]
-  (assert (de/entity? node) "get-class-extends `node` should be an entity")
-  (loop [extends (:logseq.property.class/extends node)
+  [class]
+  (assert (de/entity? class) "get-class-extends `class` should be an entity")
+  (loop [extends (:logseq.property.class/extends class)
          result #{}]
     (if (seq extends)
       (recur (set (mapcat :logseq.property.class/extends extends))

deps/db/src/logseq/db/frontend/content.cljs

@@ -113,12 +113,12 @@
                  :or {replace-tag? true}}]
   (assert (string? title))
   (let [refs' (->> refs
-                   (remove (fn [ref]
-                             ;; remove uuid references since they're introduced to detect multiple pages
-                             ;; that have the same name
-                             (and (map? ref)
-                                  (:block.temp/original-page-name ref)
-                                  (common-util/uuid-string? (:block.temp/original-page-name ref)))))
+                   (map (fn [ref]
+                          ;; remove uuid references since they're introduced to detect multiple pages
+                          ;; that have the same name
+                          (if (and (map? ref) (some-> (:block.temp/original-page-name ref) common-util/uuid-string?))
+                            (dissoc ref :block.temp/original-page-name)
+                            ref)))
                    (map
                     (fn [ref]
                       (if (and (vector? ref) (= :block/uuid (first ref)))
@@ -127,9 +127,8 @@
                         ref)))
                    sort-refs)]
     (reduce
-     (fn [content {uuid' :block/uuid :block/keys [title] :as block}]
-       (let [title' (or (:block.temp/original-page-name block) title)]
-         (replace-page-ref-with-id content title' uuid' replace-tag?)))
+     (fn [content {uuid' :block/uuid :block/keys [title]}]
+       (replace-page-ref-with-id content title uuid' replace-tag?))
      title
      (filter :block/title refs'))))
 

deps/db/src/logseq/db/frontend/kv_entity.cljs

@@ -2,35 +2,38 @@
   "kv entities used by logseq db"
   (:require [logseq.common.defkeywords :refer [defkeywords]]))
 
-(defkeywords
-  :logseq.kv/db-type                      {:doc "Set to \"db\" if it's a db-graph"}
-  :logseq.kv/graph-uuid                   {:doc "Store graph-uuid if it's a rtc enabled graph"
-                                           :rtc {:rtc/ignore-entity-when-init-upload true
-                                                 :rtc/ignore-entity-when-init-download true}}
-  :logseq.kv/import-type                  {:doc "If graph is imported, identifies how a graph is imported including which UI or CLI import process. CLI scripts can set this to a custom value.
+(def kv-entities
+  (apply
+   hash-map
+   (defkeywords
+     :logseq.kv/db-type                      {:doc "Set to \"db\" if it's a db-graph"}
+     :logseq.kv/graph-uuid                   {:doc "Store graph-uuid if it's a rtc enabled graph"
+                                              :rtc {:rtc/ignore-entity-when-init-upload true
+                                                    :rtc/ignore-entity-when-init-download true}}
+     :logseq.kv/import-type                  {:doc "If graph is imported, identifies how a graph is imported including which UI or CLI import process. CLI scripts can set this to a custom value.
                                                  UI values include :file-graph and :sqlite-db and CLI values start with :cli e.g. :cli/default."}
-  :logseq.kv/imported-at                  {:doc "Time if graph is imported"}
-  :logseq.kv/graph-local-tx               {:doc "local rtc tx-id"
-                                           :rtc {:rtc/ignore-entity-when-init-upload true
-                                                 :rtc/ignore-entity-when-init-download true}}
-  :logseq.kv/schema-version               {:doc "Graph's current schema version"}
-  :logseq.kv/remote-schema-version        {:doc "Graph's remote schema version.
+     :logseq.kv/imported-at                  {:doc "Time if graph is imported"}
+     :logseq.kv/graph-local-tx               {:doc "local rtc tx-id"
+                                              :rtc {:rtc/ignore-entity-when-init-upload true
+                                                    :rtc/ignore-entity-when-init-download true}}
+     :logseq.kv/schema-version               {:doc "Graph's current schema version"}
+     :logseq.kv/remote-schema-version        {:doc "Graph's remote schema version.
 RTC won't start when major-schema-versions don't match"
-                                           :rtc {:rtc/ignore-entity-when-init-upload true
-                                                 :rtc/ignore-entity-when-init-download true}}
-  :logseq.kv/graph-created-at             {:doc "Graph's created at time"}
-  :logseq.kv/latest-code-lang             {:doc "Latest lang used by a #Code-block"
-                                           :rtc {:rtc/ignore-entity-when-init-upload true
-                                                 :rtc/ignore-entity-when-init-download true}}
-  :logseq.kv/graph-backup-folder          {:doc "Backup folder for automated backup feature"
-                                           :rtc {:rtc/ignore-entity-when-init-upload true
-                                                 :rtc/ignore-entity-when-init-download true}}
-  :logseq.kv/graph-initial-schema-version {:doc "Graph's schema version when created"}
+                                              :rtc {:rtc/ignore-entity-when-init-upload true
+                                                    :rtc/ignore-entity-when-init-download true}}
+     :logseq.kv/graph-created-at             {:doc "Graph's created at time"}
+     :logseq.kv/latest-code-lang             {:doc "Latest lang used by a #Code-block"
+                                              :rtc {:rtc/ignore-entity-when-init-upload true
+                                                    :rtc/ignore-entity-when-init-download true}}
+     :logseq.kv/graph-backup-folder          {:doc "Backup folder for automated backup feature"
+                                              :rtc {:rtc/ignore-entity-when-init-upload true
+                                                    :rtc/ignore-entity-when-init-download true}}
+     :logseq.kv/graph-initial-schema-version {:doc "Graph's schema version when created"}
 
-  :logseq.kv/graph-last-gc-at             {:doc "Last time graph gc at"
-                                           :rtc {:rtc/ignore-entity-when-init-upload true
-                                                 :rtc/ignore-entity-when-init-download true}}
+     :logseq.kv/graph-last-gc-at             {:doc "Last time graph gc at"
+                                              :rtc {:rtc/ignore-entity-when-init-upload true
+                                                    :rtc/ignore-entity-when-init-download true}}
 
-  :logseq.kv/graph-text-embedding-model-name   {:doc "Graph's text-embedding model name"
-                                                :rtc {:rtc/ignore-entity-when-init-upload true
-                                                      :rtc/ignore-entity-when-init-download true}})
+     :logseq.kv/graph-text-embedding-model-name   {:doc "Graph's text-embedding model name"
+                                                   :rtc {:rtc/ignore-entity-when-init-upload true
+                                                         :rtc/ignore-entity-when-init-download true}})))

deps/db/src/logseq/db/frontend/malli_schema.cljs

@@ -273,8 +273,7 @@
 (def page-attrs
   "Common attributes for pages"
   [[:block/name :string]
-   [:block/title :string]
-   [:block/path-refs {:optional true} [:set :int]]])
+   [:block/title :string]])
 
 (def property-attrs
   "Common attributes for properties"
@@ -335,7 +334,8 @@
    (concat
     [:map
      [:db/ident user-property-ident]
-     [:logseq.property/type (apply vector :enum db-property-type/user-built-in-property-types)]]
+     [:logseq.property/type (apply vector :enum (into db-property-type/user-allowed-internal-property-types
+                                                      db-property-type/user-built-in-property-types))]]
     property-common-schema-attrs
     property-attrs
     page-attrs
@@ -387,7 +387,6 @@
    [:block/order block-order]
    ;; refs
    [:block/page :int]
-   [:block/path-refs {:optional true} [:set :int]]
    [:block/link {:optional true} :int]
    [:logseq.property/created-from-property {:optional true} :int]])
 
@@ -399,8 +398,7 @@
     [[:block/title :string]
      [:block/parent :int]
      ;; These blocks only associate with pages of type "whiteboard"
-     [:block/page :int]
-     [:block/path-refs {:optional true} [:set :int]]]
+     [:block/page :int]]
     page-or-block-attrs)))
 
 (def property-value-block

deps/db/src/logseq/db/frontend/property.cljs

@@ -9,17 +9,14 @@
             [logseq.db.frontend.db-ident :as db-ident]
             [logseq.db.frontend.property.type :as db-property-type]))
 
-;; Main property vars
-;; ==================
-
-;; Ignore this property when rtc,
-;; since users frequently click the sort button to view table content temporarily,
-;; but this action does not need to be synchronized with other clients.
-(def property-ignore-rtc
+(def ^:private property-ignore-rtc
   {:rtc/ignore-attr-when-init-upload true
    :rtc/ignore-attr-when-init-download true
    :rtc/ignore-attr-when-syncing true})
 
+;; Main property vars
+;; ==================
+
 (def ^:large-vars/data-var built-in-properties
   "Map of built in properties for db graphs with their :db/ident as keys.
    Each property has a config map with the following keys:
@@ -109,12 +106,6 @@
                                      :cardinality :many
                                      :public? false
                                      :hide? true}}
-     :block/path-refs      {:title "Node path references"
-                            :attribute :block/path-refs
-                            :schema {:type :entity
-                                     :cardinality :many
-                                     :public? false
-                                     :hide? true}}
      :block/link           {:title "Node links to"
                             :attribute :block/link
                             :schema {:type :entity
@@ -209,7 +200,7 @@
                                  {:logseq.property/description "Provides a way for a page to associate to another page i.e. backward compatible tagging."}}
      :logseq.property/background-color {:title "Background color"
                                         :schema {:type :default :hide? true}}
-   ;; number (1-6) or boolean for auto heading
+     ;; number (1-6) or boolean for auto heading
      :logseq.property/heading {:title "Heading"
                                :schema {:type :any :hide? true}
                                :queryable? true}
@@ -222,8 +213,8 @@
      :logseq.property/asset   {:title "Asset"
                                :schema {:type :entity
                                         :hide? true}}
-   ;; used by pdf and whiteboard
-   ;; TODO: remove ls-type
+     ;; used by pdf and whiteboard
+     ;; TODO: remove ls-type
      :logseq.property/ls-type {:schema {:type :keyword
                                         :hide? true}}
 
@@ -248,7 +239,7 @@
                                     :schema {:type :entity :hide? true}}
      :logseq.property.pdf/hl-value {:title "Annotation data"
                                     :schema {:type :map :hide? true}}
-   ;; FIXME: :logseq.property/order-list-type should updated to closed values
+     ;; FIXME: :logseq.property/order-list-type should updated to closed values
      :logseq.property/order-list-type {:title "List type"
                                        :schema {:type :default
                                                 :hide? true}}
@@ -268,7 +259,7 @@
                                     :schema {:type :map
                                              :hide? true}}
 
-   ;; Journal props
+     ;; Journal props
      :logseq.property.journal/title-format {:title "Title Format"
                                             :schema
                                             {:type :string
@@ -383,7 +374,7 @@
       :schema {:type :property
                :hide? true}}
 
-;; TODO: Add more props :Assignee, :Estimate, :Cycle, :Project
+     ;; TODO: Add more props :Assignee, :Estimate, :Cycle, :Project
 
      :logseq.property/icon {:title "Icon"
                             :schema {:type :map}}
@@ -580,13 +571,14 @@
                                                        :schema {:type :datetime
                                                                 :public? false
                                                                 :hide? true}
+                                                       :queryable? false
                                                        :rtc property-ignore-rtc})))
 
 (def db-attribute-properties
   "Internal properties that are also db schema attributes"
   #{:block/alias :block/tags :block/parent
     :block/order :block/collapsed? :block/page
-    :block/refs :block/path-refs :block/link
+    :block/refs :block/link
     :block/title :block/closed-value-property :block/journal-day
     :block/created-at :block/updated-at})
 

deps/db/src/logseq/db/frontend/property/type.cljs

@@ -20,6 +20,13 @@
   "Valid property types for users in order they appear in the UI"
   [:default :number :date :datetime :checkbox :url :node])
 
+(def user-allowed-internal-property-types
+  "Internal property types that users are allowed to store. These aren't available in the UI
+   so these would normally be created via EDN or the API."
+  #{:map})
+
+(assert (set/subset? user-allowed-internal-property-types internal-built-in-property-types))
+
 (def closed-value-property-types
   "Valid property :type for closed values"
   #{:default :number :url})

deps/db/src/logseq/db/frontend/rules.cljc

@@ -30,7 +30,19 @@
       [?e2 :block/alias ?e3]]
      [(alias ?e3 ?e1)
       [?e1 :block/alias ?e2]
-      [?e2 :block/alias ?e3]]]})
+      [?e2 :block/alias ?e3]]]
+
+   :self-ref
+   '[(self-ref ?b ?page-name)
+     [?br :block/name ?page-name]
+     [?b :block/refs ?br]]
+
+   :has-ref
+   '[[(has-ref ?b ?r)
+      [?b :block/refs ?r]]
+     [(has-ref ?b ?r)
+      (parent ?p ?b)
+      [?p :block/refs ?r]]]})
 
 ;; Rules writing advice
 ;; ====================
@@ -239,7 +251,9 @@
   "For db graphs, a map of rule names and the rules they depend on. If this map
   becomes long or brittle, we could do scan rules for their deps with something
   like find-rules-in-where"
-  {:task #{:simple-query-property}
+  {:has-ref #{:parent}
+   :page-ref #{:has-ref}
+   :task #{:simple-query-property}
    :priority #{:simple-query-property}
    :property-missing-value #{:object-has-class-property}
    :has-property-or-object-property #{:object-has-class-property}

deps/db/src/logseq/db/frontend/schema.cljs

@@ -37,7 +37,7 @@
          (map (juxt :major :minor)
               [(parse-schema-version x) (parse-schema-version y)])))
 
-(def version (parse-schema-version "65.10"))
+(def version (parse-schema-version "65.11"))
 
 (defn major-version
   "Return a number.

deps/db/test/logseq/db/sqlite/export_test.cljs

@@ -428,7 +428,8 @@
                       :user.property/node {:logseq.property/type :node
                                            :db/cardinality :db.cardinality/many
                                            :build/property-classes [:user.class/MyClass]}
-                      :user.property/p1 {:logseq.property/type :default}}
+                      :user.property/p1 {:logseq.property/type :default}
+                      :user.property/map {:logseq.property/type :map}}
          :classes {:user.class/MyClass {:build/class-properties [:user.property/p1]}}
          :pages-and-blocks
          [{:page {:block/title "page1"}
@@ -442,7 +443,9 @@
                      :build/properties {:user.property/node #{[:build/page {:block/title "page object"
                                                                             :build/tags [:user.class/MyClass]}]
                                                               [:block/uuid block-object-uuid]
-                                                              :logseq.class/Task}}}]}
+                                                              :logseq.class/Task}}}
+                    {:block/title "map block"
+                     :build/properties {:user.property/map {:foo :bar :num 2}}}]}
           {:page {:block/title "Blocks"}
            :blocks [{:block/title "myclass object"
                      :build/tags [:user.class/MyClass]

deps/db/yarn.lock

@@ -2,9 +2,9 @@
 # yarn lockfile v1
 
 
-"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v26":
-  version "1.2.173-feat-db-v25"
-  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/b26d290944234b20762ff109e5328b87ea240692"
+"@logseq/nbb-logseq@github:logseq/nbb-logseq#feat-db-v28":
+  version "1.2.173-feat-db-v28"
+  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/c0410ff81a8b0d510705581cccd39788d862dc91"
   dependencies:
     import-meta-resolve "^4.1.0"
 

deps/graph-parser/package.json

@@ -3,7 +3,7 @@
   "version": "1.0.0",
   "private": true,
   "devDependencies": {
-    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v26",
+    "@logseq/nbb-logseq": "github:logseq/nbb-logseq#feat-db-v28",
     "better-sqlite3": "11.10.0"
   },
   "dependencies": {

deps/graph-parser/src/logseq/graph_parser/exporter.cljs

@@ -800,6 +800,32 @@
         (pr-str (dissoc query-map :title :group-by-page? :collapsed?))
         query-str))))
 
+(declare extract-block-list ast->text)
+(defn- extract-block-list-item
+  [{:keys [content items number checkbox]}]
+  (let [content* (mapcat #(ast->text % {}) content)
+        number* (if number
+                  (str number ". ")
+                  "* ")
+        checkbox* (if (some? checkbox)
+                    (if (boolean checkbox)
+                      "[X]" "[ ]")
+                    "")
+        items* (extract-block-list items :in-list? true)]
+    (concat [number* checkbox* " "]
+            content*
+            ["\n"]
+            items*
+            (when (seq items*) ["\n"]))))
+
+(defn- extract-block-list
+  [l & {:keys [in-list?]}]
+  (vec (concat (when-not in-list? ["\n"])
+               (mapcat extract-block-list-item l)
+               (when (and (pos? (count l))
+                          (not in-list?))
+                 ["\n\n"]))))
+
 (defn- ast->text
   "Given an ast block, convert it to text for use as a block title. This is a
   slimmer version of handler.export.text/export-blocks-as-markdown"
@@ -841,6 +867,14 @@
               (:arguments (second node))
               (and (vector? node) (= (first node) "Example"))
               (second node)
+              (and (vector? node) (= (first node) "Latex_Fragment"))
+              (let [[type' content] (second node)
+                    wrapper (case type' "Inline" "$" "Displayed" "$$")]
+                [wrapper content wrapper])
+              (and (vector? node) (= (first node) "Displayed_Math"))
+              ["$$" (second node) "$$"]
+              (and (vector? node) (= (first node) "List"))
+              (extract-block-list (second node))
               :else
               (do
                 (log-fn :ast->text "Ignored ast node" :node node)

deps/graph-parser/src/logseq/graph_parser/whiteboard.cljs

@@ -47,12 +47,9 @@
     (concat portal-refs shape-link-refs)))
 
 (defn- with-whiteboard-block-refs
-  [shape page-id]
+  [shape]
   (let [refs (or (get-shape-refs shape) [])]
-    (merge {:block/refs (if (seq refs) refs [])
-            :block/path-refs (if (seq refs)
-                               (conj refs page-id)
-                               [])})))
+    {:block/refs (if (seq refs) refs [])}))
 
 (defn- with-whiteboard-content
   "Main purpose of this function is to populate contents when shapes are used as references in outliner."
@@ -72,7 +69,7 @@
     (merge (when shape?
              (merge
               {:block/uuid (uuid (:id shape))}
-              (with-whiteboard-block-refs shape page-id)
+              (with-whiteboard-block-refs shape)
               (with-whiteboard-content shape)))
            (when (nil? (:block/parent block)) {:block/parent page-id})
            (when (nil? (:block/format block)) {:block/format :markdown}) ;; TODO: read from config

deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs

@@ -196,7 +196,7 @@
   ;; This graph will contain basic examples of different features to import
   (p/let [file-graph-dir "test/resources/exporter-test-graph"
           conn (db-test/create-conn)
-          ;; Calculate refs and path-refs like frontend
+          ;; Calculate refs like frontend
           _ (db-pipeline/add-listener conn)
           assets (atom [])
           {:keys [import-state]} (import-file-graph-to-db file-graph-dir conn {:assets assets :convert-all-tags? true})]
@@ -214,7 +214,7 @@
       (is (= 4 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Task]] @conn))))
       (is (= 4 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Query]] @conn))))
       (is (= 2 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Card]] @conn))))
-      (is (= 3 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Quote-block]] @conn))))
+      (is (= 4 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Quote-block]] @conn))))
       (is (= 2 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Pdf-annotation]] @conn))))
 
       ;; Properties and tags aren't included in this count as they aren't a Page
@@ -587,16 +587,12 @@
       (is (= "multiline block\na 2nd\nand a 3rd" (:block/title (db-test/find-block-by-content @conn #"multiline block"))))
       (is (= "logbook block" (:block/title (db-test/find-block-by-content @conn #"logbook block")))))
 
-    (testing ":block/refs and :block/path-refs"
+    (testing ":block/refs"
       (let [page (db-test/find-page-by-title @conn "chat-gpt")]
         (is (set/subset?
              #{"type" "LargeLanguageModel"}
              (->> page :block/refs (map #(:block/title (d/entity @conn (:db/id %)))) set))
-            "Page has correct property and property value :block/refs")
-        (is (set/subset?
-             #{"type" "LargeLanguageModel"}
-             (->> page :block/path-refs (map #(:block/title (d/entity @conn (:db/id %)))) set))
-            "Page has correct property and property value :block/path-refs"))
+            "Page has correct property and property value :block/refs"))
 
       (let [block (db-test/find-block-by-content @conn "old todo block")]
         (is (set/subset?
@@ -605,14 +601,7 @@
                   :block/refs
                   (map #(:db/ident (d/entity @conn (:db/id %))))
                   set))
-            "Block has correct task tag and property :block/refs")
-        (is (set/subset?
-             #{:logseq.property/status :logseq.class/Task}
-             (->> block
-                  :block/path-refs
-                  (map #(:db/ident (d/entity @conn (:db/id %))))
-                  set))
-            "Block has correct task tag and property :block/path-refs")))
+            "Block has correct task tag and property :block/refs")))
 
     (testing "whiteboards"
       (let [block-with-props (db-test/find-block-by-content @conn #"block with props")]