Merge remote-tracking branch 'upstream/master' into feat/tldraw-basic

.carve/config.edn

@@ -5,5 +5,7 @@
                   ;; Ignore b/c too many false positives
                   frontend.db
                   ;; Used for debugging
-                  frontend.db.debug]
+                  frontend.db.debug
+                  ;; carve doesn't detect nbb only usage
+                  logseq.graph-parser.log]
  :report {:format :ignore}}

.carve/ignore

@@ -31,7 +31,7 @@ frontend.extensions.zotero.api/item
 ;; For repl
 frontend.external.roam/reset-state!
 ;; For repl
-frontend.format.mldoc/ast-export-markdown
+logseq.graph-parser.mldoc/ast-export-markdown
 ;; Protocol fn wrapper that could be used
 frontend.fs/readdir
 ;; Referenced in TODO
@@ -74,5 +74,7 @@ frontend.util/trace!
 frontend.util.pool/terminate-pool!
 ;; Repl fn
 frontend.util.property/add-page-properties
-;; Used by shadow
+;; Test runner used by shadow
 frontend.test.node-test-runner/main
+;; Test runner for nbb
+logseq.graph-parser.nbb-test-runner/run-tests

.clj-kondo/config.edn

@@ -19,9 +19,18 @@
              frontend.db.react react
              frontend.db.query-react query-react
              frontend.util util
+             frontend.util.property property
              frontend.config config
+             frontend.format.mldoc mldoc
+             frontend.format.block block
+             frontend.handler.extract extract
+             logseq.graph-parser.text text
+             logseq.graph-parser.block gp-block
+             logseq.graph-parser.mldoc gp-mldoc
              logseq.graph-parser.util gp-util
-             logseq.graph-parser.config gp-config}}}
+             logseq.graph-parser.property gp-property
+             logseq.graph-parser.config gp-config
+             logseq.graph-parser.date-time-util date-time-util}}}
 
  :hooks {:analyze-call {rum.core/defc hooks.rum/defc
                          rum.core/defcs hooks.rum/defcs}}

.github/workflows/build.yml

@@ -74,11 +74,13 @@ jobs:
       - name: Fetch yarn deps
         run: yarn install --frozen-lockfile
 
-      - name: Run ClojureScript test
+      - name: Run ClojureScript tests
         run: |
           yarn cljs:test
           node static/tests.js
 
+      - name: Run nbb tests for graph-parser
+        run: yarn nbb-logseq -cp src/main:src/test -m logseq.graph-parser.nbb-test-runner/run-tests
       # In this job because it depends on an npm package
       - name: Load nbb compatible namespaces
         run: bb test:load-nbb-compatible-namespaces

android/app/capacitor.build.gradle

@@ -13,6 +13,7 @@ dependencies {
     implementation project(':capacitor-camera')
     implementation project(':capacitor-filesystem')
     implementation project(':capacitor-keyboard')
+    implementation project(':capacitor-share')
     implementation project(':capacitor-splash-screen')
     implementation project(':capacitor-status-bar')
     implementation project(':capacitor-voice-recorder')

android/app/src/main/assets/capacitor.plugins.json

@@ -15,6 +15,10 @@
 		"pkg": "@capacitor/keyboard",
 		"classpath": "com.capacitorjs.plugins.keyboard.KeyboardPlugin"
 	},
+	{
+		"pkg": "@capacitor/share",
+		"classpath": "com.capacitorjs.plugins.share.SharePlugin"
+	},
 	{
 		"pkg": "@capacitor/splash-screen",
 		"classpath": "com.capacitorjs.plugins.splashscreen.SplashScreenPlugin"

android/capacitor.settings.gradle

@@ -14,6 +14,9 @@ project(':capacitor-filesystem').projectDir = new File('../node_modules/@capacit
 include ':capacitor-keyboard'
 project(':capacitor-keyboard').projectDir = new File('../node_modules/@capacitor/keyboard/android')
 
+include ':capacitor-share'
+project(':capacitor-share').projectDir = new File('../node_modules/@capacitor/share/android')
+
 include ':capacitor-splash-screen'
 project(':capacitor-splash-screen').projectDir = new File('../node_modules/@capacitor/splash-screen/android')
 

docs/dev-practices.md

@@ -101,9 +101,9 @@ For this workflow:
   1. Add `^:focus` metadata flags to tests e.g. `(deftest ^:focus test-name ...)`.
   2. In another shell, run `node static/tests.js -i focus` to only run those
   tests. To run all tests except those tests run `node static/tests.js -e focus`.
-3. Or focus namespaces: Using the regex option `-r`, run tests for `frontend.text-test` with `node static/tests.js -r text`.
+3. Or focus namespaces: Using the regex option `-r`, run tests for `frontend.util.page-property-test` with `node static/tests.js -r page-property`.
 
-Multiple options can be specified to AND selections. For example, to run all `frontend.text-test` tests except for the focused one: `node static/tests.js -r text -e focus`
+Multiple options can be specified to AND selections. For example, to run all `frontend.util.page-property-test` tests except for the focused one: `node static/tests.js -r page-property -e focus`
 
 For help on more options, run `node static/tests.js -h`.
 
@@ -114,7 +114,7 @@ shadow-cljs watch test --config-merge '{:autorun true}'`. The test output may
 appear where shadow-cljs was first invoked e.g. where `yarn watch` is running.
 Specific namespace(s) can be auto run with the `:ns-regexp` option e.g. `npx
 shadow-cljs watch test --config-merge '{:autorun true :ns-regexp
-"frontend.text-test"}'`.
+"frontend.util.page-property-test"}'`.
 
 ## Logging
 

externs.js

@@ -127,6 +127,9 @@ dummy.getNodesObjects = function() {};
 dummy.getEdgesObjects = function() {};
 dummy.alphaTarget = function() {};
 dummy.restart = function() {};
+dummy.observe = function() {};
+dummy.contentRect = function() {};
+dummy.height = function() {};
 
 /**
  * @typedef {{

ios/App/Podfile

@@ -14,6 +14,7 @@ def capacitor_pods
   pod 'CapacitorClipboard', :path => '../../node_modules/@capacitor/clipboard'
   pod 'CapacitorFilesystem', :path => '../../node_modules/@capacitor/filesystem'
   pod 'CapacitorKeyboard', :path => '../../node_modules/@capacitor/keyboard'
+  pod 'CapacitorShare', :path => '../../node_modules/@capacitor/share'
   pod 'CapacitorSplashScreen', :path => '../../node_modules/@capacitor/splash-screen'
   pod 'CapacitorStatusBar', :path => '../../node_modules/@capacitor/status-bar'
   pod 'CapacitorVoiceRecorder', :path => '../../node_modules/capacitor-voice-recorder'

package.json

@@ -5,6 +5,7 @@
     "main": "static/electron.js",
     "devDependencies": {
         "@capacitor/cli": "3.2.2",
+        "@logseq/nbb-logseq": "^0.3.99",
         "@playwright/test": "^1.19.2",
         "@tailwindcss/ui": "0.7.2",
         "@types/gulp": "^4.0.7",
@@ -13,7 +14,6 @@
         "del": "^6.0.0",
         "gulp": "^4.0.2",
         "gulp-clean-css": "^4.3.0",
-        "@logseq/nbb-logseq": "^0.3.10",
         "npm-run-all": "^4.1.5",
         "playwright": "^1.19.2",
         "postcss": "8.2.13",
@@ -72,6 +72,7 @@
         "@capacitor/filesystem": "1.0.6",
         "@capacitor/ios": "3.2.2",
         "@capacitor/keyboard": "^1.2.0",
+        "@capacitor/share": "^1.1.2",
         "@capacitor/splash-screen": "1.1.3",
         "@capacitor/status-bar": "1.0.6",
         "@excalidraw/excalidraw": "0.10.0",
@@ -115,6 +116,7 @@
         "react-textarea-autosize": "8.3.3",
         "react-tippy": "1.4.0",
         "react-transition-group": "4.3.0",
+        "react-visibility-sensor": "^5.1.1",
         "reakit": "0.11.1",
         "remove-accents": "0.4.2",
         "send-intent": "3.0.11",

resources/css/common.css

@@ -715,6 +715,11 @@ li p:last-child,
   background-color: var(--ls-primary-background-color, #fff);
 }
 
+.bg-base-4 {
+  background-color: var(--ls-tertiary-background-color);
+}
+
+
 .pre-white-space {
   white-space: pre;
 }
@@ -1169,3 +1174,8 @@ html[data-theme='dark'] .keyboard-shortcut > code {
     overflow: hidden;
     text-overflow: ellipsis;
 }
+
+.lazy-visibility {
+    min-width: 1px;
+    min-height: 1px;
+}

resources/package.json

@@ -37,7 +37,7 @@
     "@sentry/electron": "2.5.1",
     "posthog-js": "1.10.2",
     "@logseq/rsapi": "0.0.11",
-    "electron-deeplink": "1.0.9"
+    "electron-deeplink": "1.0.10"
   },
   "devDependencies": {
     "@electron-forge/cli": "^6.0.0-beta.57",

src/electron/electron/handler.cljs

@@ -314,6 +314,10 @@
 (defmethod handle :getLogseqDotDirRoot []
   (utils/get-ls-dotdir-root))
 
+(defmethod handle :testProxyUrl [win [_ url]]
+  (p/let [_ (utils/fetch url)]
+    (utils/send-to-renderer win :notification {:type "success" :payload (str "Successfully: " url)})))
+
 (defmethod handle :getUserDefaultPlugins []
   (utils/get-ls-default-plugins))
 

src/electron/electron/utils.cljs

@@ -118,7 +118,7 @@
   ([window kind payload]
    (when window
      (.. ^js window -webContents
-         (send kind (bean/->js payload))))))
+         (send (name kind) (bean/->js payload))))))
 
 (defn get-graph-dir
   [graph-name]

src/main/frontend/commands.cljs

@@ -511,7 +511,7 @@
 
 (defn compute-pos-delta-when-change-marker
   [edit-content marker pos]
-  (let [old-marker (some->> (first (gp-util/safe-re-find marker/bare-marker-pattern edit-content))
+  (let [old-marker (some->> (first (util/safe-re-find marker/bare-marker-pattern edit-content))
                             (string/trim))
         pos-delta (- (count marker)
                      (count old-marker))
@@ -536,7 +536,7 @@
                   (if-let [matches (seq (util/re-pos new-line-re-pattern prefix))]
                     (let [[start-pos content] (last matches)]
                       (+ start-pos (count content)))
-                    (count (gp-util/safe-re-find re-pattern prefix))))
+                    (count (util/safe-re-find re-pattern prefix))))
             new-value (str (subs edit-content 0 pos)
                            (string/replace-first (subs edit-content pos)
                                                  (marker/marker-pattern format)
@@ -577,7 +577,7 @@
       (let [edit-content (gobj/get current-input "value")
             heading-pattern #"^#+\s+"
             new-value (cond
-                        (gp-util/safe-re-find heading-pattern edit-content)
+                        (util/safe-re-find heading-pattern edit-content)
                         (string/replace-first edit-content
                                               heading-pattern
                                               (str heading " "))

src/main/frontend/components/block.cljs

@@ -1,6 +1,7 @@
 (ns frontend.components.block
   (:refer-clojure :exclude [range])
   (:require ["/frontend/utils" :as utils]
+            ["@capacitor/share" :refer [^js Share]]
             [cljs-bean.core :as bean]
             [cljs.core.match :refer [match]]
             [cljs.reader :as reader]
@@ -45,7 +46,7 @@
             [frontend.security :as security]
             [frontend.state :as state]
             [frontend.template :as template]
-            [frontend.text :as text]
+            [logseq.graph-parser.text :as text]
             [frontend.ui :as ui]
             [frontend.util :as util]
             [frontend.util.clock :as clock]
@@ -53,6 +54,8 @@
             [frontend.util.drawer :as drawer]
             [logseq.graph-parser.config :as gp-config]
             [logseq.graph-parser.util :as gp-util]
+            [logseq.graph-parser.mldoc :as gp-mldoc]
+            [logseq.graph-parser.block :as gp-block]
             [goog.dom :as gdom]
             [goog.object :as gobj]
             [lambdaisland.glogi :as log]
@@ -263,12 +266,31 @@
         href (config/get-local-asset-absolute-path href)]
     (when (or granted? (util/electron?) (mobile-util/is-native-platform?))
       (p/then (editor-handler/make-asset-url href) #(reset! src %)))
-
+    
     (when @src
-      (let [ext (util/get-file-ext @src)]
-        (if (contains? (set (map name config/audio-formats)) ext)
+      (let [ext (keyword (util/get-file-ext @src))
+            share-fn (fn [event]
+                       (util/stop event)
+                       (when (mobile-util/is-native-platform?)
+                         (p/let [url (str (config/get-repo-dir (state/get-current-repo)) href)]
+                           (.share Share #js {:url url
+                                              :title "Open file with your favorite app"}))))]
+        (cond
+          (contains? config/audio-formats ext)
           (audio-cp @src)
-          (resizable-image config title @src metadata full_text true))))))
+
+          (contains? (config/img-formats) ext)
+          (resizable-image config title @src metadata full_text true)
+
+          (= ext :pdf)
+          [:a.asset-ref.is-pdf {:href @src
+                                :on-click share-fn}
+           title]
+
+          :else
+          [:a.asset-ref.is-doc {:ref @src
+                                :on-click share-fn}
+           title])))))
 
 (defn ar-url->http-url
   [href]
@@ -436,41 +458,66 @@
 
 (rum/defc page-preview-trigger
   [{:keys [children sidebar? tippy-position tippy-distance fixed-position? open? manual?] :as config} page-name]
-  (let [page-name (util/page-name-sanity-lc page-name)
+  (let [*tippy-ref (rum/create-ref)
+        page-name (util/page-name-sanity-lc page-name)
         redirect-page-name (or (model/get-redirect-page-name page-name (:block/alias? config))
                                page-name)
         page-original-name (model/get-page-original-name redirect-page-name)
-        html-template (fn []
-                        (when redirect-page-name
-                          [:div.tippy-wrapper.overflow-y-auto.p-4
-                           {:style {:width          600
-                                    :text-align     "left"
-                                    :font-weight    500
-                                    :max-height     600
-                                    :padding-bottom 64}}
-                           (if (and (string? page-original-name) (string/includes? page-original-name "/"))
-                             [:div.my-2
-                              (->>
-                               (for [page (string/split page-original-name #"/")]
-                                 (when (and (string? page) page)
-                                   (page-reference false page {} nil)))
-                               (interpose [:span.mx-2.opacity-30 "/"]))]
-                             [:h2.font-bold.text-lg (if (= page-name redirect-page-name)
-                                                      page-original-name
-                                                      [:span
-                                                       [:span.text-sm.mr-2 "Alias:"]
-                                                       page-original-name])])
-                           (let [page (db/entity [:block/name (util/page-name-sanity-lc redirect-page-name)])]
-                             (editor-handler/insert-first-page-block-if-not-exists! redirect-page-name {:redirect? false})
-                             (when-let [f (state/get-page-blocks-cp)]
-                               (f (state/get-current-repo) page {:sidebar? sidebar? :preview? true})))]))]
+        _  #_:clj-kondo/ignore (rum/defc html-template []
+                        (let [*el-popup (rum/use-ref nil)]
+
+                          (rum/use-effect!
+                            (fn []
+                              (let [el-popup (rum/deref *el-popup)
+                                    cb (fn [^js e]
+                                         (when-not (:editor/editing? @state/state)
+                                           ;; Esc
+                                           (and (= e.which 27)
+                                                (when-let [tp (rum/deref *tippy-ref)]
+                                                  (.hideTooltip tp)))))]
+
+                                (js/setTimeout #(.focus el-popup))
+                                (.addEventListener el-popup "keyup" cb)
+                                #(.removeEventListener el-popup "keyup" cb)))
+                            [])
+
+                          (when redirect-page-name
+                            [:div.tippy-wrapper.overflow-y-auto.p-4.outline-none
+                             {:ref   *el-popup
+                              :tab-index -1
+                              :style {:width          600
+                                      :text-align     "left"
+                                      :font-weight    500
+                                      :max-height     600
+                                      :padding-bottom 64}}
+                             (if (and (string? page-original-name) (string/includes? page-original-name "/"))
+                               [:div.my-2
+                                (->>
+                                  (for [page (string/split page-original-name #"/")]
+                                    (when (and (string? page) page)
+                                      (page-reference false page {} nil)))
+                                  (interpose [:span.mx-2.opacity-30 "/"]))]
+                               [:h2.font-bold.text-lg (if (= page-name redirect-page-name)
+                                                        page-original-name
+                                                        [:span
+                                                         [:span.text-sm.mr-2 "Alias:"]
+                                                         page-original-name])])
+                             (let [page (db/entity [:block/name (util/page-name-sanity-lc redirect-page-name)])]
+                               (editor-handler/insert-first-page-block-if-not-exists! redirect-page-name {:redirect? false})
+                               (when-let [f (state/get-page-blocks-cp)]
+                                 (f (state/get-current-repo) page {:sidebar? sidebar? :preview? true})))])))]
+
     (if (or (not manual?) open?)
-      (ui/tippy {:html            html-template
+      (ui/tippy {:ref             *tippy-ref
+                 :html            html-template
                  :interactive     true
                  :delay           [1000, 100]
                  :fixed-position? fixed-position?
                  :position        (or tippy-position "top")
-                 :distance        (or tippy-distance 10)}
+                 :distance        (or tippy-distance 10)
+                 :popperOptions   {:modifiers {:preventOverflow
+                                               {:enabled           true
+                                                :boundariesElement "viewport"}}}}
                 children)
       children)))
 
@@ -478,7 +525,7 @@
   "Accepts {:block/name sanitized / unsanitized page-name}"
   [{:keys [html-export? redirect-page-name label children contents-page? preview?] :as config} page]
   (when-let [page-name-in-block (:block/name page)]
-    (let [page-name-in-block (util/remove-boundary-slashes page-name-in-block)
+    (let [page-name-in-block (gp-util/remove-boundary-slashes page-name-in-block)
           page-name (util/page-name-sanity-lc page-name-in-block)
           page-entity (db/entity [:block/name page-name])
           redirect-page-name (or (and (= :org (state/get-preferred-format))
@@ -489,8 +536,16 @@
                             page-name-in-block
                             page-name
                             redirect-page-name page-entity contents-page? children html-export? label)]
-      (if (and (not (util/mobile?)) (not preview?))
+      (cond
+        (:breadcrumb? config)
+        (or (:block/original-name page)
+            (:block/name page))
+
+        (and (not (util/mobile?))
+             (not preview?))
         (page-preview-trigger (assoc config :children inner) page-name)
+
+        :else
         inner))))
 
 (rum/defc asset-reference
@@ -652,7 +707,7 @@
   (and (= 1 (count label))
        (let [label (first label)]
          (string? (last label))
-         (last label))))
+         (js/decodeURIComponent (last label)))))
 
 (defn- get-page
   [label]
@@ -675,7 +730,7 @@
   [config id label]
   (when (and
          (not (string/blank? id))
-         (gp-util/uuid-string? id))
+         (util/uuid-string? id))
     (let [block-id (uuid id)
           block (db/pull-block block-id)
           block-type (keyword (get-in block [:block/properties :ls-type]))
@@ -740,13 +795,13 @@
    (inline-text {} format v))
   ([config format v]
    (when (string? v)
-     (let [inline-list (mldoc/inline->edn v (mldoc/default-config format))]
+     (let [inline-list (gp-mldoc/inline->edn v (gp-mldoc/default-config format))]
        [:div.inline.mr-1 (map-inline config inline-list)]))))
 
 (defn- render-macro
   [config name arguments macro-content format]
   (if macro-content
-    (let [ast (->> (mldoc/->edn macro-content (mldoc/default-config format))
+    (let [ast (->> (mldoc/->edn macro-content (gp-mldoc/default-config format))
                    (map first))
           paragraph? (and (= 1 (count ast))
                           (= "Paragraph" (ffirst ast)))]
@@ -834,23 +889,33 @@
 
 (defn- media-link
   [config url s label metadata full_text]
-  (let [ext (util/get-file-ext s)]
+  (let [ext (keyword (util/get-file-ext s))
+        label-text (get-label-text label)]
     (cond
-      (contains? (set (map name config/audio-formats)) ext)
+      (contains? config/audio-formats ext)
       (audio-link config url s label metadata full_text)
 
-      (not (contains? #{"pdf" "mp4" "webm" "mov"} ext))
-      (image-link config url s label metadata full_text)
+      (contains? (config/doc-formats) ext)
+      (asset-link config label-text s metadata full_text)
 
-      (util/electron?)
-      (if (= (util/get-file-ext s) "pdf")
+      (= ext :pdf)
+      (cond
+        (util/electron?)
         [:a.asset-ref.is-pdf
          {:href "javascript:void(0);"
           :on-mouse-down (fn [_event]
                            (when-let [current (pdf-assets/inflate-asset s)]
                              (state/set-state! :pdf/current current)))}
-         (get-label-text label)]
-        (asset-reference config label s)))))
+         label-text]
+
+        (mobile-util/is-native-platform?)
+        (asset-link config label-text s metadata full_text))
+
+      (not (contains? #{:mp4 :webm :mov} ext))
+      (image-link config url s label metadata full_text)
+
+      :else
+      (asset-reference config label s))))
 
 (defn- search-link-cp
   [config url s label title metadata full_text]
@@ -873,7 +938,7 @@
     (not (string/includes? s "."))
     (page-reference (:html-export? config) s config label)
 
-    (util/url? s)
+    (gp-util/url? s)
     (->elem :a {:href s
                 :data-href s
                 :target "_blank"}
@@ -946,7 +1011,7 @@
                (= "Complex" protocol)
                (= (string/lower-case (:protocol path)) "id")
                (string? (:link path))
-               (gp-util/uuid-string? (:link path))) ; org mode id
+               (util/uuid-string? (:link path))) ; org mode id
           (let [id (uuid (:link path))
                 block (db/entity [:block/uuid id])]
             (if (:block/pre-block? block)
@@ -966,7 +1031,7 @@
                   show-brackets? (state/show-brackets?)]
               (if (and page
                        (when-let [ext (util/get-file-ext href)]
-                         (config/mldoc-support? ext)))
+                         (gp-config/mldoc-support? ext)))
                 [:span.page-reference
                  (when show-brackets? [:span.text-gray-500 "[["])
                  (page-cp config page)
@@ -1062,7 +1127,7 @@
                        string/trim)]
         (when-let [id (and s
                            (let [s (string/trim s)]
-                             (and (gp-util/uuid-string? s)
+                             (and (util/uuid-string? s)
                                   (uuid s))))]
           (block-embed (assoc config :link-depth (inc link-depth)) id)))
 
@@ -1073,7 +1138,7 @@
   [_config arguments]
   (when-let [url (first arguments)]
     (let [Vimeo-regex #"^((?:https?:)?//)?((?:www).)?((?:player.vimeo.com|vimeo.com)?)((?:/video/)?)([\w-]+)(\S+)?$"]
-      (when-let [vimeo-id (nth (gp-util/safe-re-find Vimeo-regex url) 5)]
+      (when-let [vimeo-id (nth (util/safe-re-find Vimeo-regex url) 5)]
         (when-not (string/blank? vimeo-id)
           (let [width (min (- (util/get-width) 96)
                            560)
@@ -1094,7 +1159,7 @@
       (when-let [id (cond
                       (<= (count url) 15) url
                       :else
-                      (last (gp-util/safe-re-find id-regex url)))]
+                      (last (util/safe-re-find id-regex url)))]
         (when-not (string/blank? id)
           (let [width (min (- (util/get-width) 96)
                            560)
@@ -1224,7 +1289,7 @@
           (when-let [youtube-id (cond
                                   (== 11 (count url)) url
                                   :else
-                                  (nth (gp-util/safe-re-find YouTube-regex url) 5))]
+                                  (nth (util/safe-re-find YouTube-regex url) 5))]
             (when-not (string/blank? youtube-id)
               (youtube/youtube-video youtube-id)))))
 
@@ -1255,7 +1320,7 @@
           (when-let [id (cond
                           (<= (count url) 15) url
                           :else
-                          (last (gp-util/safe-re-find id-regex url)))]
+                          (last (util/safe-re-find id-regex url)))]
             (ui/tweet-embed id))))
 
       (= name "embed")
@@ -1293,7 +1358,7 @@
          (->elem :sub (map-inline config l))
 
          ["Tag" _]
-         (when-let [s (block/get-tag item)]
+         (when-let [s (gp-block/get-tag item)]
            (let [s (text/page-ref-un-brackets! s)]
              (page-cp (assoc config :tag? true) {:block/name s})))
 
@@ -1424,6 +1489,7 @@
 (rum/defc block-children < rum/reactive
   [config children collapsed?]
   (let [ref? (:ref? config)
+        query? (:custom-query? config)
         children (and (coll? children) (filter map? children))]
     (when (and (coll? children)
                (seq children)
@@ -1439,8 +1505,8 @@
                             (-> config
                                 (assoc :block/uuid (:block/uuid child))
                                 (dissoc :breadcrumb-show? :embed-parent))
-                             ref?
-                             (assoc :ref-child? true))]
+                             (or ref? query?)
+                             (assoc :ref-query-child? true))]
                 (rum/with-key (block-container config child)
                   (:block/uuid child)))))]]))))
 
@@ -1723,8 +1789,8 @@
          (for [elem elems]
            (rum/with-key elem (str (random-uuid)))))
 
-       (and (string? v) (util/wrapped-by-quotes? v))
-       (util/unquote-string v)
+       (and (string? v) (gp-util/wrapped-by-quotes? v))
+       (gp-util/unquote-string v)
 
        :else
        (inline-text config (:block/format block) (str v)))]))
@@ -2098,11 +2164,13 @@
                   (rest parents)
                   parents)
         more? (> (count parents) level-limit)
-        parents (if more? (take-last level-limit parents) parents)]
+        parents (if more? (take-last level-limit parents) parents)
+        config (assoc config :breadcrumb? true)]
     (when show?
       (let [page-name-props (when show-page?
                               [page
-                               (or page-original-name page-name)])
+                               (page-cp (dissoc config :breadcrumb? true) page)
+                               {:block/name (or page-original-name page-name)}])
             parents-props (doall
                            (for [{:block/keys [uuid name content] :as block} parents]
                              (when-not name ; not page
@@ -2246,37 +2314,11 @@
        (= (:id config)
           (str (:block/uuid block)))))
 
-(rum/defcs ^:large-vars/cleanup-todo block-container < rum/reactive
-  {:init (fn [state]
-           (let [[config block] (:rum/args state)
-                 block-id (:block/uuid block)]
-             (cond
-               (root-block? config block)
-               (state/set-collapsed-block! block-id false)
-
-               (:ref? config)
-               (state/set-collapsed-block! block-id
-                                           (editor-handler/block-default-collapsed? block config))
-
-               :else
-               nil)
-             (assoc state
-                    ::control-show? (atom false)
-                    ::navigating-block (atom (:block/uuid block)))))
-   :should-update (fn [old-state new-state]
-                    (let [compare-keys [:block/uuid :block/content :block/parent :block/collapsed?
-                                        :block/properties :block/left :block/children :block/_refs :ui/selected?]
-                          config-compare-keys [:show-cloze?]
-                          b1 (second (:rum/args old-state))
-                          b2 (second (:rum/args new-state))
-                          result (or
-                                  (not= (select-keys b1 compare-keys)
-                                        (select-keys b2 compare-keys))
-                                  (not= (select-keys (first (:rum/args old-state)) config-compare-keys)
-                                        (select-keys (first (:rum/args new-state)) config-compare-keys)))]
-                      (boolean result)))}
-  [state config block]
-  (let [repo (state/get-current-repo)
+(rum/defc ^:large-vars/cleanup-todo block-container-inner < rum/reactive db-mixins/query
+  [state repo config block]
+  (let [ref? (:ref? config)
+        custom-query? (boolean (:custom-query? config))
+        ref-or-custom-query? (or ref? custom-query?)
         *navigating-block (get state ::navigating-block)
         navigating-block (rum/react *navigating-block)
         navigated? (and (not= (:block/uuid block) navigating-block) navigating-block)
@@ -2298,26 +2340,27 @@
                  config)
         heading? (or (= type :heading) (and heading-level (<= heading-level 6)))
         *control-show? (get state ::control-show?)
-        ref? (:ref? config)
         db-collapsed? (util/collapsed? block)
         collapsed? (cond
-                     (or ref? (root-block? config block))
+                     (or ref-or-custom-query? (root-block? config block))
                      (state/sub-collapsed uuid)
 
                      :else
                      db-collapsed?)
+        children (if (and ref-or-custom-query?
+                          (not collapsed?))
+                   (map
+                     (fn [b] (assoc b
+                                    :block/level (inc (:block/level block))))
+                     (model/sub-block-direct-children repo uuid))
+                   children)
         breadcrumb-show? (:breadcrumb-show? config)
         slide? (boolean (:slide? config))
-        custom-query? (boolean (:custom-query? config))
         doc-mode? (:document/mode? config)
         embed? (:embed? config)
         reference? (:reference? config)
         block-id (str "ls-block-" blocks-container-id "-" uuid)
-        has-child? (boolean
-                    (and
-                     (not pre-block?)
-                     (coll? children)
-                     (seq children)))
+        has-child? (first (:block/_parent (db/entity (:db/id block))))
         attrs (on-drag-and-mouse-attrs block uuid top? block-id *move-to)
         children-refs (get-children-refs children)
         data-refs (build-refs-data-value children-refs)
@@ -2378,6 +2421,47 @@
 
      (dnd-separator-wrapper block block-id slide? false false)]))
 
+(rum/defcs block-container < rum/reactive
+  {:init (fn [state]
+           (let [[config block] (:rum/args state)
+                 block-id (:block/uuid block)]
+             (cond
+               (root-block? config block)
+               (state/set-collapsed-block! block-id false)
+
+               (or (:ref? config) (:custom-query? config))
+               (state/set-collapsed-block! block-id
+                                           (editor-handler/block-default-collapsed? block config))
+
+               :else
+               nil)
+             (assoc state
+                    ::control-show? (atom false)
+                    ::navigating-block (atom (:block/uuid block)))))
+   :should-update (fn [old-state new-state]
+                    (let [compare-keys [:block/uuid :block/content :block/parent :block/collapsed?
+                                        :block/properties :block/left :block/children :block/_refs :ui/selected?]
+                          config-compare-keys [:show-cloze?]
+                          b1 (second (:rum/args old-state))
+                          b2 (second (:rum/args new-state))
+                          result (or
+                                  (not= (select-keys b1 compare-keys)
+                                        (select-keys b2 compare-keys))
+                                  (not= (select-keys (first (:rum/args old-state)) config-compare-keys)
+                                        (select-keys (first (:rum/args new-state)) config-compare-keys)))]
+                      (boolean result)))}
+  [state config block]
+  (let [repo (state/get-current-repo)
+        ref? (:ref? config)
+        custom-query? (boolean (:custom-query? config))]
+    (if (and ref? (not custom-query?) (not (:ref-query-child? config)))
+      (ui/lazy-visible
+       (fn []
+         (block-container-inner state repo config block))
+       nil
+       false)
+      (block-container-inner state repo config block))))
+
 (defn divide-lists
   [[f & l]]
   (loop [l l
@@ -2702,7 +2786,10 @@
   [config q]
   (ui/catch-error
    (ui/block-error "Query Error:" {:content (:query q)})
-   (custom-query* config q)))
+   (ui/lazy-visible
+    (fn [] (custom-query* config q))
+    nil
+    true)))
 
 (defn admonition
   [config type result]
@@ -2728,11 +2815,11 @@
 ;;     (cond
 ;;       (= lang "quote")
 ;;       (let [content (string/trim (string/join "\n" lines))]
-;;         ["Quote" (first (mldoc/->edn content (mldoc/default-config :markdown)))])
+;;         ["Quote" (first (mldoc/->edn content (gp-mldoc/default-config :markdown)))])
 
 ;;       (contains? #{"query" "note" "tip" "important" "caution" "warning" "pinned"} lang)
 ;;       (let [content (string/trim (string/join "\n" lines))]
-;;         ["Custom" lang nil (first (mldoc/->edn content (mldoc/default-config :markdown))) content])
+;;         ["Custom" lang nil (first (mldoc/->edn content (gp-mldoc/default-config :markdown))) content])
 
 ;;       :else
 ;;       ["Src" options])))
@@ -2820,7 +2907,7 @@
 
         ["Paragraph" l]
              ;; TODO: speedup
-        (if (gp-util/safe-re-find #"\"Export_Snippet\" \"embed\"" (str l))
+        (if (util/safe-re-find #"\"Export_Snippet\" \"embed\"" (str l))
           (->elem :div (map-inline config l))
           (->elem :div.is-paragraph (map-inline config l)))
 

src/main/frontend/components/block.css

@@ -240,6 +240,19 @@ html.is-native-android {
       align-items: center;
     }
   }
+
+  &.is-doc {
+      &:before {
+          content: "[[📜";
+          opacity: .7;
+          margin-right: 4px;
+      }
+
+      &:after {
+          content: "]]";
+          opacity: .7;
+      }
+  }
 }
 
 .embed-page {

src/main/frontend/components/content.cljs

@@ -36,7 +36,7 @@
 
 (defn- lazy-load
   [format]
-  (let [format (format/normalize format)]
+  (let [format (gp-util/normalize-format format)]
     (when-let [record (format/get-format-record format)]
       (when-not (protocol/loaded? record)
         (set-format-js-loading! format true)
@@ -363,7 +363,7 @@
                            e
                            (custom-context-menu-content))
 
-                          (and block-id (gp-util/uuid-string? block-id))
+                          (and block-id (util/uuid-string? block-id))
                           (let [block (.closest target ".ls-block")]
                             (when block
                               (util/select-highlight! [block]))
@@ -388,7 +388,7 @@
                    :format format}
                   id
                   config)
-      (let [format (format/normalize format)
+      (let [format (gp-util/normalize-format format)
             loading? (get loading format)
             markup? (contains? config/html-render-formats format)
             on-click (fn [e]
@@ -446,5 +446,5 @@
   (if hiccup
     [:div
      (hiccup-content id option)]
-    (let [format (format/normalize format)]
+    (let [format (gp-util/normalize-format format)]
       (non-hiccup-content id content on-click on-hide config format))))

src/main/frontend/components/editor.cljs

@@ -408,9 +408,9 @@
                :z-index    11}
               (when set-default-width?
                 {:width max-width})
-              (let [^js/HTMLElement textarea
-                    (js/document.querySelector "textarea.ls-textarea")]
-                (if (<= (.-clientWidth textarea) (+ left (if set-default-width? max-width 500)))
+              (let [^js/HTMLElement editor
+                    (js/document.querySelector ".editor-wrapper")]
+                (if (<= (.-clientWidth editor) (+ left (if set-default-width? max-width 500)))
                   {:right 0}
                   {:left (if (and y-diff (= y-diff 0)) left 0)})))}
      cp]))
@@ -478,7 +478,6 @@
   (let [content (if content (str content) "")]
     ;; as the function is binding to the editor content, optimization is welcome
     (str
-     "ls-textarea "
      (if (or (> (.-length content) 1000)
              (string/includes? content "\n"))
        "multiline-block"

src/main/frontend/components/file.cljs

@@ -9,11 +9,11 @@
             [frontend.context.i18n :refer [t]]
             [frontend.date :as date]
             [frontend.db :as db]
-            [frontend.format :as format]
             [frontend.handler.export :as export-handler]
             [frontend.state :as state]
             [frontend.util :as util]
             [logseq.graph-parser.config :as gp-config]
+            [logseq.graph-parser.util :as gp-util]
             [goog.object :as gobj]
             [reitit.frontend.easy :as rfe]
             [rum.core :as rum]))
@@ -71,7 +71,7 @@
                    state)}
   [state]
   (let [path (get-path state)
-        format (format/get-format path)
+        format (gp-util/get-format path)
         original-name (db/get-file-page path)
         random-id (str (d/squuid))]
     [:div.file {:id (str "file-edit-wrapper-" random-id)}

src/main/frontend/components/hierarchy.cljs

@@ -4,7 +4,7 @@
             [frontend.db :as db]
             [frontend.db.model :as db-model]
             [frontend.state :as state]
-            [frontend.text :as text]
+            [logseq.graph-parser.text :as text]
             [frontend.ui :as ui]
             [medley.core :as medley]
             [rum.core :as rum]

src/main/frontend/components/journal.cljs

@@ -8,7 +8,8 @@
             [frontend.db.model :as model]
             [frontend.handler.page :as page-handler]
             [frontend.state :as state]
-            [frontend.text :as text]
+            [logseq.graph-parser.text :as text]
+            [logseq.graph-parser.util :as gp-util]
             [frontend.ui :as ui]
             [frontend.util :as util]
             [goog.object :as gobj]
@@ -21,7 +22,7 @@
   (when-let [page-e (db/pull [:block/name (util/page-name-sanity-lc page)])]
     (page/page-blocks-cp repo page-e {})))
 
-(rum/defc journal-cp < rum/reactive
+(rum/defc journal-cp-inner < rum/reactive
   [[title format]]
   (let [;; Don't edit the journal title
         page (string/lower-case title)
@@ -51,7 +52,7 @@
                          :page))
                       (.preventDefault e)))}
        [:h1.title
-        (util/capitalize-all title)]]
+        (gp-util/capitalize-all title)]]
 
       (blocks-cp repo page format)
 
@@ -63,6 +64,10 @@
        (reference/references title)
        (str title "-refs"))]))
 
+(rum/defc journal-cp
+  [journal]
+  (ui/lazy-visible (fn [] (journal-cp-inner journal)) nil true))
+
 (rum/defc journals < rum/reactive
   [latest-journals]
   [:div#journals

src/main/frontend/components/page.cljs

@@ -1,6 +1,6 @@
 (ns frontend.components.page
   (:require [clojure.string :as string]
-            [frontend.components.block :as block]
+            [frontend.components.block :as component-block]
             [frontend.components.content :as content]
             [frontend.components.editor :as editor]
             [frontend.components.hierarchy :as hierarchy]
@@ -15,9 +15,7 @@
             [frontend.db.model :as model]
             [frontend.extensions.graph :as graph]
             [frontend.extensions.pdf.assets :as pdf-assets]
-            [frontend.modules.shortcut.core :as shortcut]
-            [frontend.handler.editor.lifecycle :as lifecycle]
-            [frontend.format.block :as format-block]
+            [frontend.format.block :as block]
             [frontend.handler.common :as common-handler]
             [frontend.handler.config :as config-handler]
             [frontend.handler.editor :as editor-handler]
@@ -27,17 +25,17 @@
             [frontend.handler.plugin :as plugin-handler]
             [frontend.handler.route :as route-handler]
             [frontend.mixins :as mixins]
-            [frontend.state :as state]
-            [frontend.text :as text]
+            [frontend.mobile.util :as mobile-util]
             [frontend.search :as search]
+            [frontend.state :as state]
             [frontend.ui :as ui]
             [frontend.util :as util]
             [goog.object :as gobj]
-            [reitit.frontend.easy :as rfe]
-            [medley.core :as medley]
-            [rum.core :as rum]
+            [logseq.graph-parser.text :as text]
             [logseq.graph-parser.util :as gp-util]
-            [frontend.mobile.util :as mobile-util]))
+            [medley.core :as medley]
+            [reitit.frontend.easy :as rfe]
+            [rum.core :as rum]))
 
 (defn- get-page-name
   [state]
@@ -117,13 +115,20 @@
      [:a.add-button-link.block
       (ui/icon "circle-plus")]]]])
 
-(rum/defc page-blocks-cp < rum/reactive
-  db-mixins/query
+(rum/defc page-blocks-cp < rum/reactive db-mixins/query
+  {:will-mount (fn [state]
+                 (let [page-e (second (:rum/args state))
+                       page-name (:block/name page-e)]
+                   (when (and (db/journal-page? page-name)
+                              (>= (date/journal-title->int page-name)
+                                  (date/journal-title->int (date/today))))
+                     (state/pub-event! [:journal/insert-template page-name])))
+                 state)}
   [repo page-e {:keys [sidebar?] :as config}]
   (when page-e
     (let [page-name (or (:block/name page-e)
                         (str (:block/uuid page-e)))
-          block? (gp-util/uuid-string? page-name)
+          block? (util/uuid-string? page-name)
           block-id (and block? (uuid page-name))
           page-blocks (get-blocks repo page-name block-id)]
       (if (empty? page-blocks)
@@ -140,7 +145,7 @@
                               :document/mode? document-mode?}
                              config)
               hiccup-config (common-handler/config-with-document-mode hiccup-config)
-              hiccup (block/->hiccup page-blocks hiccup-config {})]
+              hiccup (component-block/->hiccup page-blocks hiccup-config {})]
           [:div
            (page-blocks-inner page-name page-blocks hiccup sidebar? block-id)
            (when-not config/publishing?
@@ -164,9 +169,9 @@
            (rum/with-key
              (ui/catch-error
               (ui/component-error "Failed default query:" {:content (pr-str query)})
-              (block/custom-query {:attr {:class "mt-10"}
-                                   :editor-box editor/box
-                                   :page page} query))
+              (component-block/custom-query {:attr {:class "mt-10"}
+                                             :editor-box editor/box
+                                             :page page} query))
              (str repo "-custom-query-" (:query query))))]))))
 
 (defn tagged-pages
@@ -223,8 +228,8 @@
                          (reset! *edit? false)
                          (notification/show! "Illegal page name, can not rename!" :warning))
           blur-fn (fn [e]
-                    (when (util/wrapped-by-quotes? @*title-value)
-                      (swap! *title-value util/unquote-string)
+                    (when (gp-util/wrapped-by-quotes? @*title-value)
+                      (swap! *title-value gp-util/unquote-string)
                       (gobj/set (rum/deref input-ref) "value" @*title-value))
                     (state/set-state! :editor/editing-page-title? false)
                     (cond
@@ -319,7 +324,7 @@
     (let [current-repo (state/sub :git/current-repo)
           repo (or repo current-repo)
           page-name (util/page-name-sanity-lc path-page-name)
-          block? (gp-util/uuid-string? page-name)
+          block? (util/uuid-string? page-name)
           block-id (and block? (uuid page-name))
           format (let [page (if block-id
                               (:block/name (:block/page (db/entity [:block/uuid block-id])))
@@ -334,7 +339,7 @@
                       (db/entity repo))
                  (do
                    (when-not (db/entity repo [:block/name page-name])
-                     (let [m (format-block/page-name->map path-page-name true)]
+                     (let [m (block/page-name->map path-page-name true)]
                        (db/transact! repo [m])))
                    (db/pull [:block/name page-name])))
           {:keys [icon]} (:block/properties page)
@@ -379,7 +384,7 @@
            (let [config {:id "block-parent"
                          :block? true}]
              [:div.mb-4
-              (block/breadcrumb config repo block-id {:level-limit 3})]))
+              (component-block/breadcrumb config repo block-id {:level-limit 3})]))
 
          ;; blocks
          (let [page (if block?
@@ -642,7 +647,7 @@
               (date/today))
         theme (:ui/theme @state/state)
         dark? (= theme "dark")
-        graph (if (gp-util/uuid-string? page)
+        graph (if (util/uuid-string? page)
                 (graph-handler/build-block-graph (uuid page) theme)
                 (graph-handler/build-page-graph page theme))]
     (when (seq (:nodes graph))
@@ -714,7 +719,7 @@
          [:tr {:key name}
           [:td.n.w-12 [:span.opacity-70 (str (inc n) ".")]]
           [:td.name [:a {:href     (rfe/href :page {:name (:block/name page)})}
-                     (block/page-cp {} page)]]
+                     (component-block/page-cp {} page)]]
           [:td.backlinks [:span (or backlinks "0")]]
           (when-not orphaned-pages? [:td.created-at [:span (if created-at (date/int->local-time-2 created-at) "Unknown")]])
           (when-not orphaned-pages? [:td.updated-at [:span (if updated-at (date/int->local-time-2 updated-at) "Unknown")]])])]]
@@ -956,7 +961,7 @@
                                                (:db/id page)
                                                :page))))
                               :href     (rfe/href :page {:name (:block/name page)})}
-                          (block/page-cp {} page)]]
+                          (component-block/page-cp {} page)]]
 
                (when-not mobile?
                  [:td.backlinks [:span backlinks]])

src/main/frontend/components/page_menu.cljs

@@ -14,7 +14,6 @@
             [frontend.handler.shell :as shell]
             [frontend.handler.plugin :as plugin-handler]
             [frontend.mobile.util :as mobile-util]
-            [logseq.graph-parser.util :as gp-util]
             [electron.ipc :as ipc]
             [frontend.config :as config]
             [frontend.handler.user :as user-handler]
@@ -64,7 +63,7 @@
           repo (state/sub :git/current-repo)
           page (db/entity repo [:block/name page-name])
           page-original-name (:block/original-name page)
-          block? (and page (gp-util/uuid-string? page-name))
+          block? (and page (util/uuid-string? page-name))
           contents? (= page-name "contents")
           properties (:block/properties page)
           public? (true? (:public properties))

src/main/frontend/components/plugins.cljs

@@ -8,6 +8,7 @@
             [frontend.search :as search]
             [frontend.util :as util]
             [frontend.mixins :as mixins]
+            [electron.ipc :as ipc]
             [promesa.core :as p]
             [frontend.components.svg :as svg]
             [frontend.components.plugins-settings :as plugins-settings]
@@ -334,10 +335,69 @@
     :intent "logseq"
     :target "_blank"))
 
+(rum/defc user-proxy-settings-panel
+  [{:keys [protocol] :as agent-opts}]
+  (let [[opts set-opts!] (rum/use-state agent-opts)
+        [testing? set-testing?!] (rum/use-state false)
+        *test-input (rum/create-ref)
+        disabled? (string/blank? (:protocol opts))]
+    [:div.cp__settings-network-proxy-panel
+     [:h1.mb-2.text-2xl.font-bold (t :settings-page/network-proxy)]
+     [:div.p-2
+      [:p [:label [:strong (t :type)]
+           (ui/select [{:label "Disabled" :value "" :selected disabled?}
+                       {:label "http" :value "http" :selected (= protocol "http")}
+                       {:label "https" :value "https" :selected (= protocol "https")}
+                       {:label "socks5" :value "socks5" :selected (= protocol "socks5")}]
+                      #(set-opts!
+                         (assoc opts :protocol (if (= "disabled" (util/safe-lower-case %)) nil %))) nil)]]
+      [:p.flex
+       [:label.pr-4 [:strong (t :host)]
+        [:input.form-input.is-small
+         {:value     (:host opts) :disabled disabled?
+          :on-change #(set-opts!
+                        (assoc opts :host (util/trim-safe (util/evalue %))))}]]
+
+       [:label [:strong (t :port)]
+        [:input.form-input.is-small
+         {:value (:port opts) :type "number" :disabled disabled?
+          :on-change #(set-opts!
+                        (assoc opts :port (util/trim-safe (util/evalue %))))}]]]
+
+      [:hr]
+      [:p.flex.items-center.space-x-2
+       [:span.w-60
+        [:input.form-input.is-small
+         {:ref *test-input
+          :placeholder "http://"
+          :on-change #(set-opts!
+                        (assoc opts :test (util/trim-safe (util/evalue %))))
+          :value (:test opts)}]]
+
+       (ui/button (if testing? (ui/loading "Testing") "Test URL")
+         :intent "logseq" :large? false
+         :style {:margin-top 0 :padding "5px 15px"}
+         :on-click #(let [val (util/trim-safe (.-value (rum/deref *test-input)))]
+                      (when (and (not testing?) (not (string/blank? val)))
+                        (set-testing?! true)
+                        (-> (p/let [_ (ipc/ipc :setHttpsAgent opts)
+                                    _ (ipc/ipc :testProxyUrl val)])
+                          (p/catch (fn [e] (notification/show! (str e) :error)))
+                          (p/finally (fn [] (set-testing?! false)))))
+                      ))]
+
+      [:p.pt-2
+       (ui/button (t :save)
+         :on-click (fn []
+                     (p/let [_ (ipc/ipc :setHttpsAgent opts)]
+                       (state/set-state! [:electron/user-cfgs :settings/agent] opts)
+                       (state/close-sub-modal! :https-proxy-panel))))]]]))
+
 (rum/defc ^:large-vars/cleanup-todo panel-control-tabs < rum/static
   [search-key *search-key category *category
    sort-by *sort-by filter-by *filter-by
-   selected-unpacked-pkg market? develop-mode? reload-market-fn]
+   selected-unpacked-pkg market? develop-mode?
+   reload-market-fn agent-opts]
 
   (let [*search-ref (rum/create-ref)]
     [:div.mb-2.flex.justify-between.control-tabs.relative
@@ -357,6 +417,16 @@
          (unpacked-plugin-loader selected-unpacked-pkg)])]
 
      [:div.flex.items-center.r
+      ;; extra info
+      (let [{:keys [protocol host port]} agent-opts]
+        (when (every? not-empty [protocol host port])
+          (ui/button
+            [:span.flex.items-center.text-indigo-500
+             (ui/icon "world-download") (str protocol "://" host ":" port)]
+            :small? true
+            :intent "link"
+            :on-click #(state/pub-event! [:go/proxy-settings agent-opts]))))
+
       ;; search
       (panel-tab-search search-key *search-key *search-ref)
 
@@ -440,6 +510,10 @@
                     :options {:on-click #(reload-market-fn)}}]
                   [{:title   [:span (ui/icon "rotate-clockwise") (t :plugin/check-all-updates)]
                     :options {:on-click #(plugin-handler/check-enabled-for-updates (not= :plugins category))}}])
+
+                [{:title   [:span (ui/icon "world") (t :settings-page/network-proxy)]
+                  :options {:on-click #(state/pub-event! [:go/proxy-settings agent-opts])}}]
+
                 (when (state/developer-mode?)
                   [{:hr true}
                    {:title   [:span (ui/icon "file-code") "Open Preferences"]
@@ -481,6 +555,7 @@
         installing (state/sub :plugin/installing)
         online? (state/sub :network/online?)
         develop-mode? (state/sub :ui/developer-mode?)
+        agent-opts (state/sub [:electron/user-cfgs :settings/agent])
         *search-key (::search-key state)
         *category (::category state)
         *sort-by (::sort-by state)
@@ -526,20 +601,18 @@
        @*search-key *search-key
        @*category *category
        @*sort-by *sort-by @*filter-by *filter-by
-       nil true develop-mode? (::reload state))
+       nil true develop-mode? (::reload state)
+       agent-opts)
 
      (cond
        (not online?)
-       [:p.flex.justify-center.pt-20.opacity-50
-        (svg/offline 30)]
+       [:p.flex.justify-center.pt-20.opacity-50 (svg/offline 30)]
 
        @*fetching
-       [:p.flex.justify-center.pt-20
-        svg/loading]
+       [:p.flex.justify-center.pt-20 svg/loading]
 
        @*error
-       [:p.flex.justify-center.pt-20.opacity-50
-        "Remote error: " (.-message @*error)]
+       [:p.flex.justify-center.pt-20.opacity-50 "Remote error: " (.-message @*error)]
 
        :else
        [:div.cp__plugins-marketplace-cnt
@@ -568,6 +641,7 @@
         develop-mode? (state/sub :ui/developer-mode?)
         selected-unpacked-pkg (state/sub :plugin/selected-unpacked-pkg)
         coming-updates (state/sub :plugin/updates-coming)
+        agent-opts (state/sub [:electron/user-cfgs :settings/agent])
         *filter-by (::filter-by state)
         *sort-by (::sort-by state)
         *search-key (::search-key state)
@@ -611,7 +685,8 @@
        @*sort-by *sort-by
        @*filter-by *filter-by
        selected-unpacked-pkg
-       false develop-mode? nil)
+       false develop-mode? nil
+       agent-opts)
 
      [:div.cp__plugins-item-lists.grid-cols-1.md:grid-cols-2.lg:grid-cols-3
       (for [item sorted-plugins]
@@ -752,6 +827,10 @@
         market? (= active :marketplace)
         *el-ref (rum/create-ref)]
 
+    (rum/use-effect!
+      #(state/load-app-user-cfgs)
+      [])
+
     [:div.cp__plugins-page
      {:ref       *el-ref
       :tab-index "-1"}

src/main/frontend/components/reference.cljs

@@ -12,7 +12,6 @@
             [frontend.state :as state]
             [frontend.ui :as ui]
             [frontend.util :as util]
-            [logseq.graph-parser.util :as gp-util]
             [medley.core :as medley]
             [rum.core :as rum]))
 
@@ -49,24 +48,6 @@
   (fn [close-fn]
     (filter-dialog-inner filters-atom close-fn references page-name)))
 
-(defn- block-with-ref-level
-  [block level]
-  (if (:block/children block)
-    (-> (update block :block/children
-                (fn [blocks]
-                  (map (fn [block]
-                         (let [level (inc level)
-                               block (assoc block :ref/level level)]
-                           (block-with-ref-level block level))) blocks)))
-        (assoc :ref/level level))
-    (assoc block :ref/level level)))
-
-(defn- blocks-with-ref-level
-  [page-blocks]
-  (map (fn [[page blocks]]
-         [page (map #(block-with-ref-level % 1) blocks)])
-    page-blocks))
-
 (rum/defc block-linked-references < rum/reactive db-mixins/query
   [block-id]
   (let [refed-blocks-ids (model-db/get-referenced-blocks-ids (str block-id))]
@@ -101,7 +82,7 @@
           default-collapsed? (>= (count refed-blocks-ids) threshold)
           filters-atom (get state ::filters)
           filter-state (rum/react filters-atom)
-          block? (gp-util/uuid-string? page-name)
+          block? (util/uuid-string? page-name)
           block-id (and block? (uuid page-name))
           page-name (string/lower-case page-name)
           journal? (date/valid-journal-title? (string/capitalize page-name))
@@ -163,8 +144,7 @@
                      filters (when (seq filter-state)
                                (->> (group-by second filter-state)
                                     (medley/map-vals #(map first %))))
-                     filtered-ref-blocks (->> (block-handler/filter-blocks repo ref-blocks filters true)
-                                              blocks-with-ref-level)
+                     filtered-ref-blocks (block-handler/filter-blocks repo ref-blocks filters true)
                      n-ref (apply +
                              (for [[_ rfs] filtered-ref-blocks]
                                (count rfs)))]
@@ -188,7 +168,11 @@
   [page-name]
   (ui/catch-error
    (ui/component-error "Linked References: Unexpected error")
-   (references* page-name)))
+   (ui/lazy-visible
+    (fn []
+      (references* page-name))
+    nil
+    false)))
 
 (rum/defcs unlinked-references-aux
   < rum/reactive db-mixins/query

src/main/frontend/components/repo.cljs

@@ -14,7 +14,7 @@
             [reitit.frontend.easy :as rfe]
             [rum.core :as rum]
             [frontend.mobile.util :as mobile-util]
-            [frontend.text :as text]
+            [logseq.graph-parser.text :as text]
             [promesa.core :as p]
             [electron.ipc :as ipc]
             [goog.object :as gobj]

src/main/frontend/components/search.cljs

@@ -20,7 +20,6 @@
             [clojure.string :as string]
             [frontend.context.i18n :refer [t]]
             [frontend.date :as date]
-            [logseq.graph-parser.util :as gp-util]
             [reitit.frontend.easy :as rfe]
             [frontend.modules.shortcut.core :as shortcut]))
 
@@ -33,7 +32,7 @@
             lc-content (util/search-normalize content)
             lc-q (util/search-normalize q)]
         (if (and (string/includes? lc-content lc-q)
-                 (not (gp-util/safe-re-find #" " q)))
+                 (not (util/safe-re-find #" " q)))
           (let [i (string/index-of lc-content lc-q)
                 [before after] [(subs content 0 i) (subs content (+ i (count q)))]]
             [:div

src/main/frontend/components/select.cljs

@@ -10,7 +10,7 @@
             [frontend.ui :as ui]
             [frontend.util :as util]
             [frontend.db :as db]
-            [frontend.text :as text]
+            [logseq.graph-parser.text :as text]
             [rum.core :as rum]
             [frontend.config :as config]
             [frontend.handler.repo :as repo-handler]

src/main/frontend/components/settings.cljs

@@ -1,6 +1,7 @@
 (ns frontend.components.settings
   (:require [clojure.string :as string]
             [frontend.components.svg :as svg]
+            [frontend.components.plugins :as plugins]
             [frontend.config :as config]
             [frontend.context.i18n :refer [t]]
             [frontend.storage :as storage]
@@ -134,7 +135,6 @@
                   (mobile-util/is-native-platform?))
       [:div.text-sm desc])]])
 
-
 (defn edit-config-edn []
   (row-with-button-action
     {:left-label   (t :settings-page/custom-configuration)
@@ -485,40 +485,6 @@
                     :on-click #(js/logseq.api.relaunch)
                     :small? true :intent "logseq")]])]))
 
-(rum/defc user-proxy-settings-panel
-  [{:keys [protocol] :as agent-opts}]
-  (let [[opts set-opts!] (rum/use-state agent-opts)
-        disabled? (string/blank? (:protocol opts))]
-    [:div.cp__settings-network-proxy-panel
-     [:h1.mb-2.text-2xl.font-bold (t :settings-page/network-proxy)]
-     [:div.p-2
-      [:p [:label [:strong (t :type)]
-           (ui/select [{:label "Disabled" :value "" :selected disabled?}
-                       {:label "http" :value "http" :selected (= protocol "http")}
-                       {:label "https" :value "https" :selected (= protocol "https")}
-                       {:label "socks5" :value "socks5" :selected (= protocol "socks5")}]
-                      #(set-opts!
-                        (assoc opts :protocol (if (= "disabled" (util/safe-lower-case %)) nil %))) nil)]]
-      [:p.flex
-       [:label.pr-4 [:strong (t :host)]
-        [:input.form-input.is-small
-         {:value     (:host opts) :disabled disabled?
-          :on-change #(set-opts!
-                       (assoc opts :host (util/trim-safe (util/evalue %))))}]]
-
-       [:label [:strong (t :port)]
-        [:input.form-input.is-small
-         {:value (:port opts) :type "number" :disabled disabled?
-          :on-change #(set-opts!
-                       (assoc opts :port (util/trim-safe (util/evalue %))))}]]]
-
-      [:p.pt-2
-       (ui/button (t :save)
-        :on-click (fn []
-                    (p/let [_ (ipc/ipc :setHttpsAgent opts)]
-                      (state/set-state! [:electron/user-cfgs :settings/agent] opts)
-                      (state/close-sub-modal! :https-proxy-panel))))]]]))
-
 (rum/defc user-proxy-settings
   [{:keys [protocol host port] :as agent-opts}]
   (ui/button [:span
@@ -526,7 +492,7 @@
                 [:strong.pr-1 e])
               (ui/icon "edit")]
              :on-click #(state/set-sub-modal!
-                         (fn [_] (user-proxy-settings-panel agent-opts))
+                         (fn [_] (plugins/user-proxy-settings-panel agent-opts))
                          {:id :https-proxy-panel :center? true})))
 
 (defn plugin-system-switcher-row []

src/main/frontend/config.cljs

@@ -23,6 +23,7 @@
 ;; (goog-define LOGIN-URL
 ;;              "https://logseq.auth.us-east-1.amazoncognito.com/login?client_id=7ns5v1pu8nrbs04rvdg67u4a7c&response_type=code&scope=email+openid+phone&redirect_uri=logseq%3A%2F%2Fauth-callback")
 ;; (goog-define API-DOMAIN "api-prod.logseq.com")
+;; (goog-define WS-URL "wss://b2rp13onu2.execute-api.us-east-1.amazonaws.com/production?graphuuid=%s")
 
 ;; dev env
 (goog-define FILE-SYNC-PROD? false)
@@ -84,6 +85,15 @@
      config-formats
      #{:gif :svg :jpeg :ico :png :jpg :bmp :webp})))
 
+(defn doc-formats
+  []
+  (let [config-formats (some->> (get-in @state/state [:config :document-formats])
+                                (map :keyword)
+                                (set))]
+    (set/union
+     config-formats
+     #{:doc :docx :xls :xlsx :ppt :pptx :one :epub})))
+
 (def audio-formats #{:mp3 :ogg :mpeg :wav :m4a :flac :wma :aac})
 
 (def media-formats (set/union (img-formats) audio-formats))
@@ -96,17 +106,9 @@
   (set/union (text-formats)
              (img-formats)))
 
-;; TODO: rename
-(defonce mldoc-support-formats
-  #{:org :markdown :md})
-
-(defn mldoc-support?
-  [format]
-  (contains? mldoc-support-formats (keyword format)))
-
 (def mobile?
   (when-not util/node-test?
-    (gp-util/safe-re-find #"Mobi" js/navigator.userAgent)))
+    (util/safe-re-find #"Mobi" js/navigator.userAgent)))
 
 ;; TODO: protocol design for future formats support
 

src/main/frontend/date.cljs

@@ -5,9 +5,10 @@
             [cljs-time.core :as t]
             [cljs-time.format :as tf]
             [cljs-time.local :as tl]
-            [clojure.string :as string]
             [frontend.state :as state]
             [frontend.util :as util]
+            [logseq.graph-parser.util :as gp-util]
+            [logseq.graph-parser.date-time-util :as date-time-util]
             [goog.object :as gobj]
             [lambdaisland.glogi :as log]))
 
@@ -16,11 +17,6 @@
   (when (string? s)
     ((gobj/get chrono "parseDate") s)))
 
-(defn format
-  [date]
-  (when-let [formatter-string (state/get-date-formatter)]
-    (tf/unparse (tf/formatter formatter-string) date)))
-
 (def custom-formatter (tf/formatter "yyyy-MM-dd'T'HH:mm:ssZZ"))
 
 (defn journal-title-formatters
@@ -55,13 +51,6 @@
      "yyyy年MM月dd日"}
    (state/get-date-formatter)))
 
-;; (tf/parse (tf/formatter "dd.MM.yyyy") "2021Q4") => 20040120T000000
-(defn safe-journal-title-formatters
-  []
-  (->> [(state/get-date-formatter) "yyyy-MM-dd" "yyyy_MM_dd"]
-       (remove string/blank?)
-       distinct))
-
 (defn get-date-time-string
   ([]
    (get-date-time-string (t/now)))
@@ -115,7 +104,7 @@
   ([]
    (journal-name (tl/local-now)))
   ([date]
-   (format date)))
+   (date-time-util/format date (state/get-date-formatter))))
 
 (defn journal-name-s [s]
   (try
@@ -183,34 +172,19 @@
 (defn valid-journal-title?
   [title]
   (and title
-       (valid? (util/capitalize-all title))))
+       (valid? (gp-util/capitalize-all title))))
 
 (defn journal-title->
   ([journal-title then-fn]
-   (journal-title-> journal-title then-fn (safe-journal-title-formatters)))
+   (journal-title-> journal-title then-fn (date-time-util/safe-journal-title-formatters (state/get-date-formatter))))
   ([journal-title then-fn formatters]
-   (when-not (string/blank? journal-title)
-     (when-let [time (->> (map
-                            (fn [formatter]
-                              (try
-                                (tf/parse (tf/formatter formatter) (util/capitalize-all journal-title))
-                                (catch js/Error _e
-                                  nil)))
-                            formatters)
-                          (filter some?)
-                          first)]
-       (then-fn time)))))
+   (date-time-util/journal-title-> journal-title then-fn formatters)))
 
 (defn journal-title->int
   [journal-title]
-  (when journal-title
-    (let [journal-title (util/capitalize-all journal-title)]
-      (journal-title-> journal-title #(util/parse-int (tf/unparse (tf/formatter "yyyyMMdd") %))))))
-
-(defn int->journal-title
-  [day]
-  (when day
-    (format (tf/parse (tf/formatter "yyyyMMdd") (str day)))))
+  (date-time-util/journal-title->int
+   journal-title
+   (date-time-util/safe-journal-title-formatters (state/get-date-formatter))))
 
 (defn journal-day->ts
   [day]
@@ -231,9 +205,16 @@
                     default-journal-title-formatter)]
     (journal-title-> journal-title #(tf/unparse formatter %))))
 
+(defn date->file-name
+  [date]
+  (let [formatter (if-let [format (state/get-journal-file-name-format)]
+                    (tf/formatter format)
+                    default-journal-title-formatter)]
+    (tf/unparse formatter date)))
+
 (defn journal-title->custom-format
   [journal-title]
-  (journal-title-> journal-title format))
+  (journal-title-> journal-title #(date-time-util/format % (state/get-date-formatter))))
 
 (defn int->local-time-2
   [n]

src/main/frontend/db/conn.cljs

@@ -7,7 +7,7 @@
             [frontend.mobile.util :as mobile-util]
             [frontend.state :as state]
             [frontend.config :as config]
-            [frontend.text :as text]
+            [logseq.graph-parser.text :as text]
             [logseq.graph-parser.util :as gp-util]
             [datascript.core :as d]))
 

src/main/frontend/db/model.cljs

@@ -12,7 +12,6 @@
             [frontend.db.conn :as conn]
             [frontend.db.react :as react]
             [frontend.db.utils :as db-utils]
-            [frontend.format :as format]
             [frontend.state :as state]
             [frontend.util :as util :refer [react]]
             [logseq.graph-parser.util :as gp-util]
@@ -294,7 +293,7 @@
       (:block/format page)
       (when-let [file (:block/file page)]
         (when-let [path (:file/path (db-utils/entity (:db/id file)))]
-          (format/get-format path)))))
+          (gp-util/get-format path)))))
    (state/get-preferred-format)
    :markdown))
 
@@ -815,6 +814,20 @@
           block-uuid)
         (sort-by-left (db-utils/entity [:block/uuid block-uuid])))))
 
+(defn sub-block-direct-children
+  "Doesn't include nested children."
+  [repo block-uuid]
+  (when-let [db (conn/get-db repo)]
+    (-> (react/q repo [:frontend.db.react/block-direct-children block-uuid] {}
+          '[:find [(pull ?b [*]) ...]
+            :in $ ?parent-id
+            :where
+            [?parent :block/uuid ?parent-id]
+            [?b :block/parent ?parent]]
+          block-uuid)
+        react
+        (sort-by-left (db-utils/entity [:block/uuid block-uuid])))))
+
 (defn get-block-children
   "Including nested children."
   [repo block-uuid]
@@ -899,7 +912,7 @@
 
 (defn get-page
   [page-name]
-  (if (gp-util/uuid-string? page-name)
+  (if (util/uuid-string? page-name)
     (db-utils/entity [:block/uuid (uuid page-name)])
     (db-utils/entity [:block/name (util/page-name-sanity-lc page-name)])))
 
@@ -970,16 +983,6 @@
         (reverse)
         (take n))))))
 
-(defn journal-day-exists?
-  [graph day]
-  (d/q
-    '[:find ?p .
-      :in $ ?day
-      :where
-      [?p :block/journal-day ?day]]
-    (conn/get-db graph)
-    day))
-
 ;; get pages that this page referenced
 (defn get-page-referenced-pages
   [repo page]
@@ -1082,38 +1085,20 @@
        (let [page-id (:db/id (db-utils/entity [:block/name (util/safe-page-name-sanity-lc page)]))
              pages (page-alias-set repo page)
              aliases (set/difference pages #{page-id})
-             query-result (if (seq aliases)
-                            (let [rules '[[(find-blocks ?block ?ref-page ?pages ?alias ?aliases)
-                                           [?block :block/page ?alias]
-                                           [(contains? ?aliases ?alias)]]
-                                          [(find-blocks ?block ?ref-page ?pages ?alias ?aliases)
-                                           [?block :block/refs ?ref-page]
-                                           [(contains? ?pages ?ref-page)]]]]
-                              (react/q repo
-                                       [:frontend.db.react/page<-blocks-or-block<-blocks page-id]
-                                       {}
-                                       '[:find [(pull ?block ?block-attrs) ...]
-                                         :in $ % ?pages ?aliases ?block-attrs
-                                         :where
-                                         (find-blocks ?block ?ref-page ?pages ?alias ?aliases)]
-                                       rules
-                                       pages
-                                       aliases
-                                       block-attrs))
-                            (react/q repo
-                                     [:frontend.db.react/page<-blocks-or-block<-blocks page-id]
-                                     {:use-cache? false}
-                                     '[:find [(pull ?ref-block ?block-attrs) ...]
-                                       :in $ ?page ?block-attrs
-                                       :where
-                                       [?ref-block :block/refs ?page]]
-                                     page-id
-                                     block-attrs))
+             query-result (react/q repo
+                            [:frontend.db.react/page<-blocks-or-block<-blocks page-id]
+                            {}
+                            '[:find [(pull ?block ?block-attrs) ...]
+                              :in $ [?ref-page ...] ?block-attrs
+                              :where
+                              [?block :block/refs ?ref-page]]
+                            pages
+                            (butlast block-attrs))
              result (->> query-result
                          react
-                         (sort-by-left-recursive)
                          (remove (fn [block]
                                    (= page-id (:db/id (:block/page block)))))
+                         (sort-by-left-recursive)
                          db-utils/group-by-page
                          (map (fn [[k blocks]]
                                 (let [k (if (contains? aliases (:db/id k))
@@ -1129,35 +1114,14 @@
   ([repo page]
    (when repo
      (when-let [db (conn/get-db repo)]
-       (let [page-id (:db/id (db-utils/entity [:block/name (util/safe-page-name-sanity-lc page)]))
-             pages (page-alias-set repo page)
-             aliases (set/difference pages #{page-id})
-             query-result (if (seq aliases)
-                            (let [rules '[[(find-blocks ?block ?ref-page ?pages ?alias ?aliases)
-                                           [?block :block/page ?alias]
-                                           [(contains? ?aliases ?alias)]]
-                                          [(find-blocks ?block ?ref-page ?pages ?alias ?aliases)
-                                           [?block :block/refs ?ref-page]
-                                           [(contains? ?pages ?ref-page)]]]]
-                              (d/q
-                                '[:find ?block
-                                  :in $ % ?pages ?aliases ?block-attrs
-                                  :where
-                                  (find-blocks ?block ?ref-page ?pages ?alias ?aliases)]
-                                db
-                                rules
-                                pages
-                                aliases
-                                block-attrs))
-                            (d/q
-                              '[:find ?ref-block
-                                :in $ ?page ?block-attrs
-                                :where
-                                [?ref-block :block/refs ?page]]
-                              db
-                              page-id
-                              block-attrs))]
-         query-result)))))
+       (let [pages (page-alias-set repo page)]
+         (d/q
+           '[:find ?block
+             :in $ [?ref-page ...]
+             :where
+             [?block :block/refs ?ref-page]]
+           db
+           pages))))))
 
 (defn get-date-scheduled-or-deadlines
   [journal-title]
@@ -1257,7 +1221,7 @@
 
 (defn get-referenced-blocks-ids
   [page-name-or-block-uuid]
-  (if (gp-util/uuid-string? (str page-name-or-block-uuid))
+  (if (util/uuid-string? (str page-name-or-block-uuid))
     (let [id (uuid page-name-or-block-uuid)]
       (get-block-referenced-blocks-ids id))
     (get-page-referenced-blocks-ids page-name-or-block-uuid)))
@@ -1356,7 +1320,7 @@
   [name]
   (when (string? name)
     (->> (d/q
-           '[:find (pull ?b [*])
+           '[:find [(pull ?b [*]) ...]
              :in $ ?name
              :where
              [?b :block/properties ?p]
@@ -1364,7 +1328,8 @@
              [(= ?t ?name)]]
            (conn/get-db)
            name)
-         ffirst)))
+         (sort-by :block/name)
+         (first))))
 
 (defonce blocks-count-cache (atom nil))
 

src/main/frontend/db/query_dsl.cljs

@@ -6,15 +6,15 @@
             [clojure.set :as set]
             [clojure.string :as string]
             [clojure.walk :as walk]
+            [frontend.state :as state]
             [frontend.date :as date]
             [frontend.db.model :as model]
             [frontend.db.query-react :as query-react]
             [frontend.db.utils :as db-utils]
             [frontend.db.rules :as rules]
             [frontend.template :as template]
-            [frontend.text :as text]
-            [frontend.util :as util]
-            [logseq.graph-parser.util :as gp-util]))
+            [logseq.graph-parser.text :as text]
+            [frontend.util :as util]))
 
 
 ;; Query fields:
@@ -238,7 +238,7 @@
   (let [k (string/replace (name (nth e 1)) "_" "-")
         v (nth e 2)
         v (if-not (nil? v)
-            (text/parse-property k v)
+            (text/parse-property k v (state/get-config))
             v)
         v (if (coll? v) (first v) v)]
     {:query (list 'property '?b (keyword k) v)
@@ -283,7 +283,7 @@
   (let [[k v] (rest e)
         k (string/replace (name k) "_" "-")]
     (if (some? v)
-      (let [v' (text/parse-property k v)
+      (let [v' (text/parse-property k v (state/get-config))
             val (if (coll? v') (first v') v')]
         {:query (list 'page-property '?p (keyword k) val)
          :rules [:page-property]})
@@ -310,8 +310,8 @@
   [e sample]
   (when-let [num (second e)]
     (when (integer? num)
-      (reset! sample num))
-    nil))
+      (reset! sample num)
+      {:query [['?p :block/uuid]]})))
 
 (defn- build-sort-by
   [e sort-by_]
@@ -448,7 +448,7 @@ Some bindings in this fn:
                                                  (remove string/blank?)
                                                  (map (fn [x]
                                                         (if (or (contains? #{"+" "-"} (first x))
-                                                                (and (gp-util/safe-re-find #"\d" (first x))
+                                                                (and (util/safe-re-find #"\d" (first x))
                                                                      (some #(string/ends-with? x %) ["y" "m" "d" "h" "min"])))
                                                           (keyword (name x))
                                                           x)))

src/main/frontend/db/query_react.cljs

@@ -10,9 +10,8 @@
             [frontend.debug :as debug]
             [frontend.extensions.sci :as sci]
             [frontend.state :as state]
-            [frontend.text :as text]
+            [logseq.graph-parser.text :as text]
             [frontend.util :as util]
-            [logseq.graph-parser.util :as gp-util]
             [lambdaisland.glogi :as log]))
 
 (defn resolve-input
@@ -32,12 +31,12 @@
     ;; This sometimes runs when there isn't a current page e.g. :home route
     (some-> (state/get-current-page) string/lower-case)
     (and (keyword? input)
-         (gp-util/safe-re-find #"^\d+d(-before)?$" (name input)))
+         (util/safe-re-find #"^\d+d(-before)?$" (name input)))
     (let [input (name input)
           days (util/parse-int (subs input 0 (dec (count input))))]
       (date->int (t/minus (t/today) (t/days days))))
     (and (keyword? input)
-         (gp-util/safe-re-find #"^\d+d(-after)?$" (name input)))
+         (util/safe-re-find #"^\d+d(-after)?$" (name input)))
     (let [input (name input)
           days (util/parse-int (subs input 0 (dec (count input))))]
       (date->int (t/plus (t/today) (t/days days))))

src/main/frontend/db/react.cljs

@@ -10,7 +10,6 @@
             [frontend.db.utils :as db-utils]
             [frontend.state :as state]
             [frontend.util :as util :refer [react]]
-            [logseq.graph-parser.util :as gp-util]
             [cljs.spec.alpha :as s]
             [clojure.core.async :as async]))
 
@@ -24,6 +23,8 @@
 ;; ::block-and-children
 ;; get block&children react-query
 (s/def ::block-and-children (s/tuple #(= ::block-and-children %) uuid?))
+
+(s/def ::block-direct-children (s/tuple #(= ::block-direct-children %) uuid?))
 ;; ::journals
 ;; get journal-list react-query
 (s/def ::journals (s/tuple #(= ::journals %)))
@@ -48,6 +49,7 @@
 (s/def ::react-query-keys (s/or :block ::block
                                 :page-blocks ::page-blocks
                                 :block-and-children ::block-and-children
+                                :block-direct-children ::block-direct-children
                                 :journals ::journals
                                 :page->pages ::page->pages
                                 :page<-pages ::page<-pages
@@ -213,9 +215,8 @@
 
 (defn get-affected-queries-keys
   "Get affected queries through transaction datoms."
-  [{:keys [tx-data]}]
+  [{:keys [tx-data db-before]}]
   {:post [(s/valid? ::affected-keys %)]}
-
   (let [blocks (->> (filter (fn [datom] (contains? #{:block/left :block/parent :block/page} (:a datom))) tx-data)
                     (map :v)
                     (distinct))
@@ -228,7 +229,7 @@
         affected-keys (concat
                        (mapcat
                         (fn [block-id]
-                          (let [block-id (if (and (string? block-id) (gp-util/uuid-string? block-id))
+                          (let [block-id (if (and (string? block-id) (util/uuid-string? block-id))
                                            [:block/uuid block-id]
                                            block-id)]
                             (when-let [block (db-utils/entity block-id)]
@@ -237,8 +238,15 @@
                                              (:db/id (:block/page block)))
                                     blocks [[::block (:block/uuid block)]]
                                     others (when page-id
-                                             [[::page-blocks page-id]
-                                              [::page->pages page-id]])]
+                                             (let [db-after-parent-uuid (:block/uuid (:block/parent block))
+                                                   db-before-parent-uuid (:block/uuid (:block/parent (d/entity db-before
+                                                                                                               [:block/uuid (:block/uuid block)])))]
+                                               [[::page-blocks page-id]
+                                                [::page->pages page-id]
+                                                [::block-direct-children db-after-parent-uuid]
+                                                (when (and db-before-parent-uuid
+                                                           (not= db-before-parent-uuid db-after-parent-uuid))
+                                                  [::block-direct-children db-before-parent-uuid])]))]
                                 (concat blocks others)))))
                         blocks)
 
@@ -251,7 +259,7 @@
                                 (if (:block/name entity) ; page
                                   [::page-blocks ref]
                                   [::page-blocks (:db/id (:block/page entity))])))
-                            refs))
+                         refs))
         others (->>
                 (keys @query-state)
                 (filter (fn [ks]

src/main/frontend/dicts.cljc

@@ -3073,28 +3073,28 @@
                           :default "tutorial-en.md")
         :tutorial/dummy-notes #?(:cljs (rc/inline "dummy-notes-en.md")
                                  :default "dummy-notes-en.md")
-        :on-boarding/demo-graph "Questo è un diagramma dimostrativo, le modifiche non saranno salvate finchè non aprirai una cartella locale."
-        :on-boarding/add-graph "Aggiungi un diagramma"
+        :on-boarding/demo-graph "Questo è un grafo dimostrativo, le modifiche non saranno salvate finché non aprirai una cartella locale."
+        :on-boarding/add-graph "Aggiungi un grafo"
         :on-boarding/open-local-dir "Apri una cartella locale"
-        :on-boarding/new-graph-desc-1 "Logseq supporta sia Markdown che Org-mode. Puoi aprire una cartella esistente o crearne una nuova sul tuo dispositivo. I tuoi dati saranno salvati in questo dispositivo."
+        :on-boarding/new-graph-desc-1 "Logseq supporta sia Markdown che Org-mode. Puoi aprire una cartella esistente o crearne una nuova sul tuo dispositivo. I tuoi dati saranno salvati solo sul tuo dispositivo."
         :on-boarding/new-graph-desc-2 "Dopo che hai aperto la tua cartella, saranno create al suo interno tre cartelle:"
-        :on-boarding/new-graph-desc-3 "/journals - conserva le tue pagine quotidiane"
-        :on-boarding/new-graph-desc-4 "/pages - conserva le altre pagine"
-        :on-boarding/new-graph-desc-5 "/logseq - conserva la configurazione, custom.css, e alcuni metadati."
-        :help/start "Per cominciare"
-        :help/about "Riguardo a Logseq"
+        :on-boarding/new-graph-desc-3 "/journals - contiene le pagine del diario giornaliero"
+        :on-boarding/new-graph-desc-4 "/pages - contiene le altre pagine"
+        :on-boarding/new-graph-desc-5 "/logseq - contiene i dati di configurazione, custom.css, e alcuni metadati."
+        :help/start "Per iniziare"
+        :help/about "Informazioni su Logseq"
         :help/roadmap "Roadmap"
-        :help/bug "Riportare un errore"
-        :help/feature "Richiedere una funzione"
+        :help/bug "Segnala un problema"
+        :help/feature "Richiedi una funzionalità"
         :help/changelog "Registro delle modifiche"
         :help/blog "blog di Logseq"
         :help/docs "Documentazione"
         :help/privacy "Politica sulla riservatezza"
-        :help/terms "Termini"
+        :help/terms "Termini di Servizio"
         :help/community "Comunità su Discord"
-        :help/awesome-logseq "Fantastico Logseq"
+        :help/awesome-logseq "Awesome Logseq"
         :help/shortcuts "Scorciatoie da tastiera"
-        :help/shortcuts-triggers "Inneschi"
+        :help/shortcuts-triggers "Attivazione delle scorciatoie"
         :help/shortcut "Scorciatoia"
         :help/slash-autocomplete "Barra di completamento automatico"
         :help/block-content-autocomplete "Autocompletamento del contenuto di blocco"
@@ -3106,12 +3106,12 @@
         :undo "Annulla"
         :redo "Ripeti"
         :general "Generale"
-        :more "Ancora"
+        :more "Altro"
         :search/result-for "Cerca i risultati per "
         :search/items "oggetti"
-        :search/page-names "Cerca nomi di pagina"
+        :search/page-names "Cerca pagine per nome"
         :help/context-menu "Menu contestuale del blocco"
-        :help/fold-unfold "Comprimi/Decomprimi blocchi (quando non sei in modalità di modifica)"
+        :help/fold-unfold "Comprimi/Espandi blocchi (quando non sei in modalità di modifica)"
         :help/markdown-syntax "Sintassi Markdown"
         :help/org-mode-syntax "Sintassi Org mode"
         :bold "Grassetto"
@@ -3131,17 +3131,17 @@
         :right-side-bar/block-ref "Riferimento di Blocco"
         :right-side-bar/graph-view "Vista del grafico"
         :right-side-bar/all-pages "Tutte le pagine"
-        :right-side-bar/flashcards "Carte flash"
+        :right-side-bar/flashcards "Flashcard"
         :right-side-bar/new-page "Nuova pagina"
-        :left-side-bar/journals "Diari"
+        :left-side-bar/journals "Diario"
         :left-side-bar/new-page "Nuova pagina"
         :left-side-bar/nav-favorites "Preferiti"
-        :left-side-bar/nav-shortcuts "Scrciatoie"
+        :left-side-bar/nav-shortcuts "Scorciatoie"
         :left-side-bar/nav-recent-pages "Recenti"
-        :format/preferred-mode "Qual'è la tua modalità preferita?"
+        :format/preferred-mode "Qual è la tua modalità preferita?"
         :format/markdown "Markdown"
         :format/org-mode "Org mode"
-        :reference/linked "Riferimento collegato"
+        :reference/linked "Riferimenti collegati"
         :reference/unlinked-ref "Riferimenti non collegati"
         :page/presentation-mode "Presentazione"
         :page/edit-properties-placeholder "Proprietà"
@@ -3152,41 +3152,41 @@
         :page/copy-to-json "Copia l'intera pagina in JSON"
         :page/rename "Rinomina pagina"
         :page/open-in-finder "Apri nella cartella"
-        :page/open-with-default-app "Apri con la app predefinita"
+        :page/open-with-default-app "Apri con l'app predefinita"
         :page/action-publish "Pubblica"
-        :page/make-public "Rendi pubblico per la pubblicazione"
+        :page/make-public "Segna come pubblico per la pubblicazione"
         :page/version-history "Controlla la cronologia della pagina"
         :page/open-backup-directory "Apri la cartella dei backup delle pagine"
         :page/file-sync-versions "Versioni delle pagine"
-        :page/make-private "Rendi privato"
+        :page/make-private "Segna come privato"
         :page/delete "Elimina pagina"
         :page/add-to-favorites "Aggiungi ai Preferiti"
         :page/unfavorite "Rimuovi la pagina dai Preferiti"
-        :page/show-journals "Mostra Diari"
+        :page/show-journals "Mostra diario"
         :page/show-name "Mostra il nome della pagina"
         :page/hide-name "Nascondi il nome della pagina"
         :block/name "Nome pagina"
         :page/last-modified "Ultima modifica alle"
-        :page/new-title "Qual'è il titolo della tua nuova pagina?"
+        :page/new-title "Qual è il titolo della tua nuova pagina?"
         :page/earlier "Prima"
         :page/no-more-journals "Non ci sono altri diari"
         :page/copy-page-url "Copia URL pagina"
-        :journal/multiple-files-with-different-formats "Sembra che tu abbia più file Diario (con formati diversi) per lo stesso mese, per favore conserva un solo file journal per ogni mese."
+        :journal/multiple-files-with-different-formats "Sembra che tu abbia più file diario (con formati diversi) per lo stesso mese, per favore conserva un solo file diario per ogni mese."
         :journal/go-to "Vai ai file"
         :file/name "Nome File"
         :file/file "File: "
         :file/last-modified-at "Ultima modifica alle"
-        :file/no-data "No dati"
+        :file/no-data "Nessun dato"
         :file/format-not-supported "Il formato .{1} non è supportato."
         :page/created-at "Creato alle"
         :page/updated-at "Aggiornato alle"
         :page/backlinks "Collegamenti a ritroso"
         :editor/block-search "Cerca un blocco"
-        :editor/image-uploading "Caricando"
+        :editor/image-uploading "Caricamento"
         :draw/invalid-file "Non ho potuto caricare questo file excalidraw"
         :draw/specify-title "Per favore specifica un titolo prima!"
         :draw/rename-success "Il file è stato rinominato con successo!"
-        :draw/rename-failure "La rinomina del file è fallita, ragione: "
+        :draw/rename-failure "La rinominazione del file è fallita, ragione: "
         :draw/title-placeholder "Senza titolo"
         :draw/save "Salva"
         :draw/save-changes "Salva modifiche"
@@ -3197,7 +3197,7 @@
         :draw/back-to-logseq "Torna a logseq"
         :text/image "Immagine"
         :asset/confirm-delete "Sei sicuro di voler eliminare questo {1}?"
-        :asset/physical-delete "Rimuovi anche i file (nota che non può essere ripristinato)"
+        :asset/physical-delete "Rimuovi anche i file (non possono essere ripristinati)"
         :content/copy "Copia"
         :content/cut "Taglia"
         :content/make-todos "Crea {1}"
@@ -3219,19 +3219,19 @@
         :settings-page/spell-checker "Correttore ortografico"
         :settings-page/auto-updater "Aggiornamento automatico"
         :settings-page/disable-sentry "Invia dati di utilizzo e diagnostica a Logseq"
-        :settings-page/preferred-outdenting "Distacco logico"
+        :settings-page/preferred-outdenting "Indentamento logico"
         :settings-page/custom-date-format "Formato data preferito"
         :settings-page/preferred-file-format "Formato file preferito"
         :settings-page/preferred-workflow "Flusso di lavoro preferito"
         :settings-page/enable-shortcut-tooltip "Abilita suggerimenti scorciatoie"
         :settings-page/enable-timetracking "Tracciamento del tempo"
         :settings-page/enable-tooltip "Suggerimenti"
-        :settings-page/enable-journals "Diari"
+        :settings-page/enable-journals "Diario"
         :settings-page/enable-all-pages-public "Tutte le pagine pubbliche durante la pubblicazione"
-        :settings-page/customize-shortcuts "Scorciatoie di tastiera"
+        :settings-page/customize-shortcuts "Scorciatoie da tastiera"
         :settings-page/shortcut-settings "Personalizza scorciatoie"
         :settings-page/home-default-page "Imposta la home page predefinita"
-        :settings-page/enable-block-time "Marche temporali sui blocchi"
+        :settings-page/enable-block-time "Indicatori temporali sui blocchi"
         :settings-page/clear-cache "Pulisci cache"
         :settings-page/clear "Pulisci"
         :settings-page/developer-mode "Modalità sviluppatore"
@@ -3239,19 +3239,19 @@
         :settings-page/disable-developer-mode "Disabilita modalità sviluppatore"
         :settings-page/developer-mode-desc "La modalità sviluppatore aiuta i contributori e gli sviluppatori di estensioni a testare le loro integrazioni con Logseq in modo più efficiente."
         :settings-page/current-version "Versione attuale"
-        :settings-page/current-graph "Diagramma attuale"
+        :settings-page/current-graph "Grafo attuale"
         :settings-page/tab-general "Generale"
         :settings-page/tab-editor "Editor"
         :settings-page/tab-shortcuts "Scorciatoie"
-        :settings-page/tab-version-control "Controllo versione"
+        :settings-page/tab-version-control "Controllo di versione"
         :settings-page/tab-advanced "Avanzate"
-        :settings-page/plugin-system "Sistema dei Plug-in"
+        :settings-page/plugin-system "Sistema di plugin"
         :settings-page/network-proxy "Proxy di rete"
         :logseq "Logseq"
         :on "ON"
         :more-options "Più opzioni"
         :to "a"
-        :yes "Si"
+        :yes "Sì"
         :no "No"
         :submit "Invia"
         :cancel "Annulla"
@@ -3262,63 +3262,63 @@
         :host "Host"
         :port "Porta"
         :re-index "Re-indicizza"
-        :re-index-detail "Ricostruisci il diagramma"
-        :re-index-multiple-windows-warning "È necessario chiudere le altre finestre prima di reindicizzare questo diagramma."
-        :re-index-discard-unsaved-changes-warning "Reindicizza elimina il diagramma corrente, quindi elabora nuovamente tutti i file, poiché sono attualmente archiviati su disco. Perderai le modifiche non salvate e potrebbe volerci del tempo. Continuare?"
+        :re-index-detail "Ricostruisci il grafo"
+        :re-index-multiple-windows-warning "È necessario chiudere le altre finestre prima di reindicizzare questo grafo."
+        :re-index-discard-unsaved-changes-warning "La reindicizzazione elimina il grafo corrente, quindi elabora nuovamente tutti i file poiché sono attualmente archiviati su disco. Perderai le modifiche non salvate e potrebbe volerci del tempo. Continuare?"
         :open-new-window "Nuova finestra"
         :sync-from-local-files "Ricarica"
         :sync-from-local-files-detail "Importa cambiamenti da un file locale"
-        :sync-from-local-changes-detected "Ricarica rileva ed elabora i file modificati sul disco e divergenti dal contenuto effettivo della pagina Logseq. Continuare?"
+        :sync-from-local-changes-detected "Il ricaricamento rileva ed elabora i file modificati sul disco e divergenti dal contenuto effettivo della pagina Logseq. Continuare?"
 
         :unlink "disconnetti"
         :search/publishing "Cerca"
         :search "Cerca o crea una pagina"
         :page-search "Cerca nella pagina corrente"
-        :graph-search "Cerca diagramma"
+        :graph-search "Cerca nel grafo"
         :new-page "Nuova pagina"
         :new-file "Nuovo file"
-        :new-graph "Aggiungi nuovo diagramma"
-        :graph "Diagramma"
-        :graph-view "Visualizza diagramma"
-        :graph/persist "Logseq sta sincronizzando lo stato interno, per favore aspetta molti secondi."
+        :new-graph "Aggiungi nuovo grafo"
+        :graph "Grafo"
+        :graph-view "Visualizza grafo"
+        :graph/persist "Logseq sta sincronizzando lo stato interno, per favore attendi alcuni secondi."
         :graph/persist-error "Sincronizzazione dello stato interno fallita."
-        :graph/save "Salvando..."
+        :graph/save "Salvataggio..."
         :graph/save-success "Salvato con successo"
         :graph/save-error "Salvataggio fallito"
-        :cards-view "Visualizza carte"
+        :cards-view "Visualizza flashcard"
         :publishing "Pubblicazione"
         :export "Esporta"
-        :export-graph "Esporta diagramma"
+        :export-graph "Esporta grafo"
         :export-page "Esporta pagina"
-        :export-markdown "Esporta come Markdown standard (nessuna proprietà di blocco)"
+        :export-markdown "Esporta come Markdown standard (senza le proprietà dei blocchi)"
         :export-opml "Esporta come OPML"
-        :export-public-pages "Esporta pagine pubbliche"
+        :export-public-pages "Esporta le pagine pubbliche"
         :export-json "Esporta come JSON"
         :export-roam-json "Esporta come Roam JSON"
         :export-edn "Esporta come EDN"
         :export-datascript-edn "Esporta datascript EDN"
         :convert-markdown "Converti le intestazioni di Markdown in elenchi non ordinati (# -> -)"
-        :all-graphs "Tutti i diagrammi"
+        :all-graphs "Tutti i grafi"
         :all-pages "Tutte le pagine"
         :all-files "Tutti i file"
         :remove-orphaned-pages "Rimuovi pagine orfane"
-        :all-journals "Tutti i diari"
+        :all-journals "Tutte le pagine di diario"
         :my-publishing "Le mie pubblicazioni"
         :settings "Impostazioni"
         :settings-of-plugins "Impostazioni plugin"
         :plugins "Plugin"
         :themes "Temi"
-        :developer-mode-alert "È necessario riavviare l'app per abilitare il plug-in. Vuoi riavviarlo ora?"
-        :relaunch-confirm-to-work "Bisogna riavviare l'app per farla funzionare. Vuoi riavviarla ora?"
+        :developer-mode-alert "È necessario riavviare l'app per abilitare il plugin. Vuoi riavviarla ora?"
+        :relaunch-confirm-to-work "È necessario riavviare l'app per farla funzionare. Vuoi riavviarla ora?"
         :import "Importa"
         :join-community "Unisciti alla comunità"
-        :sponsor-us "Sponsorizzaci"
+        :sponsor-us "Supportaci"
         :discord-title "Il nostro gruppo Discord!"
-        :help-shortcut-title "Clicca per controllare le scorciatoie e altri suggerimenti"
-        :loading "Caricando"
-        :cloning "Clonando"
-        :parsing-files "Analizzando i file"
-        :loading-files "Caricando i file"
+        :help-shortcut-title "Clicca per conoscere le scorciatoie e altri suggerimenti"
+        :loading "Caricamento"
+        :cloning "Clonazione"
+        :parsing-files "Analisi dei file"
+        :loading-files "Caricamento dei file"
         :login "Accedi"
         :logout "Esci"
         :go-to "Vai a "
@@ -3331,41 +3331,41 @@
         :remove-background "Rimuovi lo sfondo"
         :open "Apri"
         :open-a-directory "Apri una cartella locale"
-        :user/delete-account "Elimina account"
-        :user/delete-your-account "Elimina il tuo account"
+        :user/delete-account "Elimina profilo"
+        :user/delete-your-account "Elimina il tuo profilo"
         :user/delete-account-notice "Tutte le tue pagine pubblicate su logseq.com saranno eliminate."
 
-        :help/shortcut-page-title "Scorciatoie di tastiera"
+        :help/shortcut-page-title "Scorciatoie da tastiera"
 
         :plugin/installed "Installato"
         :plugin/not-installed "Non installato"
-        :plugin/installing "Installando"
+        :plugin/installing "Installazione"
         :plugin/install "Installa"
         :plugin/reload "Ricarica"
         :plugin/update "Aggiorna"
         :plugin/check-update "Controlla aggiornamenti"
         :plugin/check-all-updates "Controlla tutti gli aggiornamenti"
-        :plugin/refresh-lists "Ricarica liste"
+        :plugin/refresh-lists "Ricarica lista"
         :plugin/enabled "Abilitato"
         :plugin/disabled "Disabilitato"
         :plugin/update-available "Aggiornamento disponibile"
-        :plugin/updating "Aggiornando"
-        :plugin/uninstall "Disinstallare"
-        :plugin/marketplace "Mercato"
-        :plugin/downloads "Downloads"
+        :plugin/updating "Aggiornamento"
+        :plugin/uninstall "Disinstalla"
+        :plugin/marketplace "Libreria"
+        :plugin/downloads "Numero di scaricamenti"
         :plugin/stars "Stelle"
         :plugin/title "Titolo"
         :plugin/all "Tutti"
-        :plugin/unpacked "Spacchettati"
-        :plugin/delete-alert "Sei sicuro di disinstallare [{1}]?"
+        :plugin/unpacked "Non pacchettizzati"
+        :plugin/delete-alert "Sei sicuro di voler disinstallare [{1}]?"
         :plugin/open-settings "Apri impostazioni"
         :plugin/open-package "Apri pacchetto"
-        :plugin/load-unpacked "Carica plugin non impacchettato"
-        :plugin/open-preferences "Apri il file preferenze del plugin"
-        :plugin/restart "Riavvia App"
+        :plugin/load-unpacked "Carica plugin non pacchettizzato"
+        :plugin/open-preferences "Apri il file delle preferenze del plugin"
+        :plugin/restart "Riavvia app"
         :plugin/unpacked-tips "Seleziona la cartella del plugin"
-        :plugin/contribute "✨ Scrivi e sottoponi un nuovo plugin"
-        :plugin/marketplace-tips "Se il plug-in non funziona correttamente alla prima installazione, provare a riavviare Logseq."
+        :plugin/contribute "✨ Svilupppa e sottoponici un nuovo plugin"
+        :plugin/marketplace-tips "Se il plugin non funziona correttamente alla prima installazione, provare a riavviare Logseq."
         :plugin/up-to-date "È aggiornato"
         :plugin/custom-js-alert "Trovato il file custom.js, è consentito eseguirlo? (Se non si comprende il contenuto di questo file, si consiglia di non consentire l'esecuzione, che presenta alcuni rischi per la sicurezza.)"
 
@@ -3374,23 +3374,23 @@
         :pdf/linked-ref "Riferimenti collegati"
         :pdf/toggle-dashed "Stile tratteggiato per evidenziare l'area"
 
-        :updater/new-version-install "Una nuova versione è stata caricata."
-        :updater/quit-and-install "Riavvia per installare"
+        :updater/new-version-install "Una nuova versione è stata scaricata."
+        :updater/quit-and-install "Riavvia per installarla"
 
         :paginates/pages "Totale {1} pagine"
         :paginates/prev "Precedente"
-        :paginates/next "Prossimo"
+        :paginates/next "Successivo"
 
-        :tips/all-done "Tutto Fatto"
+        :tips/all-done "Completato"
 
-        :command-palette/prompt "Scrivi un comando"
-        :select/default-prompt "Seleziona uno"
-        :select.graph/prompt "Seleziona un diagramma"
-        :select.graph/empty-placeholder-description "Non ci sono diagrammi corrispondenti. Vuoi aggiungerne uno nuovo?"
-        :select.graph/add-graph "Si, aggiungi un nuovo diagramma"
+        :command-palette/prompt "Digita un comando"
+        :select/default-prompt "Selezionane uno"
+        :select.graph/prompt "Seleziona un grafo"
+        :select.graph/empty-placeholder-description "Non ci sono grafi corrispondenti. Vuoi aggiungerne uno nuovo?"
+        :select.graph/add-graph "Sì, aggiungi un nuovo grafo"
 
-        :file-sync/other-user-graph "Il diagramma locale attuale è associato al diagramma remoto di un altro utente. Quindi non è possibile avviare la sincronizzazione."
-        :file-sync/graph-deleted "Il diagramma attuale è stato eliminato"}
+        :file-sync/other-user-graph "Il grafo locale attuale è associato al grafo remoto di un altro utente. Non è quindi possibile avviare la sincronizzazione."
+        :file-sync/graph-deleted "Il grafo attuale è stato eliminato"}
 
    :tr {:tutorial/text #?(:cljs (rc/inline "tutorial-tr.md")
                                 :default "tutorial-tr.md")
@@ -3542,7 +3542,7 @@
         :settings-page/spell-checker "Yazım denetleyici"
         :settings-page/auto-updater "Otomatik güncelleme"
         :settings-page/disable-sentry "Kullanım verilerini ve tanılamayı Logseq'e gönderin"
-        :settings-page/preferred-outdenting "mantıksal girinti"
+        :settings-page/preferred-outdenting "Mantıksal girinti"
         :settings-page/custom-date-format "Tercih edilen tarih biçimi"
         :settings-page/preferred-file-format "Tercih edilen dosya biçimi"
         :settings-page/preferred-workflow "Tercih edilen iş akışı"
@@ -3551,6 +3551,7 @@
         :settings-page/enable-tooltip "Araç ipuçları"
         :settings-page/enable-journals "Günlük"
         :settings-page/enable-all-pages-public "Yayımlanan tüm sayfaları herkese açık yap"
+        :settings-page/enable-encryption "Şifreleme"
         :settings-page/customize-shortcuts "Klavye kısayolları"
         :settings-page/shortcut-settings "Kısayolları özelleştir"
         :settings-page/home-default-page "Varsayılan ana sayfayı ayarla"

src/main/frontend/diff.cljs

@@ -6,7 +6,7 @@
             [cljs-bean.core :as bean]
             [frontend.util :as util]
             [logseq.graph-parser.util :as gp-util]
-            [frontend.text :as text]))
+            [logseq.graph-parser.text :as text]))
 
 (defn diff
   [s1 s2]

src/main/frontend/encrypt.cljs

@@ -1,5 +1,5 @@
 (ns frontend.encrypt
-  (:require [frontend.utf8 :as utf8]
+  (:require [logseq.graph-parser.utf8 :as utf8]
             [frontend.db.utils :as db-utils]
             [frontend.db :as db]
             [promesa.core :as p]

src/main/frontend/extensions/code.cljs

@@ -132,7 +132,7 @@
             [frontend.handler.editor :as editor-handler]
             [frontend.handler.file :as file-handler]
             [frontend.state :as state]
-            [frontend.utf8 :as utf8]
+            [logseq.graph-parser.utf8 :as utf8]
             [frontend.util :as util]
             [frontend.config :as config]
             [goog.dom :as gdom]

src/main/frontend/extensions/html_parser.cljs

@@ -4,7 +4,6 @@
             [clojure.walk :as walk]
             [frontend.config :as config]
             [frontend.util :as util]
-            [logseq.graph-parser.util :as gp-util]
             [hickory.core :as hickory]))
 
 (defonce *inside-pre? (atom false))
@@ -75,7 +74,7 @@
                                 :h6 (block-transform 6 children)
                                 :a (let [href (:href attrs)
                                          label (map-join children)
-                                         has-img-tag? (gp-util/safe-re-find #"\[:img" (str x))]
+                                         has-img-tag? (util/safe-re-find #"\[:img" (str x))]
                                      (if has-img-tag?
                                        (export-hiccup x)
                                        (case format

src/main/frontend/external/roam.cljs

@@ -7,7 +7,7 @@
             [clojure.string :as string]
             [frontend.util :as util]
             [logseq.graph-parser.util :as gp-util]
-            [frontend.text :as text]))
+            [logseq.graph-parser.text :as text]))
 
 (defonce all-refed-uids (atom #{}))
 (defonce uid->uuid (atom {}))

src/main/frontend/format.cljs

@@ -2,30 +2,21 @@
   (:require [frontend.format.mldoc :refer [->MldocMode] :as mldoc]
             [frontend.format.adoc :refer [->AdocMode]]
             [frontend.format.protocol :as protocol]
-            [frontend.text :as text]
+            [logseq.graph-parser.mldoc :as gp-mldoc]
+            [logseq.graph-parser.text :as text]
+            [logseq.graph-parser.util :as gp-util]
             [clojure.string :as string]))
 
-(set! mldoc/parse-property text/parse-property)
+;; TODO: Properly fix this circular dependency:
+;; mldoc/->edn > text/parse-property > mldoc/link? ->mldoc/inline->edn + mldoc/default-config
+(set! gp-mldoc/parse-property text/parse-property)
 
 (defonce mldoc-record (->MldocMode))
 (defonce adoc-record (->AdocMode))
 
-(defn normalize
-  [format]
-  (case (keyword format)
-    :md :markdown
-    :asciidoc :adoc
-    ;; default
-    (keyword format)))
-
-(defn get-format
-  [file]
-  (when file
-    (normalize (keyword (string/lower-case (last (string/split file #"\.")))))))
-
 (defn get-format-record
   [format]
-  (case (normalize format)
+  (case (gp-util/normalize-format format)
     :org
     mldoc-record
     :markdown
@@ -37,9 +28,9 @@
 ;; html
 (defn get-default-config
   ([format]
-   (mldoc/default-config format))
+   (gp-mldoc/default-config format))
   ([format options]
-   (mldoc/default-config format options)))
+   (gp-mldoc/default-config format options)))
 
 (defn to-html
   ([content format]
@@ -49,7 +40,7 @@
      (if (string/blank? content)
        ""
        (if-let [record (get-format-record format)]
-         (protocol/toHtml record content config mldoc/default-references)
+         (protocol/toHtml record content config gp-mldoc/default-references)
          content)))))
 
 (defn to-edn

src/main/frontend/format/block.cljs

@@ -1,665 +1,30 @@
 (ns frontend.format.block
+  "Block code needed by app but not graph-parser"
   (:require [clojure.string :as string]
-            [clojure.walk :as walk]
-            [cljs.core.match :as match]
+            [logseq.graph-parser.block :as gp-block]
             [frontend.config :as config]
-            [frontend.date :as date]
             [frontend.db :as db]
             [frontend.format :as format]
             [frontend.state :as state]
-            [frontend.text :as text]
-            [frontend.utf8 :as utf8]
-            [frontend.util :as util]
-            [frontend.util.property :as property]
-            [logseq.graph-parser.util :as gp-util]
-            [logseq.graph-parser.config :as gp-config]
-            [lambdaisland.glogi :as log]
-            [medley.core :as medley]
-            [frontend.format.mldoc :as mldoc]))
+            [logseq.graph-parser.property :as gp-property]
+            [logseq.graph-parser.mldoc :as gp-mldoc]))
 
-(defn heading-block?
-  [block]
-  (and
-   (vector? block)
-   (= "Heading" (first block))))
-
-(defn get-tag
-  [block]
-  (when-let [tag-value (and (vector? block)
-                            (= "Tag" (first block))
-                            (second block))]
-    (->
-     (map (fn [e]
-            (match/match e
-              ["Plain" s]
-              s
-              ["Link" t]
-              (let [{full_text :full_text} t]
-                full_text)
-              ["Nested_link" t]
-              (let [ {content :content} t]
-                content)
-              :else
-              ""
-              )) tag-value)
-     (string/join))))
-
-(defn get-page-reference
-  [block]
-  (let [page (cond
-               (and (vector? block) (= "Link" (first block)))
-               (let [typ (first (:url (second block)))
-                     value (second (:url (second block)))]
-                 ;; {:url ["File" "file:../pages/hello_world.org"], :label [["Plain" "hello world"]], :title nil}
-                 (or
-                  (and
-                   (= typ "Page_ref")
-                   (and (string? value)
-                        (not (or (gp-config/local-asset? value)
-                                 (gp-config/draw? value))))
-                   value)
-
-                  (and
-                   (= typ "Search")
-                   (text/page-ref? value)
-                   (text/page-ref-un-brackets! value))
-
-                  (and
-                   (= typ "Search")
-                   (not (contains? #{\# \* \/ \[} (first value)))
-                   (let [ext (some-> (util/get-file-ext value) keyword)]
-                     (when (and (not (util/starts-with? value "http:"))
-                                (not (util/starts-with? value "https:"))
-                                (not (util/starts-with? value "file:"))
-                                (not (gp-config/local-asset? value))
-                                (or (some #{ext} [:excalidraw :tldraw])
-                                    (not (contains? (config/supported-formats) ext))))
-                       value)))
-
-                  (and
-                   (= typ "Complex")
-                   (= (:protocol value) "file")
-                   (:link value))
-
-                  (and
-                   (= typ "File")
-                   (second (first (:label (second block)))))))
-
-               (and (vector? block) (= "Nested_link" (first block)))
-               (let [content (:content (last block))]
-                 (subs content 2 (- (count content) 2)))
-
-               (and (vector? block)
-                    (= "Macro" (first block)))
-               (let [{:keys [name arguments]} (second block)
-                     argument (string/join ", " arguments)]
-                   (when (= name "embed")
-                     (text/page-ref-un-brackets! argument)))
-
-               (and (vector? block)
-                    (= "Tag" (first block)))
-               (let [text (get-tag block)]
-                 (text/page-ref-un-brackets! text))
-
-               :else
-               nil)]
-    (text/block-ref-un-brackets! page)))
-
-(defn get-block-reference
-  [block]
-  (when-let [block-id (cond
-                        (and (vector? block)
-                             (= "Block_reference" (first block)))
-                        (last block)
-
-                        (and (vector? block)
-                             (= "Link" (first block))
-                             (map? (second block))
-                             (= "Block_ref" (first (:url (second block)))))
-                        (second (:url (second block)))
-
-                        (and (vector? block)
-                             (= "Macro" (first block)))
-                        (let [{:keys [name arguments]} (second block)]
-                          (when (and (= name "embed")
-                                     (string? (first arguments))
-                                     (string/starts-with? (first arguments) "((")
-                                     (string/ends-with? (first arguments) "))"))
-                            (subs (first arguments) 2 (- (count (first arguments)) 2))))
-
-                        (and (vector? block)
-                             (= "Link" (first block))
-                             (map? (second block)))
-                        (if (= "id" (:protocol (second (:url (second block)))))
-                          (:link (second (:url (second block))))
-                          (let [id (second (:url (second block)))]
-                            (text/block-ref-un-brackets! id)))
-
-                        :else
-                        nil)]
-    (when (and block-id
-               (gp-util/uuid-string? block-id))
-      block-id)))
-
-(defn paragraph-block?
-  [block]
-  (and
-   (vector? block)
-   (= "Paragraph" (first block))))
-
-(defn timestamp-block?
-  [block]
-  (and
-   (vector? block)
-   (= "Timestamp" (first block))))
-
-;; TODO: we should move this to mldoc
-(defn extract-properties
-  [format properties]
-  (when (seq properties)
-    (let [properties (seq properties)
-          page-refs (->>
-                     properties
-                     (remove (fn [[k _]]
-                               (contains? #{:background-color :background_color} (keyword k))))
-                     (map last)
-                     (map (fn [v]
-                            (when (and (string? v)
-                                       (not (mldoc/link? format v)))
-                              (let [v (string/trim v)
-                                    result (text/split-page-refs-without-brackets v {:un-brackets? false})]
-                                (if (coll? result)
-                                  (map text/page-ref-un-brackets! result)
-                                  [])))))
-                     (apply concat)
-                     (remove string/blank?))
-          properties (->> properties
-                          (map (fn [[k v]]
-                                 (let [k (-> (string/lower-case (name k))
-                                             (string/replace " " "-")
-                                             (string/replace "_" "-"))
-                                       k (if (contains? #{"custom_id" "custom-id"} k)
-                                           "id"
-                                           k)
-                                       v (if (coll? v)
-                                           (remove string/blank? v)
-                                           (if (string/blank? v)
-                                             nil
-                                             (text/parse-property format k v)))
-                                       k (keyword k)
-                                       v (if (and
-                                              (string? v)
-                                              (contains? #{:alias :aliases :tags} k))
-                                           (set [v])
-                                           v)
-                                       v (if (coll? v) (set v) v)]
-                                   [k v])))
-                          (remove #(nil? (second %))))]
-      {:properties (into {} properties)
-       :properties-order (map first properties)
-       :page-refs page-refs})))
-
-(defn- paragraph-timestamp-block?
-  [block]
-  (and (paragraph-block? block)
-       (or (timestamp-block? (first (second block)))
-           (timestamp-block? (second (second block))))))
-
-(defn extract-timestamps
-  [block]
-  (some->>
-   (second block)
-   (filter timestamp-block?)
-   (map last)
-   (into {})))
-
-;; {"Deadline" {:date {:year 2020, :month 10, :day 20}, :wday "Tue", :time {:hour 8, :min 0}, :repetition [["DoublePlus"] ["Day"] 1], :active true}}
-(defn timestamps->scheduled-and-deadline
-  [timestamps]
-  (let [timestamps (medley/map-keys (comp keyword string/lower-case) timestamps)
-        m (some->> (select-keys timestamps [:scheduled :deadline])
-                   (map (fn [[k v]]
-                          (let [{:keys [date repetition]} v
-                                {:keys [year month day]} date
-                                day (js/parseInt (str year (util/zero-pad month) (util/zero-pad day)))]
-                            (cond->
-                             (case k
-                               :scheduled
-                               {:scheduled day}
-                               :deadline
-                               {:deadline day})
-                              repetition
-                              (assoc :repeated? true))))))]
-    (apply merge m)))
-
-(defn convert-page-if-journal
-  "Convert journal file name to user' custom date format"
-  [original-page-name]
-  (when original-page-name
-    (let [page-name (util/page-name-sanity-lc original-page-name)
-          day (date/journal-title->int page-name)]
-     (if day
-       (let [original-page-name (date/int->journal-title day)]
-         [original-page-name (util/page-name-sanity-lc original-page-name) day])
-       [original-page-name page-name day]))))
+(defn extract-blocks
+  "Wrapper around logseq.graph-parser.block/extract-blocks that adds in system state"
+  [blocks content with-id? format]
+  (gp-block/extract-blocks blocks content with-id? format
+                           {:user-config (state/get-config)
+                            :block-pattern (config/get-block-pattern format)
+                            :supported-formats (config/supported-formats)
+                            :db (db/get-db (state/get-current-repo))
+                            :date-formatter (state/get-date-formatter)}))
 
 (defn page-name->map
-  "Create a page's map structure given a original page name (string).
-   map as input is supported for legacy compatibility.
-   with-timestamp?: assign timestampes to the map structure.
-    Useful when creating new pages from references or namespaces,
-    as there's no chance to introduce timestamps via editing in page"
+  "Wrapper around logseq.graph-parser.block/page-name->map that adds in db"
   ([original-page-name with-id?]
    (page-name->map original-page-name with-id? true))
   ([original-page-name with-id? with-timestamp?]
-   (cond
-     (and original-page-name (string? original-page-name))
-     (let [original-page-name (util/remove-boundary-slashes original-page-name)
-           [original-page-name page-name journal-day] (convert-page-if-journal original-page-name)
-           namespace? (and (not (boolean (text/get-nested-page-name original-page-name)))
-                           (text/namespace-page? original-page-name))
-           page-entity (db/entity [:block/name page-name])
-           original-page-name (or (:block/original-name page-entity) original-page-name)]
-       (merge
-        {:block/name page-name
-         :block/original-name original-page-name}
-        (when with-id?
-          (if page-entity
-            {:block/uuid (:block/uuid page-entity)}
-            {:block/uuid (db/new-block-id)}))
-        (when namespace?
-          (let [namespace (first (gp-util/split-last "/" original-page-name))]
-            (when-not (string/blank? namespace)
-              {:block/namespace {:block/name (util/page-name-sanity-lc namespace)}})))
-        (when (and with-timestamp? (not page-entity)) ;; Only assign timestamp on creating new entity
-          (let [current-ms (util/time-ms)]
-            {:block/created-at current-ms
-             :block/updated-at current-ms}))
-        (if journal-day
-          {:block/journal? true
-           :block/journal-day journal-day}
-          {:block/journal? false})))
-
-     (and (map? original-page-name) (:block/uuid original-page-name))
-     original-page-name
-
-     (and (map? original-page-name) with-id?)
-     (assoc original-page-name :block/uuid (db/new-block-id))
-
-     :else
-     nil)))
-
-(defn with-page-refs
-  [{:keys [title body tags refs marker priority] :as block} with-id?]
-  (let [refs (->> (concat tags refs [marker priority])
-                  (remove string/blank?)
-                  (distinct))
-        refs (atom refs)]
-    (walk/prewalk
-     (fn [form]
-       ;; skip custom queries
-       (when-not (and (vector? form)
-                      (= (first form) "Custom")
-                      (= (second form) "query"))
-         (when-let [page (get-page-reference form)]
-           (swap! refs conj page))
-         (when-let [tag (get-tag form)]
-           (let [tag (text/page-ref-un-brackets! tag)]
-             (when (gp-util/tag-valid? tag)
-               (swap! refs conj tag))))
-         form))
-     (concat title body))
-    (let [refs (remove string/blank? @refs)
-          children-pages (->> (mapcat (fn [p]
-                                        (let [p (if (map? p)
-                                                  (:block/original-name p)
-                                                  p)]
-                                          (when (string? p)
-                                            (let [p (or (text/get-nested-page-name p) p)]
-                                              (when (text/namespace-page? p)
-                                                (util/split-namespace-pages p))))))
-                                      refs)
-                              (remove string/blank?)
-                              (distinct))
-          refs (->> (distinct (concat refs children-pages))
-                    (remove nil?))
-          refs (map (fn [ref] (page-name->map ref with-id?)) refs)]
-      (assoc block :refs refs))))
-
-(defn with-block-refs
-  [{:keys [title body] :as block}]
-  (let [ref-blocks (atom nil)]
-    (walk/postwalk
-     (fn [form]
-       (when-let [block (get-block-reference form)]
-         (swap! ref-blocks conj block))
-       form)
-     (concat title body))
-    (let [ref-blocks (->> @ref-blocks
-                          (filter gp-util/uuid-string?))
-          ref-blocks (map
-                       (fn [id]
-                         [:block/uuid (medley/uuid id)])
-                       ref-blocks)
-          refs (distinct (concat (:refs block) ref-blocks))]
-      (assoc block :refs refs))))
-
-(defn- block-keywordize
-  [block]
-  (medley/map-keys
-   (fn [k]
-     (if (namespace k)
-       k
-       (keyword "block" k)))
-   block))
-
-(defn- sanity-blocks-data
-  [blocks]
-  (map (fn [block]
-         (if (map? block)
-           (block-keywordize (gp-util/remove-nils block))
-           block))
-       blocks))
-
-(defn with-path-refs
-  [blocks]
-  (loop [blocks blocks
-         acc []
-         parents []]
-    (if (empty? blocks)
-      acc
-      (let [block (first blocks)
-            cur-level (:block/level block)
-            level-diff (- cur-level
-                          (get (last parents) :block/level 0))
-            [path-refs parents]
-            (cond
-              (zero? level-diff)            ; sibling
-              (let [path-refs (mapcat :block/refs (drop-last parents))
-                    parents (conj (vec (butlast parents)) block)]
-                [path-refs parents])
-
-              (> level-diff 0)              ; child
-              (let [path-refs (mapcat :block/refs parents)]
-                [path-refs (conj parents block)])
-
-              (< level-diff 0)              ; new parent
-              (let [parents (vec (take-while (fn [p] (< (:block/level p) cur-level)) parents))
-                    path-refs (mapcat :block/refs parents)]
-                [path-refs (conj parents block)]))
-            path-ref-pages (->> path-refs
-                                (concat (:block/refs block))
-                                (map (fn [ref]
-                                       (cond
-                                         (map? ref)
-                                         (:block/name ref)
-
-                                         :else
-                                         ref)))
-                                (remove string/blank?)
-                                (map (fn [ref]
-                                       (if (string? ref)
-                                         {:block/name (util/page-name-sanity-lc ref)}
-                                         ref)))
-                                (remove vector?)
-                                (remove nil?)
-                                (distinct))]
-        (recur (rest blocks)
-               (conj acc (assoc block :block/path-refs path-ref-pages))
-               parents)))))
-
-(defn block-tags->pages
-  [{:keys [tags] :as block}]
-  (if (seq tags)
-    (assoc block :tags (map (fn [tag]
-                              (let [tag (text/page-ref-un-brackets! tag)]
-                                [:block/name (util/page-name-sanity-lc tag)])) tags))
-    block))
-
-(defn- get-block-content
-  [utf8-content block format meta]
-  (let [content (if-let [end-pos (:end_pos meta)]
-                  (utf8/substring utf8-content
-                                  (:start_pos meta)
-                                  end-pos)
-                  (utf8/substring utf8-content
-                                  (:start_pos meta)))
-        content (when content
-                  (let [content (text/remove-level-spaces content format)]
-                    (if (or (:pre-block? block)
-                            (= (:format block) :org))
-                      content
-                      (mldoc/remove-indentation-spaces content (inc (:level block)) false))))]
-    (if (= format :org)
-      content
-      (property/->new-properties content))))
-
-(defn get-custom-id-or-new-id
-  [properties]
-  (or (when-let [custom-id (or (get-in properties [:properties :custom-id])
-                               (get-in properties [:properties :custom_id])
-                               (get-in properties [:properties :id]))]
-        (let [custom-id (and (string? custom-id) (string/trim custom-id))]
-          (when (and custom-id (gp-util/uuid-string? custom-id))
-            (uuid custom-id))))
-      (db/new-block-id)))
-
-(defn get-page-refs-from-properties
-  [properties]
-  (let [page-refs (mapcat (fn [v] (cond
-                                   (coll? v)
-                                   v
-
-                                   (text/page-ref? v)
-                                   [(text/page-ref-un-brackets! v)]
-
-                                   :else
-                                   nil)) (vals properties))
-        page-refs (remove string/blank? page-refs)]
-    (map (fn [page] (page-name->map page true)) page-refs)))
-
-(defn with-page-block-refs
-  [block with-id?]
-  (some-> block
-          (with-page-refs with-id?)
-          with-block-refs
-          block-tags->pages
-          (update :refs (fn [col] (remove nil? col)))))
-
-(defn with-pre-block-if-exists
-  [blocks body pre-block-properties encoded-content]
-  (let [first-block (first blocks)
-        first-block-start-pos (get-in first-block [:block/meta :start_pos])
-
-        ;; Add pre-block
-        blocks (if (or (> first-block-start-pos 0)
-                       (empty? blocks))
-                 (cons
-                  (merge
-                   (let [content (utf8/substring encoded-content 0 first-block-start-pos)
-                         {:keys [properties properties-order]} pre-block-properties
-                         id (get-custom-id-or-new-id {:properties properties})
-                         property-refs (->> (get-page-refs-from-properties properties)
-                                            (map :block/original-name))
-                         block {:uuid id
-                                :content content
-                                :level 1
-                                :properties properties
-                                :properties-order properties-order
-                                :refs property-refs
-                                :pre-block? true
-                                :unordered true
-                                :body body}
-                         block (with-page-block-refs block false)]
-                     (block-keywordize block))
-                   (select-keys first-block [:block/format :block/page]))
-                  blocks)
-                 blocks)]
-    (with-path-refs blocks)))
-
-(defn- construct-block
-  [block properties timestamps body encoded-content format pos-meta with-id?]
-  (let [id (get-custom-id-or-new-id properties)
-        ref-pages-in-properties (->> (:page-refs properties)
-                                     (remove string/blank?))
-        block (second block)
-        unordered? (:unordered block)
-        markdown-heading? (and (:size block) (= :markdown format))
-        block (if markdown-heading?
-                (assoc block
-                       :type :heading
-                       :level (if unordered? (:level block) 1)
-                       :heading-level (or (:size block) 6))
-                block)
-        block (cond->
-                (assoc block
-                       :uuid id
-                       :refs ref-pages-in-properties
-                       :format format
-                       :meta pos-meta)
-                (seq (:properties properties))
-                (assoc :properties (:properties properties))
-
-                (seq (:properties-order properties))
-                (assoc :properties-order (:properties-order properties)))
-        block (if (get-in block [:properties :collapsed])
-                (assoc block :collapsed? true)
-                block)
-        block (assoc block
-                     :content (get-block-content encoded-content block format pos-meta))
-        block (if (seq timestamps)
-                (merge block (timestamps->scheduled-and-deadline timestamps))
-                block)
-        block (assoc block :body body)
-        block (with-page-block-refs block with-id?)
-        {:keys [created-at updated-at]} (:properties properties)
-        block (cond-> block
-                (and created-at (integer? created-at))
-                (assoc :block/created-at created-at)
-
-                (and updated-at (integer? updated-at))
-                (assoc :block/updated-at updated-at))]
-    (dissoc block :title :body :anchor)))
-
-(defn extract-blocks
-  "Extract headings from mldoc ast.
-  Args:
-    `blocks`: mldoc ast.
-    `content`: markdown or org-mode text.
-    `with-id?`: If `with-id?` equals to true, all the referenced pages will have new db ids.
-    `format`: content's format, it could be either :markdown or :org-mode."
-  [blocks content with-id? format]
-  {:pre [(seq blocks) (string? content) (boolean? with-id?) (contains? #{:markdown :org} format)]}
-  (try
-    (let [encoded-content (utf8/encode content)
-          [blocks body pre-block-properties]
-          (loop [headings []
-                 blocks (reverse blocks)
-                 timestamps {}
-                 properties {}
-                 body []]
-            (if (seq blocks)
-              (let [[block pos-meta] (first blocks)
-                    ;; fix start_pos
-                    pos-meta (assoc pos-meta :end_pos
-                                    (if (seq headings)
-                                      (get-in (last headings) [:meta :start_pos])
-                                      nil))]
-                (cond
-                  (paragraph-timestamp-block? block)
-                  (let [timestamps (extract-timestamps block)
-                        timestamps' (merge timestamps timestamps)]
-                    (recur headings (rest blocks) timestamps' properties body))
-
-                  (property/properties-ast? block)
-                  (let [properties (extract-properties format (second block))]
-                    (recur headings (rest blocks) timestamps properties body))
-
-                  (heading-block? block)
-                  (let [block (construct-block block properties timestamps body encoded-content format pos-meta with-id?)]
-                    (recur (conj headings block) (rest blocks) {} {} []))
-
-                  :else
-                  (recur headings (rest blocks) timestamps properties (conj body block))))
-              [(-> (reverse headings)
-                   sanity-blocks-data)
-               body
-               properties]))
-          result (with-pre-block-if-exists blocks body pre-block-properties encoded-content)]
-      (map #(dissoc % :block/meta) result))
-    (catch js/Error e
-      (js/console.error "extract-blocks-failed")
-      (log/error :exception e))))
-
-(defn with-parent-and-left
-  [page-id blocks]
-  (loop [blocks (map (fn [block] (assoc block :block/level-spaces (:block/level block))) blocks)
-         parents [{:page/id page-id     ; db id or a map {:block/name "xxx"}
-                   :block/level 0
-                   :block/level-spaces 0}]
-         result []]
-    (if (empty? blocks)
-      (map #(dissoc % :block/level-spaces) result)
-      (let [[block & others] blocks
-            level-spaces (:block/level-spaces block)
-            {:block/keys [uuid level parent] :as last-parent} (last parents)
-            parent-spaces (:block/level-spaces last-parent)
-            [blocks parents result]
-            (cond
-              (= level-spaces parent-spaces)        ; sibling
-              (let [block (assoc block
-                                 :block/parent parent
-                                 :block/left [:block/uuid uuid]
-                                 :block/level level)
-                    parents' (conj (vec (butlast parents)) block)
-                    result' (conj result block)]
-                [others parents' result'])
-
-              (> level-spaces parent-spaces)         ; child
-              (let [parent (if uuid [:block/uuid uuid] (:page/id last-parent))
-                    block (cond->
-                            (assoc block
-                                  :block/parent parent
-                                  :block/left parent)
-                            ;; fix block levels with wrong order
-                            ;; For example:
-                            ;;   - a
-                            ;; - b
-                            ;; What if the input indentation is two spaces instead of 4 spaces
-                            (>= (- level-spaces parent-spaces) 1)
-                            (assoc :block/level (inc level)))
-                    parents' (conj parents block)
-                    result' (conj result block)]
-                [others parents' result'])
-
-              (< level-spaces parent-spaces)
-              (cond
-                (some #(= (:block/level-spaces %) (:block/level-spaces block)) parents) ; outdent
-                (let [parents' (vec (filter (fn [p] (<= (:block/level-spaces p) level-spaces)) parents))
-                      left (last parents')
-                      blocks (cons (assoc (first blocks)
-                                          :block/level (dec level)
-                                          :block/left [:block/uuid (:block/uuid left)])
-                                   (rest blocks))]
-                  [blocks parents' result])
-
-                :else
-                (let [[f r] (split-with (fn [p] (<= (:block/level-spaces p) level-spaces)) parents)
-                      left (first r)
-                      parent-id (if-let [block-id (:block/uuid (last f))]
-                                  [:block/uuid block-id]
-                                  page-id)
-                      block (cond->
-                              (assoc block
-                                     :block/parent parent-id
-                                     :block/left [:block/uuid (:block/uuid left)]
-                                     :block/level (:block/level left)
-                                     :block/level-spaces (:block/level-spaces left)))
-
-                      parents' (->> (concat f [block]) vec)
-                      result' (conj result block)]
-                  [others parents' result'])))]
-        (recur blocks parents result)))))
+   (gp-block/page-name->map original-page-name with-id? (db/get-db (state/get-current-repo)) with-timestamp? (state/get-date-formatter))))
 
 (defn parse-block
   ([block]
@@ -702,12 +67,12 @@
                        (str (config/get-block-pattern format) " " (string/triml content)))]
        (if-let [result (state/get-block-ast block-uuid content)]
          result
-         (let [ast (->> (format/to-edn content format (mldoc/default-config format))
+         (let [ast (->> (format/to-edn content format (gp-mldoc/default-config format))
                         (map first))
-               title (when (heading-block? (first ast))
+               title (when (gp-block/heading-block? (first ast))
                        (:title (second (first ast))))
                body (vec (if title (rest ast) ast))
-               body (drop-while property/properties-ast? body)
+               body (drop-while gp-property/properties-ast? body)
                result (cond->
                         (if (seq body) {:block/body body} {})
                         title

src/main/frontend/format/mldoc.cljs

@@ -1,68 +1,19 @@
 (ns frontend.format.mldoc
-  (:require [cljs-bean.core :as bean]
-            [clojure.string :as string]
+  "Mldoc code needed by app but not graph-parser"
+  (:require [clojure.string :as string]
             [frontend.format.protocol :as protocol]
-            [frontend.utf8 :as utf8]
+            [frontend.state :as state]
             [goog.object :as gobj]
             [lambdaisland.glogi :as log]
-            [medley.core :as medley]
             ["mldoc" :as mldoc :refer [Mldoc]]
-            [linked.core :as linked]
-            [logseq.graph-parser.util :as gp-util]
-            [logseq.graph-parser.config :as gp-config]))
+            [logseq.graph-parser.mldoc :as gp-mldoc]
+            [logseq.graph-parser.util :as gp-util]))
 
-(defonce parseJson (gobj/get Mldoc "parseJson"))
-(defonce parseInlineJson (gobj/get Mldoc "parseInlineJson"))
-(defonce parseOPML (gobj/get Mldoc "parseOPML"))
-(defonce export (gobj/get Mldoc "export"))
 (defonce anchorLink (gobj/get Mldoc "anchorLink"))
+(defonce parseOPML (gobj/get Mldoc "parseOPML"))
 (defonce parseAndExportMarkdown (gobj/get Mldoc "parseAndExportMarkdown"))
 (defonce parseAndExportOPML (gobj/get Mldoc "parseAndExportOPML"))
-(defonce astExportMarkdown (gobj/get Mldoc "astExportMarkdown"))
-
-(defn convert-export-md-remove-options [opts]
-  (->>
-   (mapv (fn [opt]
-             (case opt
-               :page-ref ["Page_ref"]
-               :emphasis ["Emphasis"]
-               []))
-         opts)
-   (remove empty?)))
-
-
-(defn default-config
-  ([format]
-   (default-config format {:export-heading-to-list? false}))
-  ([format {:keys [export-heading-to-list? export-keep-properties? export-md-indent-style export-md-remove-options parse_outline_only?]}]
-   (let [format (string/capitalize (name (or format :markdown)))]
-     (->> {:toc false
-           :parse_outline_only (or parse_outline_only? false)
-           :heading_number false
-           :keep_line_break true
-           :format format
-           :heading_to_list (or export-heading-to-list? false)
-           :exporting_keep_properties export-keep-properties?
-           :export_md_indent_style export-md-indent-style
-           :export_md_remove_options
-           (convert-export-md-remove-options export-md-remove-options)}
-          (filter #(not(nil? (second %))))
-          (into {})
-          (bean/->js)
-          (js/JSON.stringify)))))
-
-(def default-references
-  (js/JSON.stringify
-   (clj->js {:embed_blocks []
-             :embed_pages []})))
-
-(defn parse-json
-  [content config]
-  (parseJson content config))
-
-(defn inline-parse-json
-  [text config]
-  (parseInlineJson text config))
+(defonce export (gobj/get Mldoc "export"))
 
 (defn parse-opml
   [content]
@@ -72,118 +23,14 @@
   [content config references]
   (parseAndExportMarkdown content
                           config
-                          (or references default-references)))
+                          (or references gp-mldoc/default-references)))
 
 (defn parse-export-opml
   [content config title references]
   (parseAndExportOPML content
                       config
                       title
-                      (or references default-references)))
-
-(defn ast-export-markdown
-  [ast config references]
-  (astExportMarkdown ast
-                     config
-                     (or references default-references)))
-
-(defn remove-indentation-spaces
-  [s level remove-first-line?]
-  (let [lines (string/split-lines s)
-        [f & r] lines
-        body (map (fn [line]
-                    (if (string/blank? (gp-util/safe-subs line 0 level))
-                      (gp-util/safe-subs line level)
-                      line))
-               (if remove-first-line? lines r))
-        content (if remove-first-line? body (cons f body))]
-    (string/join "\n" content)))
-
-(defn- ->vec
-  [s]
-  (if (string? s) [s] s))
-
-(defn- ->vec-concat
-  [& coll]
-  (->> (map ->vec coll)
-       (remove nil?)
-       (apply concat)
-       (distinct)))
-
-(defn collect-page-properties
-  [ast parse-property]
-  (if (seq ast)
-    (let [original-ast ast
-          ast (map first ast)           ; without position meta
-          directive? (fn [[item _]] (= "directive" (string/lower-case (first item))))
-          grouped-ast (group-by directive? original-ast)
-          directive-ast (take-while directive? original-ast)
-          [properties-ast other-ast] (if (= "Property_Drawer" (ffirst ast))
-                                       [(last (first ast))
-                                        (rest original-ast)]
-                                       [(->> (map first directive-ast)
-                                             (map rest))
-                                        (get grouped-ast false)])
-          properties (->>
-                      properties-ast
-                      (map (fn [[k v]]
-                             (let [k (keyword (string/lower-case k))
-                                   v (if (contains? #{:title :description :filters :macro} k)
-                                       v
-                                       (parse-property k v))]
-                               [k v]))))
-          properties (into (linked/map) properties)
-          macro-properties (filter (fn [x] (= :macro (first x))) properties)
-          macros (if (seq macro-properties)
-                   (->>
-                    (map
-                     (fn [[_ v]]
-                       (let [[k v] (gp-util/split-first " " v)]
-                         (mapv
-                          string/trim
-                          [k v])))
-                     macro-properties)
-                    (into {}))
-                   {})
-          properties (->> (remove (fn [x] (= :macro (first x))) properties)
-                          (into (linked/map)))
-          properties (cond-> properties
-                       (seq macros)
-                       (assoc :macros macros))
-          alias (:alias properties)
-          alias (when alias
-                  (if (coll? alias)
-                    (remove string/blank? alias)
-                    [alias]))
-          filetags (when-let [org-file-tags (:filetags properties)]
-                     (->> (string/split org-file-tags ":")
-                          (remove string/blank?)))
-          tags (:tags properties)
-          tags (->> (->vec-concat tags filetags)
-                    (remove string/blank?))
-          properties (assoc properties :tags tags :alias alias)
-          properties (-> properties
-                         (update :filetags (constantly filetags)))
-          properties (medley/remove-kv (fn [_k v] (or (nil? v) (and (coll? v) (empty? v)))) properties)]
-      (if (seq properties)
-        (cons [["Properties" properties] nil] other-ast)
-        original-ast))
-    ast))
-
-(defn update-src-full-content
-  [ast content]
-  (let [content (utf8/encode content)]
-    (map (fn [[block pos-meta]]
-          (if (and (vector? block)
-                   (= "Src" (first block)))
-            (let [{:keys [start_pos end_pos]} pos-meta
-                  content (utf8/substring content start_pos end_pos)
-                  spaces (re-find #"^[\t ]+" (first (string/split-lines content)))
-                  content (if spaces (remove-indentation-spaces content (count spaces) true)
-                              content)
-                  block ["Src" (assoc (second block) :full_content content)]]
-              [block pos-meta])
-            [block pos-meta])) ast)))
+                      (or references gp-mldoc/default-references)))
 
 (defn block-with-title?
   [type]
@@ -192,45 +39,21 @@
                "Hiccup"
                "Heading"} type))
 
-(def parse-property nil)
-
-(defn ->edn
-  [content config]
-  (if (string? content)
-    (try
-      (if (string/blank? content)
-        []
-        (-> content
-            (parse-json config)
-            (gp-util/json->clj)
-            (update-src-full-content content)
-            (collect-page-properties parse-property)))
-      (catch js/Error e
-        (js/console.error e)
-        []))
-    (log/error :edn/wrong-content-type content)))
-
 (defn opml->edn
   [content]
   (try
     (if (string/blank? content)
       {}
       (let [[headers blocks] (-> content (parse-opml) (gp-util/json->clj))]
-        [headers (collect-page-properties blocks parse-property)]))
+        [headers (gp-mldoc/collect-page-properties blocks gp-mldoc/parse-property (state/get-config))]))
     (catch js/Error e
       (log/error :edn/convert-failed e)
       [])))
 
-(defn inline->edn
-  [text config]
-  (try
-    (if (string/blank? text)
-      {}
-      (-> text
-          (inline-parse-json config)
-          (gp-util/json->clj)))
-    (catch js/Error _e
-      [])))
+(defn ->edn
+  "Wrapper around gp-mldoc/->edn which provides config state"
+  [content config]
+  (gp-mldoc/->edn content config (state/get-config)))
 
 (defrecord MldocMode []
   protocol/Format
@@ -259,21 +82,3 @@
   [ast typ]
   (and (contains? #{"Drawer"} (ffirst ast))
        (= typ (second (first ast)))))
-
-(defn link?
-  [format link]
-  (when (string? link)
-    (let [[type link] (first (inline->edn link (default-config format)))
-          [ref-type ref-value] (:url link)]
-      (and (= "Link" type)
-           (or
-            ;; 1. url
-            (not (contains? #{"Page_ref" "Block_ref"} ref-type))
-
-            (and (contains? #{"Page_ref"} ref-type)
-                 (or
-                  ;; 2. excalidraw link
-                  (gp-config/draw? ref-value)
-
-                  ;; 3. local asset link
-                  (boolean (gp-config/local-asset? ref-value)))))))))

src/main/frontend/fs/sync.cljs

@@ -52,7 +52,12 @@
 ;;       and re-produce a new same-file-delete diff.
 
 ;;; ### specs
-(s/def ::state #{::idle
+(s/def ::state #{;; do following jobs when ::starting:
+                 ;; - wait seconds for file-change-events from file-watcher
+                 ;; - drop redundant file-change-events
+                 ;; - setup states in `frontend.state`
+                 ::starting
+                 ::idle
                  ;; sync local-changed files
                  ::local->remote
                  ;; sync remote latest-transactions
@@ -71,6 +76,7 @@
 (s/def ::sync-state (s/keys :req-un [::state
                                      ::current-local->remote-files
                                      ::current-remote->local-files
+                                     ::queued-local->remote-files
                                      ::history]))
 
 ;; diff
@@ -106,7 +112,9 @@
 
 (def graphs-txid (persist-var/persist-var nil "graphs-txid"))
 
-(defn update-graphs-txid! [latest-txid graph-uuid user-uuid repo]
+(defn update-graphs-txid!
+  [latest-txid graph-uuid user-uuid repo]
+  {:pre [(int? latest-txid) (>= latest-txid 0)]}
   (persist-var/-reset-value! graphs-txid [user-uuid graph-uuid latest-txid] repo)
   (persist-var/persist-save graphs-txid))
 
@@ -837,6 +845,71 @@
                 not)
     (go (>! local-changes-chan (->FileChangeEvent type dir path stat)))))
 
+;;; ### sync state
+
+
+(defn sync-state
+  "create a new sync-state"
+  []
+  {:post [(s/valid? ::sync-state %)]}
+  {:state ::starting
+   :current-local->remote-files #{}
+   :current-remote->local-files #{}
+   :queued-local->remote-files #{}
+   :history '()})
+
+(defn- sync-state--update-state
+  [sync-state next-state]
+  {:pre [(s/valid? ::state next-state)]
+   :post [(s/valid? ::sync-state %)]}
+  (assoc sync-state :state next-state))
+
+(defn sync-state--add-current-remote->local-files
+  [sync-state paths]
+  {:post [(s/valid? ::sync-state %)]}
+  (update sync-state :current-remote->local-files into paths))
+
+(defn sync-state--add-current-local->remote-files
+  [sync-state paths]
+  {:post [(s/valid? ::sync-state %)]}
+  (update sync-state :current-local->remote-files into paths))
+
+(defn sync-state--update-queued-local->remote-files
+  [sync-state paths]
+  {:post [(s/valid? ::sync-state %)]}
+  (update sync-state :queued-local->remote-files (fn [_ n] (set n)) paths))
+
+(defn- add-history-items
+  [history paths now]
+  (sequence
+   (comp
+    ;; only reserve the latest one of same-path-items
+    (dedupe-by :path)
+    ;; reserve the latest 20 history items
+    (take 20))
+   (into history
+         (map (fn [path] {:path path :time now}) paths))))
+
+(defn sync-state--remove-current-remote->local-files
+  [sync-state paths]
+  {:post [(s/valid? ::sync-state %)]}
+  (let [now (t/now)]
+    (-> sync-state
+        (update :current-remote->local-files set/difference paths)
+        (update :history add-history-items paths now))))
+
+(defn sync-state--remove-current-local->remote-files
+  [sync-state paths]
+  {:post [(s/valid? ::sync-state %)]}
+  (let [now (t/now)]
+    (-> sync-state
+        (update :current-local->remote-files set/difference paths)
+        (update :history add-history-items paths now))))
+
+(defn sync-state--stopped?
+  [sync-state]
+  (= ::stop (:state sync-state)))
+
 ;;; ### remote->local syncer & local->remote syncer
 
 (defprotocol IRemote->LocalSync
@@ -953,32 +1026,26 @@
 (defn- contains-path? [regexps path]
   (reduce #(when (re-find %2 path) (reduced true)) false regexps))
 
-(defn- filter-local-changes
+(defn- filter-local-changes-pred
   "filter local-change events:
   - for 'unlink' event
     - when related file exists on local dir, ignore this event
   - for 'add' | 'change' event
     - when related file's content is same as remote file, ignore it"
-  [to-ch from-ch basepath graph-uuid]
-  (async/pipeline-async
-   1 to-ch
-   (fn [^FileChangeEvent e result]
-     (go
-       (case (.-type e)
-         "unlink"
-         (let [r (<! (get-local-files-meta rsapi "" basepath [(relative-path e)]))]
-           (when (some-> r ex-cause ;; str (string/index-of "No such file or directory")
-                         )
-             (>! result e)))
-
-         ("add" "change")
-         (let [path (relative-path e)]
-           (when (and (<! (local-file-exists? path basepath))
-                      (<! (file-changed? graph-uuid path basepath)))
-             (>! result e))))
-       (async/close! result)))
-   from-ch false))
-
+  [^FileChangeEvent e basepath graph-uuid]
+  (go
+    (let [r-path (relative-path e)]
+      (case (.-type e)
+        "unlink"
+        (let [r (<! (get-local-files-meta rsapi "" basepath [r-path]))]
+          ;; keep this e when it's not found
+          (some-> r ex-cause))
+
+        ("add" "change")
+        ;; 1. local file exists
+        ;; 2. compare with remote file, and changed
+        (and (<! (local-file-exists? r-path basepath))
+             (<! (file-changed? graph-uuid r-path basepath)))))))
 
 (defrecord ^:large-vars/cleanup-todo
     Local->RemoteSyncer [user-uuid graph-uuid base-path repo *sync-state
@@ -1009,7 +1076,7 @@
       (let [c (.filtered-chan this 10000)
             filter-e-fn (.filter-file-change-events-fn this)]
         (go-loop [timeout-c (timeout rate)
-                  tcoll (transient [])]
+                  coll []]
           (let [{:keys [timeout ^FileChangeEvent e stop]}
                 (async/alt! timeout-c {:timeout true}
                             from-chan ([e] {:e e})
@@ -1019,22 +1086,22 @@
               (async/close! c)
 
               timeout
-              (let [from-c (chan 10000)]
-                (<! (async/onto-chan! from-c (distinct (persistent! tcoll)) false))
-                (filter-local-changes c from-c base-path graph-uuid)
-                (async/close! from-c)
-                (recur (async/timeout rate) (transient [])))
+              (do (async/onto-chan! c coll false)
+                  (swap! *sync-state sync-state--update-queued-local->remote-files nil)
+                  (recur (async/timeout rate) []))
 
               (some? e)
-              (do
-                (when (filter-e-fn e)
-                  (conj! tcoll e))
-                (recur timeout-c tcoll))
+              (if (and (filter-e-fn e)
+                           (<! (filter-local-changes-pred e base-path graph-uuid)))
+                    (let [coll* (distinct (conj coll e))]
+                      (swap! *sync-state sync-state--update-queued-local->remote-files
+                             (mapv relative-path coll*))
+                      (recur timeout-c coll*))
+                    (recur timeout-c coll))
 
               (nil? e)
-              (do
-                (println "close ratelimit chan")
-                (async/close! c)))))
+              (do (println "close ratelimit chan")
+                  (async/close! c)))))
         c))
 
 
@@ -1114,64 +1181,7 @@
                     (or need-sync-remote unknown) r)))))))))
 
 
-;;; ### sync state
-
-
-(defn sync-state
-  "create a new sync-state"
-  []
-  {:post [(s/valid? ::sync-state %)]}
-  {:state ::idle
-   :current-local->remote-files #{}
-   :current-remote->local-files #{}
-   :history '()})
-
-(defn- sync-state--update-state
-  [sync-state next-state]
-  {:pre [(s/valid? ::state next-state)]
-   :post [(s/valid? ::sync-state %)]}
-  (assoc sync-state :state next-state))
-
-(defn sync-state--add-current-remote->local-files
-  [sync-state paths]
-  {:post [(s/valid? ::sync-state %)]}
-  (update sync-state :current-remote->local-files into paths))
-
-(defn sync-state--add-current-local->remote-files
-  [sync-state paths]
-  {:post [(s/valid? ::sync-state %)]}
-  (update sync-state :current-local->remote-files into paths))
-
-(defn- add-history-items
-  [history paths now]
-  (sequence
-   (comp
-    ;; only reserve the latest one of same-path-items
-    (dedupe-by :path)
-    ;; reserve the latest 20 history items
-    (take 20))
-   (into history
-         (map (fn [path] {:path path :time now}) paths))))
-
-(defn sync-state--remove-current-remote->local-files
-  [sync-state paths]
-  {:post [(s/valid? ::sync-state %)]}
-  (let [now (t/now)]
-    (-> sync-state
-        (update :current-remote->local-files set/difference paths)
-        (update :history add-history-items paths now))))
-
-(defn sync-state--remove-current-local->remote-files
-  [sync-state paths]
-  {:post [(s/valid? ::sync-state %)]}
-  (let [now (t/now)]
-    (-> sync-state
-        (update :current-local->remote-files set/difference paths)
-        (update :history add-history-items paths now))))
 
-(defn sync-state--stopped?
-  [sync-state]
-  (= ::stop (:state sync-state)))
 
 
 ;;; ### put all stuff together
@@ -1412,24 +1422,34 @@
       ;; 1. if remote graph has been deleted, clear graphs-txid.edn
       ;; 2. if graphs-txid.edn's content isn't [user-uuid graph-uuid txid], clear it
       (if (not= 3 (count @graphs-txid))
-        (clear-graphs-txid! repo)
+        (do (clear-graphs-txid! repo)
+            (state/set-file-sync-state nil))
         (when (check-graph-belong-to-current-user current-user-uuid user-uuid)
           (if-not (<! (check-remote-graph-exists graph-uuid))
             (clear-graphs-txid! repo)
             (do
               ;; set-env
               (set-env rsapi config/FILE-SYNC-PROD?)
-
+              (state/set-file-sync-state @*sync-state)
+              (state/set-file-sync-manager sm)
+              ;; wait seconds to receive all file change events,
+              ;; and then drop all of them.
+              ;; WHY: when opening a graph(or switching to another graph),
+              ;;      file-watcher will send a lot of file-change-events,
+              ;;      actually, each file corresponds to a file-change-event,
+              ;;      we need to ignore all of them.
+              (<! (timeout 5000))
               (drain-chan local-changes-chan)
               (poll! stop-sync-chan)
               (poll! remote->local-sync-chan)
+
               ;; update global state when *sync-state changes
               (add-watch *sync-state ::update-global-state
                          (fn [_ _ _ n]
                            (state/set-file-sync-state n)))
               (.start sm)
 
-              (state/set-file-sync-manager sm)
+
               (offer! remote->local-sync-chan true)
               (offer! full-sync-chan true)
 

src/main/frontend/fs/watcher_handler.cljs

@@ -4,7 +4,7 @@
             [frontend.db :as db]
             [frontend.db.model :as model]
             [frontend.handler.editor :as editor]
-            [frontend.handler.extract :as extract]
+            [logseq.graph-parser.extract :as extract]
             [frontend.handler.file :as file-handler]
             [frontend.handler.page :as page-handler]
             [frontend.handler.repo :as repo-handler]

src/main/frontend/handler/block.cljs

@@ -5,7 +5,7 @@
             [frontend.db.model :as db-model]
             [frontend.db.react :as react]
             [frontend.state :as state]
-            [frontend.format.block :as block]
+            [logseq.graph-parser.block :as gp-block]
             [frontend.util :as util]))
 
 
@@ -83,7 +83,7 @@
   [block typ]
   (walk-block block
               (fn [x]
-                (and (block/timestamp-block? x)
+                (and (gp-block/timestamp-block? x)
                      (= typ (first (second x)))))
               #(second (second %))))
 

src/main/frontend/handler/editor.cljs

@@ -34,8 +34,8 @@
             [frontend.search :as search]
             [frontend.state :as state]
             [frontend.template :as template]
-            [frontend.text :as text]
-            [frontend.utf8 :as utf8]
+            [logseq.graph-parser.text :as text]
+            [logseq.graph-parser.utf8 :as utf8]
             [frontend.util :as util :refer [profile]]
             [frontend.util.clock :as clock]
             [frontend.util.cursor :as cursor]
@@ -53,6 +53,8 @@
             [promesa.core :as p]
             [frontend.util.keycode :as keycode]
             [logseq.graph-parser.util :as gp-util]
+            [logseq.graph-parser.mldoc :as gp-mldoc]
+            [logseq.graph-parser.block :as gp-block]
             ["path" :as path]))
 
 ;; FIXME: should support multiple images concurrently uploading
@@ -255,7 +257,7 @@
 (defn- another-block-with-same-id-exists?
   [current-id block-id]
   (and (string? block-id)
-       (gp-util/uuid-string? block-id)
+       (util/uuid-string? block-id)
        (not= current-id (cljs.core/uuid block-id))
        (db/entity [:block/uuid (cljs.core/uuid block-id)])))
 
@@ -336,7 +338,7 @@
   (if (and (state/enable-timetracking?)
            (not= (:block/content block) value))
     (let [format (:block/format block)
-          new-marker (last (gp-util/safe-re-find (marker/marker-pattern format) (or value "")))
+          new-marker (last (util/safe-re-find (marker/marker-pattern format) (or value "")))
           new-value (with-marker-time value block format
                       new-marker
                       (:block/marker block))]
@@ -356,7 +358,7 @@
         content (drawer/with-logbook block content)
         content (with-timetracking block content)
         first-block? (= left page)
-        ast (mldoc/->edn (string/trim content) (mldoc/default-config format))
+        ast (mldoc/->edn (string/trim content) (gp-mldoc/default-config format))
         first-elem-type (first (ffirst ast))
         first-elem-meta (second (ffirst ast))
         properties? (contains? #{"Property_Drawer" "Properties"} first-elem-type)
@@ -453,11 +455,12 @@
 (declare save-current-block!)
 (defn outliner-insert-block!
   [config current-block new-block {:keys [sibling? keep-uuid? replace-empty-target?]}]
-  (let [ref-top-block? (and (:ref? config)
-                            (not (:ref-child? config)))
+  (let [ref-query-top-block? (and (or (:ref? config)
+                                      (:custom-query? config))
+                                  (not (:ref-query-child? config)))
         has-children? (db/has-children? (:block/uuid current-block))
         sibling? (cond
-                   ref-top-block?
+                   ref-query-top-block?
                    false
 
                    (boolean? sibling?)
@@ -470,7 +473,7 @@
                    (not has-children?))]
     (outliner-tx/transact!
       {:outliner-op :insert-blocks}
-      (save-current-block!)
+      (save-current-block! {:current-block current-block})
       (outliner-core/insert-blocks! [new-block] current-block {:sibling? sibling?
                                                                :keep-uuid? keep-uuid?
                                                                :replace-empty-target? replace-empty-target?}))))
@@ -480,10 +483,10 @@
   (let [current-page (state/get-current-page)
         block-id (or
                   (and (:id config)
-                       (gp-util/uuid-string? (:id config))
+                       (util/uuid-string? (:id config))
                        (:id config))
                   (and current-page
-                       (gp-util/uuid-string? current-page)
+                       (util/uuid-string? current-page)
                        current-page))]
     (= uuid (and block-id (medley/uuid block-id)))))
 
@@ -667,7 +670,9 @@
 (defn properties-block
   [properties format page]
   (let [content (property/insert-properties format "" properties)
-        refs (block/get-page-refs-from-properties properties)]
+        refs (gp-block/get-page-refs-from-properties properties
+                                                     (db/get-db (state/get-current-repo))
+                                                     (state/get-date-formatter))]
     {:block/pre-block? true
      :block/uuid (db/new-block-id)
      :block/properties properties
@@ -1142,7 +1147,7 @@
   []
   (when-let [page (get-nearest-page)]
     (let [page-name (string/lower-case page)
-          block? (gp-util/uuid-string? page-name)]
+          block? (util/uuid-string? page-name)]
       (when-let [page (db/get-page page-name)]
         (if block?
           (state/sidebar-add-block!
@@ -1172,7 +1177,7 @@
     (let [page (state/get-current-page)
           block-id (and
                     (string? page)
-                    (gp-util/uuid-string? page)
+                    (util/uuid-string? page)
                     (medley/uuid page))]
       (when block-id
         (let [block-parent (db/get-block-parent block-id)]
@@ -1273,7 +1278,7 @@
   "skip-properties? if set true, when editing block is likely be properties, skip saving"
   ([]
    (save-current-block! {}))
-  ([{:keys [force? skip-properties?] :as opts}]
+  ([{:keys [force? skip-properties? current-block] :as opts}]
    ;; non English input method
    (when-not (state/editor-in-composition?)
      (when (state/get-current-repo)
@@ -1294,7 +1299,8 @@
                  db-content (:block/content db-block)
                  db-content-without-heading (and db-content
                                                  (gp-util/safe-subs db-content (:block/level db-block)))
-                 value (and elem (gobj/get elem "value"))]
+                 value (or (:block/content current-block)
+                           (and elem (gobj/get elem "value")))]
              (cond
                force?
                (save-block-aux! db-block value opts)
@@ -1313,7 +1319,7 @@
 
 (defn- clean-content!
   [format content]
-  (->> (text/remove-level-spaces content format)
+  (->> (text/remove-level-spaces content format (config/get-block-pattern format))
        (drawer/remove-logbook)
        (property/remove-properties format)
        string/trim))
@@ -1941,7 +1947,7 @@
                     props (into [] (:properties block))
                     content* (str (if (= :markdown format) "- " "* ")
                                   (property/insert-properties format content props))
-                    ast (mldoc/->edn content* (mldoc/default-config format))
+                    ast (mldoc/->edn content* (gp-mldoc/default-config format))
                     blocks (block/extract-blocks ast content* true format)
                     fst-block (first blocks)]
                 (assert fst-block "fst-block shouldn't be nil")
@@ -1954,7 +1960,7 @@
   (let [blocks (block-tree->blocks tree-vec format)
         target-block (db/pull target-block-id)
         page-id (:db/id (:block/page target-block))
-        blocks (block/with-parent-and-left page-id blocks)]
+        blocks (gp-block/with-parent-and-left page-id blocks)]
     (paste-blocks
      blocks
      {:target-block target-block
@@ -2034,7 +2040,7 @@
 (defn- last-top-level-child?
   [{:keys [id]} current-node]
   (when id
-    (when-let [entity (if (gp-util/uuid-string? (str id))
+    (when-let [entity (if (util/uuid-string? (str id))
                         (db/entity [:block/uuid (uuid id)])
                         (db/entity [:block/name (util/page-name-sanity-lc id)]))]
       (= (:block/uuid entity) (tree/-get-parent-id current-node)))))
@@ -2853,8 +2859,8 @@
   (when-let [editing-block (state/get-edit-block)]
     (let [page-id (:db/id (:block/page editing-block))
           blocks (block/extract-blocks
-                  (mldoc/->edn text (mldoc/default-config format)) text true format)
-          blocks' (block/with-parent-and-left page-id blocks)]
+                  (mldoc/->edn text (gp-mldoc/default-config format)) text true format)
+          blocks' (gp-block/with-parent-and-left page-id blocks)]
       (paste-blocks blocks' {}))))
 
 (defn- paste-segmented-text
@@ -2864,7 +2870,7 @@
         (string/join "\n"
                      (mapv (fn [p] (->> (string/trim p)
                                         ((fn [p]
-                                           (if (gp-util/safe-re-find (if (= format :org)
+                                           (if (util/safe-re-find (if (= format :org)
                                                                     #"\s*\*+\s+"
                                                                     #"\s*-\s+") p)
                                              p
@@ -2886,11 +2892,13 @@
   [text e]
   (let [copied-blocks (state/get-copied-blocks)
         copied-block-ids (:copy/block-ids copied-blocks)
+        copied-graph (:copy/graph copied-blocks)
         input (state/get-input)
         *stop-event? (atom true)]
     (cond
       ;; Internal blocks by either copy or cut blocks
       (and
+       (= copied-graph (state/get-current-repo))
        (or (seq copied-block-ids)
            (seq (:copy/full-blocks copied-blocks)))
        text
@@ -2904,17 +2912,17 @@
           (state/set-copied-full-blocks! blocks)
           (paste-blocks blocks {})))
 
-      (and (util/url? text)
+      (and (gp-util/url? text)
            (not (string/blank? (util/get-selected-text))))
       (html-link-format! text)
 
-      (and (util/url? text)
+      (and (gp-util/url? text)
            (or (string/includes? text "youtube.com")
                (string/includes? text "youtu.be"))
            (mobile-util/is-native-platform?))
       (commands/simple-insert! (state/get-edit-input-id) (util/format "{{youtube %s}}" text) nil)
 
-      (and (util/url? text)
+      (and (gp-util/url? text)
            (string/includes? text "twitter.com")
            (mobile-util/is-native-platform?))
       (commands/simple-insert! (state/get-edit-input-id) (util/format "{{twitter %s}}" text) nil)
@@ -2927,9 +2935,9 @@
       ;; from external
       (let [format (or (db/get-page-format (state/get-current-page)) :markdown)]
         (match [format
-                (nil? (gp-util/safe-re-find #"(?m)^\s*(?:[-+*]|#+)\s+" text))
-                (nil? (gp-util/safe-re-find #"(?m)^\s*\*+\s+" text))
-                (nil? (gp-util/safe-re-find #"(?:\r?\n){2,}" text))]
+                (nil? (util/safe-re-find #"(?m)^\s*(?:[-+*]|#+)\s+" text))
+                (nil? (util/safe-re-find #"(?m)^\s*\*+\s+" text))
+                (nil? (util/safe-re-find #"(?:\r?\n){2,}" text))]
           [:markdown false _ _]
           (paste-text-parseable format text)
 
@@ -3171,7 +3179,7 @@
   [format content semantic?]
   (and (string/includes? content "\n")
        (if semantic?
-         (let [ast (mldoc/->edn content (mldoc/default-config format))
+         (let [ast (mldoc/->edn content (gp-mldoc/default-config format))
                first-elem-type (first (ffirst ast))]
            (mldoc/block-with-title? first-elem-type))
          true)))
@@ -3214,7 +3222,7 @@
     :or {collapse? false expanded? false incremental? true root-block nil}}]
   (when-let [page (or (state/get-current-page)
                       (date/today))]
-    (let [block? (gp-util/uuid-string? page)
+    (let [block? (util/uuid-string? page)
           block-id (or root-block (and block? (uuid page)))
           blocks (if block-id
                    (db/get-block-and-children (state/get-current-repo) block-id)
@@ -3476,12 +3484,11 @@
   1. References.
   2. Custom queries."
   [block config]
-  (if (or (:ref? config)
-          (:custom-query? config))
-    (and
-     (seq (:block/children block))
-     (or
-      (:custom-query? config)
-      (>= (:ref/level block)
-          (state/get-ref-open-blocks-level))))
-    (util/collapsed? block)))
+  (or
+   (and
+    (or (:ref? config) (:custom-query? config))
+    (>= (inc (:block/level block))
+        (state/get-ref-open-blocks-level))
+    ;; has children
+    (first (:block/_parent (db/entity (:db/id block)))))
+   (util/collapsed? block)))

src/main/frontend/handler/events.cljs

@@ -59,8 +59,8 @@
 (defn- file-sync-stop-when-switch-graph []
   (p/do! (persist-var/load-vars)
          (sync/sync-stop)
-         ;; trigger rerender file-sync-header
-         (state/set-file-sync-state nil)))
+         (sync/sync-start)
+))
 
 (defn- graph-switch [graph]
   (state/set-current-repo! graph)
@@ -98,7 +98,6 @@
      (graph-switch graph))))
 
 (defmethod handle :graph/switch [[_ graph]]
-  (file-sync-stop-when-switch-graph)
   (if (outliner-file/writes-finished?)
     (graph-switch-on-persisted graph)
     (notification/show!
@@ -269,6 +268,11 @@
       (plugin/open-focused-settings-modal! title))
     (state/close-sub-modal! "ls-focused-settings-modal")))
 
+(defmethod handle :go/proxy-settings [[_ agent-opts]]
+  (state/set-sub-modal!
+    (fn [_] (plugin/user-proxy-settings-panel agent-opts))
+    {:id :https-proxy-panel :center? true}))
+
 
 (defmethod handle :redirect-to-home [_]
   (page-handler/create-today-journal!))
@@ -346,12 +350,11 @@
 
 (defmethod handle :file-watcher/changed [[_ ^js event]]
   (let [type (.-event event)
-        payload (js->clj event :keywordize-keys true)
-        payload' (-> payload
-                     (update :path js/decodeURI))]
-    (prn ::fs-watcher payload)
-    (fs-watcher/handle-changed! type payload')
-    (sync/file-watch-handler type payload')))
+        payload (-> event
+                    (js->clj :keywordize-keys true)
+                    (update :path js/decodeURI))]
+    (fs-watcher/handle-changed! type payload)
+    (sync/file-watch-handler type payload)))
 
 (defmethod handle :rebuild-slash-commands-list [[_]]
   (page-handler/rebuild-slash-commands-list!))
@@ -388,6 +391,16 @@
     db-encrypted-secret
     close-fn)))
 
+(defmethod handle :journal/insert-template [[_ page-name]]
+  (let [page-name (util/page-name-sanity-lc page-name)]
+    (when-let [page (db/pull [:block/name page-name])]
+      (when (db/page-empty? (state/get-current-repo) page-name)
+        (when-let [template (state/get-default-journal-template)]
+          (editor-handler/insert-template!
+           nil
+           template
+           {:target page}))))))
+
 (defn run!
   []
   (let [chan (state/get-events-chan)]

src/main/frontend/handler/export.cljs

@@ -16,6 +16,8 @@
             [frontend.state :as state]
             [frontend.util :as util]
             [frontend.format.mldoc :as mldoc]
+            [logseq.graph-parser.mldoc :as gp-mldoc]
+            [logseq.graph-parser.util :as gp-util]
             [goog.dom :as gdom]
             [promesa.core :as p])
   (:import [goog.string StringBuffer]))
@@ -214,7 +216,7 @@
   (let [block (db/entity [:block/uuid (uuid block-uuid)])
         block-content (get-blocks-contents repo (:block/uuid block))
         format (:block/format block)
-        ast (mldoc/->edn block-content (mldoc/default-config format))
+        ast (mldoc/->edn block-content (gp-mldoc/default-config format))
         embed-pages-new  (get-embed-pages-from-ast ast)
         embed-blocks-new  (get-embed-blocks-from-ast ast)
         block-refs-new (get-block-refs-from-ast ast)
@@ -258,7 +260,7 @@
   (let [page-name* (util/page-name-sanity-lc page-name)
         page-content (get-page-content repo page-name*)
         format (:block/format (db/entity [:block/name page-name*]))
-        ast (mldoc/->edn page-content (mldoc/default-config format))
+        ast (mldoc/->edn page-content (gp-mldoc/default-config format))
         embed-pages-new (get-embed-pages-from-ast ast)
         embed-blocks-new (get-embed-blocks-from-ast ast)
         block-refs-new (get-block-refs-from-ast ast)
@@ -378,7 +380,7 @@
                                               [?e2 :block/file ?e]
                                               [?e2 :block/name ?n]
                                               [?e2 :block/original-name ?n2]] db path)
-                                :format (f/get-format path)})))))
+                                :format (gp-util/get-format path)})))))
 
 
 (defn export-repo-as-markdown!

src/main/frontend/handler/external.cljs

@@ -9,6 +9,8 @@
             [frontend.db :as db]
             [frontend.format.mldoc :as mldoc]
             [frontend.format.block :as block]
+            [logseq.graph-parser.util :as gp-util]
+            [logseq.graph-parser.date-time-util :as date-time-util]
             [frontend.handler.page :as page]
             [frontend.handler.editor :as editor]
             [frontend.util :as util]))
@@ -26,7 +28,7 @@
                                       (when journal?
                                         (date/journal-title->default title))
                                       (string/replace title "/" "-"))
-                               title (-> (util/page-name-sanity title)
+                               title (-> (gp-util/page-name-sanity title)
                                          (string/replace "\n" " "))
                                path (str (if journal?
                                            (config/get-journals-directory)
@@ -49,7 +51,7 @@
                              (map
                               (fn [title]
                                 (let [day (date/journal-title->int title)
-                                      page-name (util/page-name-sanity-lc (date/int->journal-title day))]
+                                      page-name (util/page-name-sanity-lc (date-time-util/int->journal-title day (state/get-date-formatter)))]
                                   {:block/name page-name
                                    :block/journal? true
                                    :block/journal-day day}))

src/main/frontend/handler/file.cljs

@@ -8,15 +8,15 @@
             [clojure.core.async :as async]
             [frontend.config :as config]
             [frontend.db :as db]
-            [frontend.format :as format]
             [frontend.fs :as fs]
             [frontend.fs.nfs :as nfs]
             [frontend.handler.common :as common-handler]
-            [frontend.handler.extract :as extract-handler]
+            [logseq.graph-parser.extract :as extract]
             [frontend.handler.ui :as ui-handler]
             [frontend.state :as state]
             [frontend.util :as util]
             [logseq.graph-parser.util :as gp-util]
+            [logseq.graph-parser.config :as gp-config]
             [lambdaisland.glogi :as log]
             [promesa.core :as p]
             [frontend.mobile.util :as mobile]
@@ -43,7 +43,7 @@
   [files formats]
   (filter
    (fn [file]
-     (let [format (format/get-format file)]
+     (let [format (gp-util/get-format file)]
        (contains? formats format)))
    files))
 
@@ -119,10 +119,19 @@
          file (gp-util/path-normalize file)
          new? (nil? (db/entity [:file/path file]))]
      (db/set-file-content! repo-url file content)
-     (let [format (format/get-format file)
+     (let [format (gp-util/get-format file)
            file-content [{:file/path file}]
-           tx (if (contains? config/mldoc-support-formats format)
-                (let [[pages blocks] (extract-handler/extract-blocks-pages repo-url file content)
+           tx (if (contains? gp-config/mldoc-support-formats format)
+                (let [[pages blocks]
+                      (extract/extract-blocks-pages
+                       file
+                       content
+                       {:user-config (state/get-config)
+                        :date-formatter (state/get-date-formatter)
+                        :page-name-order (state/page-name-order)
+                        :block-pattern (config/get-block-pattern format)
+                        :supported-formats (config/supported-formats)
+                        :db (db/get-db (state/get-current-repo))})
                       first-page (first pages)
                       delete-blocks (->
                                      (concat
@@ -144,7 +153,7 @@
                                           (seq))
                       ;; To prevent "unique constraint" on datascript
                       block-ids (set/union (set block-ids) (set block-refs-ids))
-                      pages (extract-handler/with-ref-pages pages blocks)
+                      pages (extract/with-ref-pages pages blocks)
                       pages-index (map #(select-keys % [:block/name]) pages)]
                   ;; does order matter?
                   (concat file-content pages-index delete-blocks pages block-ids blocks))

src/main/frontend/handler/graph.cljs

@@ -4,8 +4,7 @@
             [frontend.db :as db]
             [frontend.db.default :as default-db]
             [frontend.state :as state]
-            [frontend.util :as util]
-            [logseq.graph-parser.util :as gp-util]))
+            [frontend.util :as util]))
 
 (defn- build-links
   [links]
@@ -46,7 +45,7 @@
                   ;; slow
 (defn- uuid-or-asset?
   [id]
-  (or (gp-util/uuid-string? id)
+  (or (util/uuid-string? id)
       (string/starts-with? id "../assets/")
       (= id "..")
       (string/starts-with? id "assets/")

src/main/frontend/handler/page.cljs

@@ -11,7 +11,6 @@
             [frontend.db.model :as model]
             [frontend.db.utils :as db-utils]
             [frontend.db.conn :as conn]
-            [frontend.format.block :as block]
             [frontend.fs :as fs]
             [frontend.handler.common :as common-handler]
             [frontend.handler.editor :as editor-handler]
@@ -35,6 +34,8 @@
             [frontend.mobile.util :as mobile-util]
             [logseq.graph-parser.util :as gp-util]
             [logseq.graph-parser.config :as gp-config]
+            [logseq.graph-parser.block :as gp-block]
+            [frontend.format.block :as block]
             [goog.functions :refer [debounce]]))
 
 (defn- get-directory
@@ -47,7 +48,7 @@
   [journal? title]
   (when-let [s (if journal?
                  (date/journal-title->default title)
-                 (util/page-name-sanity (string/lower-case title)))]
+                 (gp-util/page-name-sanity (string/lower-case title)))]
     ;; Win10 file path has a length limit of 260 chars
     (gp-util/safe-subs s 0 200)))
 
@@ -72,7 +73,9 @@
    (let [p (common-handler/get-page-default-properties title)
          ps (merge p properties)
          content (page-property/insert-properties format "" ps)
-         refs (block/get-page-refs-from-properties properties)]
+         refs (gp-block/get-page-refs-from-properties properties
+                                                      (db/get-db (state/get-current-repo))
+                                                      (state/get-date-formatter))]
      {:block/uuid (db/new-block-id)
       :block/properties ps
       :block/properties-order (keys ps)
@@ -118,12 +121,12 @@
                   properties          nil
                   split-namespace?    true}}]
    (let [title (string/trim title)
-         title (util/remove-boundary-slashes title)
+         title (gp-util/remove-boundary-slashes title)
          page-name (util/page-name-sanity-lc title)
          repo (state/get-current-repo)]
      (when (db/page-empty? repo page-name)
        (let [pages    (if split-namespace?
-                        (util/split-namespace-pages title)
+                        (gp-util/split-namespace-pages title)
                         [title])
              format   (or format (state/get-preferred-format))
              pages    (map (fn [page]
@@ -170,7 +173,7 @@
 (defn- compute-new-file-path
   [old-path new-name]
   (let [result (string/split old-path "/")
-        file-name (util/page-name-sanity new-name true)
+        file-name (gp-util/page-name-sanity new-name true)
         ext (last (string/split (last result) "."))
         new-file (str file-name "." ext)
         parts (concat (butlast result) [new-file])]
@@ -398,7 +401,7 @@
 
         ;; If page name changed after sanitization
         (when (or (util/create-title-property? new-page-name)
-                  (not= (util/page-name-sanity new-name false) new-name))
+                  (not= (gp-util/page-name-sanity new-name false) new-name))
           (page-property/add-property! new-page-name :title new-name))
 
         (when (and file (not journal?))
@@ -634,7 +637,7 @@
   (->> (db/get-all-pages repo)
        (remove (fn [p]
                  (let [name (:block/name p)]
-                   (or (gp-util/uuid-string? name)
+                   (or (util/uuid-string? name)
                        (gp-config/draw? name)
                        (db/built-in-pages-names (string/upper-case name))))))
        (common-handler/fix-pages-timestamps)))
@@ -688,7 +691,7 @@
               chosen (if (string/starts-with? chosen "New page: ") ;; FIXME: What if a page named "New page: XXX"?
                        (subs chosen 10)
                        chosen)
-              chosen (if (and (gp-util/safe-re-find #"\s+" chosen) (not wrapped?))
+              chosen (if (and (util/safe-re-find #"\s+" chosen) (not wrapped?))
                        (util/format "[[%s]]" chosen)
                        chosen)
               q (if @editor-handler/*selected-text "" q)
@@ -728,31 +731,24 @@
                 (and (= "local" repo) (not (mobile-util/is-native-platform?))))
         (let [title (date/today)
               today-page (util/page-name-sanity-lc title)
-              template (state/get-default-journal-template)
               format (state/get-preferred-format repo)
               file-name (date/journal-title->default title)
               path (str (config/get-journals-directory) "/" file-name "."
                         (config/get-file-extension format))
               file-path (str "/" path)
-              repo-dir (config/get-repo-dir repo)]
+              repo-dir (config/get-repo-dir repo)
+              template (state/get-default-journal-template)]
           (p/let [file-exists? (fs/file-exists? repo-dir file-path)
                   file-content (when file-exists?
                                  (fs/read-file repo-dir file-path))]
             (when (and (db/page-empty? repo today-page)
-                       (not (model/journal-day-exists? repo
-                                                       (date/journal-title->int (date/today))))
                        (or (not file-exists?)
                            (and file-exists? (string/blank? file-content))))
               (create! title {:redirect? false
                               :split-namespace? false
                               :create-first-block? (not template)
                               :journal? true})
-              (when template
-                (let [page (db/pull [:block/name today-page])]
-                  (editor-handler/insert-template!
-                   nil
-                   template
-                   {:target page})))
+              (state/pub-event! [:journal/insert-template today-page])
               (ui-handler/re-render-root!))))))))
 
 (defn open-today-in-sidebar

src/main/frontend/handler/plugin.cljs

@@ -3,7 +3,7 @@
             [rum.core :as rum]
             [frontend.util :as util]
             [clojure.walk :as walk]
-            [frontend.format.mldoc :as mldoc]
+            [logseq.graph-parser.mldoc :as gp-mldoc]
             [frontend.handler.notification :as notifications]
             [camel-snake-kebab.core :as csk]
             [frontend.state :as state]
@@ -403,7 +403,7 @@
                             (string/replace matched link (util/node-path.join url link))
                             matched)))
                       content)]
-        (format/to-html content :markdown (mldoc/default-config :markdown))))
+        (format/to-html content :markdown (gp-mldoc/default-config :markdown))))
     (catch js/Error e
       (log/error :parse-user-md-exception e)
       content)))

src/main/frontend/handler/repo.cljs

@@ -5,7 +5,6 @@
             [frontend.context.i18n :refer [t]]
             [frontend.date :as date]
             [frontend.db :as db]
-            [frontend.format :as format]
             [frontend.fs :as fs]
             [frontend.fs.nfs :as nfs]
             [frontend.handler.common :as common-handler]
@@ -23,6 +22,7 @@
             [shadow.resource :as rc]
             [frontend.db.persist :as db-persist]
             [logseq.graph-parser.util :as gp-util]
+            [logseq.graph-parser.config :as gp-config]
             [electron.ipc :as ipc]
             [clojure.set :as set]
             [clojure.core.async :as async]
@@ -185,8 +185,6 @@
 
 (defn- parse-and-load-file!
   [repo-url file new-graph?]
-  (state/set-parsing-state! (fn [m]
-                              (assoc m :current-parsing-file (:file/path file))))
   (try
     (file-handler/alter-file repo-url
                              (:file/path file)
@@ -212,14 +210,15 @@
     (ui-handler/re-render-root! re-render-opts))
   (state/pub-event! [:graph/added repo-url opts])
   (state/reset-parsing-state!)
+  (state/set-loading-files! repo-url false)
   (async/offer! graph-added-chan true))
 
 (defn- parse-files-and-create-default-files-inner!
   [repo-url files delete-files delete-blocks file-paths db-encrypted? re-render? re-render-opts opts]
   (let [support-files (filter
                        (fn [file]
-                         (let [format (format/get-format (:file/path file))]
-                           (contains? (set/union #{:edn :css} config/mldoc-support-formats) format)))
+                         (let [format (gp-util/get-format (:file/path file))]
+                           (contains? (set/union #{:edn :css} gp-config/mldoc-support-formats) format)))
                        files)
         support-files (sort-by :file/path support-files)
         {journals true non-journals false} (group-by (fn [file] (string/includes? (:file/path file) "journals/")) support-files)
@@ -240,13 +239,17 @@
     (if util/node-test?
       (do
         (doseq [file support-files']
+          (state/set-parsing-state! (fn [m]
+                                      (assoc m :current-parsing-file (:file/path file))))
           (parse-and-load-file! repo-url file new-graph?))
         (after-parse repo-url files file-paths db-encrypted? re-render? re-render-opts opts graph-added-chan))
       (async/go-loop []
         (if-let [file (async/<! chan)]
           (do
-            (parse-and-load-file! repo-url file new-graph?)
+            (state/set-parsing-state! (fn [m]
+                                        (assoc m :current-parsing-file (:file/path file))))
             (async/<! (async/timeout 10))
+            (parse-and-load-file! repo-url file new-graph?)
             (recur))
           (after-parse repo-url files file-paths db-encrypted? re-render? re-render-opts opts graph-added-chan))))
     graph-added-chan))
@@ -262,15 +265,9 @@
       (parse-files-and-create-default-files-inner! repo-url files delete-files delete-blocks file-paths db-encrypted? re-render? re-render-opts opts))
     (parse-files-and-create-default-files-inner! repo-url files delete-files delete-blocks file-paths db-encrypted? re-render? re-render-opts opts)))
 
-(defn- update-parsing-state!
-  [repo-url]
-  (state/set-loading-files! repo-url false))
-
 (defn parse-files-and-load-to-db!
   [repo-url files {:keys [delete-files delete-blocks re-render? re-render-opts _refresh?] :as opts
                    :or {re-render? true}}]
-  (update-parsing-state! repo-url)
-
   (let [file-paths (map :file/path files)
         metadata-file (config/get-metadata-path)
         metadata-content (some #(when (= (:file/path %) metadata-file)
@@ -298,7 +295,7 @@
                      (common-handler/read-config content)))
         relate-path-fn (fn [m k]
                          (some-> (get m k)
-                                 (string/replace (str (config/get-local-dir repo-url) "/") "")))
+                                 (string/replace (js/decodeURI (config/get-local-dir repo-url)) "")))
         nfs-files (common-handler/remove-hidden-files nfs-files config #(relate-path-fn % :file/path))
         diffs (common-handler/remove-hidden-files diffs config #(relate-path-fn % :path))
         load-contents (fn [files option]

src/main/frontend/handler/route.cljs

@@ -1,14 +1,14 @@
 (ns frontend.handler.route
   (:require [clojure.string :as string]
+            [frontend.config :as config]
             [frontend.date :as date]
             [frontend.db :as db]
             [frontend.handler.ui :as ui-handler]
             [frontend.handler.recent :as recent-handler]
             [frontend.handler.search :as search-handler]
             [frontend.state :as state]
-            [frontend.text :as text]
+            [logseq.graph-parser.text :as text]
             [frontend.util :as util]
-            [logseq.graph-parser.util :as gp-util]
             [medley.core :as medley]
             [reitit.frontend.easy :as rfe]))
 
@@ -78,11 +78,11 @@
     "Create a new page"
     :page
     (let [name (:name path-params)
-          block? (gp-util/uuid-string? name)]
+          block? (util/uuid-string? name)]
       (if block?
         (if-let [block (db/entity [:block/uuid (medley/uuid name)])]
           (let [content (text/remove-level-spaces (:block/content block)
-                                                  (:block/format block))]
+                                                  (:block/format block) (config/get-block-pattern (:block/format block)))]
             (if (> (count content) 48)
               (str (subs content 0 48) "...")
               content))

src/main/frontend/handler/search.cljs

@@ -1,12 +1,13 @@
 (ns frontend.handler.search
   (:require [clojure.string :as string]
+            [frontend.config :as config]
             [frontend.db :as db]
             [frontend.handler.notification :as notification-handler]
             [frontend.search :as search]
             [frontend.state :as state]
             [frontend.util :as util]
             [promesa.core :as p]
-            [frontend.text :as text]
+            [logseq.graph-parser.text :as text]
             [frontend.util.drawer :as drawer]
             [frontend.util.property :as property]))
 
@@ -21,7 +22,7 @@
 (defn sanity-search-content
   "Convert a block to the display contents for searching"
   [format content]
-  (->> (text/remove-level-spaces content format)
+  (->> (text/remove-level-spaces content format (config/get-block-pattern format))
        (drawer/remove-logbook)
        (property/remove-built-in-properties format)))
 
@@ -80,4 +81,4 @@
        (p/let [_ (search/rebuild-indices! repo)]
          (notification-handler/show!
           "Stale search cache detected. Search indices rebuilt successfully!"
-          :success))))))
+          :success))))))

src/main/frontend/handler/ui.cljs

@@ -14,7 +14,6 @@
             [clojure.string :as string]
             [rum.core :as rum]
             [frontend.mobile.util :as mobile]
-            [logseq.graph-parser.util :as gp-util]
             [electron.ipc :as ipc]))
 
 (defn- get-css-var-value
@@ -112,7 +111,7 @@
   (let [id (and
             (> (count fragment) 36)
             (subs fragment (- (count fragment) 36)))]
-    (if (and id (gp-util/uuid-string? id))
+    (if (and id (util/uuid-string? id))
       (let [elements (array-seq (js/document.getElementsByClassName id))]
         (when (first elements)
           (util/scroll-to-element (gobj/get (first elements) "id")))

src/main/frontend/mobile/intent.cljs

@@ -10,7 +10,8 @@
             [frontend.date :as date]
             [frontend.util :as util]
             [frontend.config :as config]
-            [frontend.format.mldoc :as mldoc]
+            [logseq.graph-parser.mldoc :as gp-mldoc]
+            [logseq.graph-parser.config :as gp-config]
             ["path" :as path]
             [frontend.mobile.util :as mobile-util]
             [frontend.handler.notification :as notification]
@@ -23,11 +24,11 @@
                  (string/lower-case (date/journal-name)))
         format (db/get-page-format page)
         time (date/get-current-time)
-        url (if (and (mldoc/link? format title) (not url))
+        url (if (and (gp-mldoc/link? format title) (not url))
               title
               url)
         text (if (= url title) nil title)
-        [text url] (if (or (mldoc/link? format url) (not url))
+        [text url] (if (or (gp-mldoc/link? format url) (not url))
                      [text url]
                      (string/split url "\"\n"))
         text (some-> text (string/replace #"^\"" ""))
@@ -109,10 +110,11 @@
           format (db/get-page-format page)
           application-type (last (string/split type "/"))
           content (cond
-                    (config/mldoc-support? application-type)
+                    (gp-config/mldoc-support? application-type)
                     (embed-text-file url title)
 
-                    (contains? (set/union #{:pdf} config/media-formats) (keyword application-type))
+                    (contains? (set/union (config/doc-formats) config/media-formats)
+                               (keyword application-type))
                     (embed-asset-file url format)
 
                     :else

src/main/frontend/modules/file/core.cljs

@@ -125,7 +125,7 @@
                     (config/get-pages-directory))
                   "/"
                   (if journal-page?
-                    (date/journal-title->default title)
+                    (date/date->file-name journal-page?)
                     (-> (or (:block/original-name page) (:block/name page))
                         (util/file-name-sanity))) "."
                   (if (= format "markdown") "md" format))

src/main/frontend/modules/outliner/datascript.cljc

@@ -45,7 +45,7 @@
            txs (map (fn [m] (if (map? m)
                               (dissoc m
                                       :block/children :block/meta :block/top? :block/bottom? :block/anchor
-                                      :block/title :block/body :block/level :block/container)
+                                      :block/title :block/body :block/level :block/container :db/other-tx)
                               m)) txs)]
        (when (and (seq txs)
                   (not (:skip-transact? opts)))

src/main/frontend/modules/outliner/transaction.cljc

@@ -18,18 +18,23 @@
     (delete-blocks! ...))"
   [opts & body]
   (assert (map? opts))
-  `(if (some? frontend.modules.outliner.core/*transaction-data*)
-     (do ~@body)
-     (binding [frontend.modules.outliner.core/*transaction-data* (transient [])]
-       ~@body
-       (let [r# (persistent! frontend.modules.outliner.core/*transaction-data*)
-             tx# (mapcat :tx-data r#)
-             ;; FIXME: should we merge all the tx-meta?
-             tx-meta# (first (map :tx-meta r#))
-             all-tx# (concat tx# (:additional-tx ~opts))
-             opts# (merge (dissoc ~opts :additional-tx) tx-meta#)]
-         (when (seq all-tx#)
-           (let [result# (frontend.modules.outliner.datascript/transact! all-tx# opts#)]
-             {:tx-report result#
-              :tx-data all-tx#
-              :tx-meta tx-meta#}))))))
+  `(let [transact-data# frontend.modules.outliner.core/*transaction-data*
+         opts# (if transact-data#
+                 (assoc ~opts :nested-transaction? true)
+                 ~opts)]
+     (if transact-data#
+       (do ~@body)
+       (binding [frontend.modules.outliner.core/*transaction-data* (transient [])]
+         ~@body
+         (let [r# (persistent! frontend.modules.outliner.core/*transaction-data*)
+               tx# (mapcat :tx-data r#)
+               ;; FIXME: should we merge all the tx-meta?
+               tx-meta# (first (map :tx-meta r#))
+               all-tx# (concat tx# (:additional-tx opts#))
+               opts## (merge (dissoc opts# :additional-tx) tx-meta#)]
+           (when (seq all-tx#)
+             (when-not (:nested-transaction? opts#) ; transact only for the whole transaction
+               (let [result# (frontend.modules.outliner.datascript/transact! all-tx# opts##)]
+                 {:tx-report result#
+                  :tx-data all-tx#
+                  :tx-meta tx-meta#}))))))))

src/main/frontend/modules/outliner/tree.cljs

@@ -1,6 +1,6 @@
 (ns frontend.modules.outliner.tree
   (:require [frontend.db :as db]
-            [logseq.graph-parser.util :as gp-util]
+            [frontend.util :as util]
             [clojure.string :as string]
             [frontend.state :as state]))
 
@@ -45,7 +45,7 @@
 (defn- get-root-and-page
   [repo root-id]
   (if (string? root-id)
-    (if (gp-util/uuid-string? root-id)
+    (if (util/uuid-string? root-id)
       [false (db/entity repo [:block/uuid (uuid root-id)])]
       [true (db/entity repo [:block/name (string/lower-case root-id)])])
     [false root-id]))

src/main/frontend/search.cljs

@@ -1,7 +1,7 @@
 (ns frontend.search
   (:require [cljs-bean.core :as bean]
             [clojure.string :as string]
-            [frontend.config :as config]
+            [logseq.graph-parser.config :as gp-config]
             [frontend.db :as db]
             [frontend.regex :as regex]
             [frontend.search.browser :as search-browser]
@@ -143,7 +143,7 @@
   ([q limit]
    (let [q (clean-str q)]
      (when-not (string/blank? q)
-       (let [mldoc-exts (set (map name config/mldoc-support-formats))
+       (let [mldoc-exts (set (map name gp-config/mldoc-support-formats))
              files (->> (db/get-files (state/get-current-repo))
                         (map first)
                         (remove (fn [file]
@@ -237,4 +237,4 @@
 (defn cache-stale?
   [repo]
   (when-let [engine (get-engine repo)]
-    (protocol/cache-stale? engine repo)))
+    (protocol/cache-stale? engine repo)))

src/main/frontend/security.cljs

@@ -1,6 +1,6 @@
 (ns frontend.security
   (:require [clojure.walk :as walk]
-            [logseq.graph-parser.util :as gp-util]))
+            [frontend.util :as util]))
 
 ;; To prevent from cross-site scripting vulnerability, we should add security checks for both hiccup and raw html.
 ;; Hiccup: [:a {:href "javascript:alert('hei')"} "click me"]
@@ -12,7 +12,7 @@
    (= :a (first f))
    (:href (second f))
    (:href (second f))
-   (gp-util/safe-re-find #"(?i)javascript" (:href (second f)))))
+   (util/safe-re-find #"(?i)javascript" (:href (second f)))))
 
 (defn remove-javascript-links-in-href
   [hiccup]

src/main/frontend/state.cljs

@@ -13,7 +13,6 @@
             [goog.object :as gobj]
             [promesa.core :as p]
             [rum.core :as rum]
-            [logseq.graph-parser.util :as gp-util]
             [frontend.mobile.util :as mobile-util]))
 
 (defonce ^:large-vars/data-var state
@@ -182,7 +181,9 @@
      :graph/parsing-state                   {}
 
      ;; copied blocks
-     :copy/blocks                           {:copy/content nil :copy/block-ids nil}
+     :copy/blocks                           {:copy/content nil
+                                             :copy/block-ids nil
+                                             :copy/graph nil}
 
      :copy/export-block-text-indent-style   (or (storage/get :copy/export-block-text-indent-style)
                                                 "dashes")
@@ -450,7 +451,7 @@
     (or
       (when-let [workflow (:preferred-workflow (get-config))]
         (let [workflow (name workflow)]
-          (if (gp-util/safe-re-find #"now|NOW" workflow)
+          (if (util/safe-re-find #"now|NOW" workflow)
             :now
             :todo)))
       (get-in @state [:me :preferred_workflow] :now))))
@@ -1428,13 +1429,15 @@
 
 (defn set-copied-blocks
   [content ids]
-  (set-state! :copy/blocks {:copy/content content
+  (set-state! :copy/blocks {:copy/graph (get-current-repo)
+                            :copy/content content
                             :copy/block-ids ids
                             :copy/full-blocks nil}))
 
 (defn set-copied-full-blocks
   [content blocks]
-  (set-state! :copy/blocks {:copy/content content
+  (set-state! :copy/blocks {:copy/graph (get-current-repo)
+                            :copy/content content
                             :copy/full-blocks blocks}))
 
 (defn set-copied-full-blocks!
@@ -1666,6 +1669,7 @@
 
 (defn get-file-sync-manager []
   (:file-sync/sync-manager @state))
+
 (defn get-file-sync-state []
   (:file-sync/sync-state @state))
 

src/main/frontend/ui.cljs

@@ -25,6 +25,7 @@
             ["react-tippy" :as react-tippy]
             ["react-transition-group" :refer [CSSTransition TransitionGroup]]
             ["@logseq/react-tweet-embed" :as react-tweet-embed]
+            ["react-visibility-sensor" :as rvs]
             [rum.core :as rum]
             [frontend.db-mixins :as db-mixins]
             [frontend.mobile.util :as mobile-util]
@@ -37,6 +38,7 @@
 (def resize-consumer (r/adapt-class (gobj/get Resize "ResizeConsumer")))
 (def Tippy (r/adapt-class (gobj/get react-tippy "Tooltip")))
 (def ReactTweetEmbed (r/adapt-class react-tweet-embed))
+(def visibility-sensor (r/adapt-class (gobj/get rvs "default")))
 
 (defn reset-ios-whole-page-offset!
   []
@@ -682,7 +684,7 @@
                         :margin-left -30}}
              (not title-trigger?)
              (assoc :on-mouse-down on-mouse-down))
-           [:span {:class (if @control? "control-show cursor-pointer" "control-hide")}
+           [:span {:class (if (or @control? @collapsed?) "control-show cursor-pointer" "control-hide")}
             (rotating-arrow @collapsed?)]])
         (if (fn? header)
           (header @collapsed?)
@@ -827,11 +829,9 @@
                     :open (if manual open? @*mounted?)
                     :trigger (if manual "manual" "mouseenter focus")
                     ;; See https://github.com/tvkhoa/react-tippy/issues/13
-                    :popperOptions (if fixed-position?
-                                     {:modifiers {:flip {:enabled false}
-                                                  :hide {:enabled false}
-                                                  :preventOverflow {:enabled false}}}
-                                     {})
+                    :popperOptions {:modifiers {:flip {:enabled (not fixed-position?)}
+                                                :hide {:enabled false}
+                                                :preventOverflow {:enabled false}}}
                     :onShow #(reset! *mounted? true)
                     :onHide #(reset! *mounted? false)}
                    opts)
@@ -908,3 +908,52 @@
     [:span.text-sm.font-medium
      label-right]]
    (progress-bar width)])
+
+(rum/defcs lazy-visible-inner <
+  {:init (fn [state]
+           (assoc state
+                  ::ref (atom nil)
+                  ::height (atom 26)))
+   :did-mount (fn [state]
+                (when (last (:rum/args state))
+                  (let [observer (js/ResizeObserver. (fn [entries]
+                                                      (let [entry (first entries)
+                                                            *height (::height state)
+                                                            height' (.-height (.-contentRect entry))]
+                                                        (when (and (> height' @*height)
+                                                                   (not= height' 64))
+                                                          (reset! *height height')))))]
+                   (.observe observer @(::ref state))))
+                state)}
+  [state visible? content-fn _reset-height?]
+  [:div.lazy-visibility {:ref #(reset! (::ref state) %)
+                         :style {:min-height @(::height state)}}
+
+   (if visible?
+     (when (fn? content-fn) (content-fn))
+     [:div.shadow.rounded-md.p-4.w-full.mx-auto {:style {:height 64}}
+      [:div.animate-pulse.flex.space-x-4
+       [:div.flex-1.space-y-3.py-1
+        [:div.h-2.bg-base-4.rounded]
+        [:div.space-y-3
+         [:div.grid.grid-cols-3.gap-4
+          [:div.h-2.bg-base-4.rounded.col-span-2]
+          [:div.h-2.bg-base-4.rounded.col-span-1]]
+         [:div.h-2.bg-base-4.rounded]]]]])])
+
+(rum/defcs lazy-visible <
+  (rum/local false ::visible?)
+  [state content-fn sensor-opts reset-height?]
+  (if (or (util/mobile?) (mobile-util/is-native-platform?))
+    (content-fn)
+    (let [*visible? (::visible? state)]
+      (visibility-sensor
+       (merge
+        {:on-change #(reset! *visible? %)
+         :partialVisibility true
+         :offset {:top -300
+                  :bottom -300}
+         :scrollCheck true
+         :scrollThrottle 1}
+        sensor-opts)
+       (lazy-visible-inner @*visible? content-fn reset-height?)))))

src/main/frontend/util.cljc

@@ -39,6 +39,23 @@
 #?(:cljs (defn app-scroll-container-node []
            (gdom/getElement "main-content-container")))
 
+#?(:cljs
+   (defn safe-re-find
+     [pattern s]
+     (when-not (string? s)
+       ;; TODO: sentry
+       (js/console.trace))
+     (when (string? s)
+       (re-find pattern s))))
+
+#?(:cljs
+  (do
+    (def uuid-pattern "[0-9a-f]{8}-[0-9a-f]{4}-[0-5][0-9a-f]{3}-[089ab][0-9a-f]{3}-[0-9a-f]{12}")
+    (defonce exactly-uuid-pattern (re-pattern (str "(?i)^" uuid-pattern "$")))
+    (defn uuid-string?
+      [s]
+      (safe-re-find exactly-uuid-pattern s))))
+
 #?(:cljs
    (defn ios?
      []
@@ -55,7 +72,7 @@
    (defn mobile?
      []
      (when-not node-test?
-       (gp-util/safe-re-find #"Mobi" js/navigator.userAgent))))
+       (safe-re-find #"Mobi" js/navigator.userAgent))))
 
 #?(:cljs
    (defn electron?
@@ -351,7 +368,7 @@
 #?(:cljs
    (defn scroll-to-element
      [elem-id]
-     (when-not (gp-util/safe-re-find #"^/\d+$" elem-id)
+     (when-not (safe-re-find #"^/\d+$" elem-id)
        (when elem-id
          (when-let [elem (gdom/getElement elem-id)]
            (.scroll (app-scroll-container-node)
@@ -715,11 +732,6 @@
 (defn drop-nth [n coll]
   (keep-indexed #(when (not= %1 n) %2) coll))
 
-(defn capitalize-all [s]
-  (some->> (string/split s #" ")
-           (map string/capitalize)
-           (string/join " ")))
-
 #?(:cljs
    (defn react
      [ref]
@@ -836,8 +848,8 @@
      []
      (let [user-agent js/navigator.userAgent
            vendor js/navigator.vendor]
-       (and (gp-util/safe-re-find #"Chrome" user-agent)
-            (gp-util/safe-re-find #"Google Inc" vendor)))))
+       (and (safe-re-find #"Chrome" user-agent)
+            (safe-re-find #"Google Inc" vendor)))))
 
 #?(:cljs
    (defn indexeddb-check?
@@ -878,7 +890,7 @@
      [block-id]
      (when block-id
        (let [block-id (str block-id)]
-         (when (gp-util/uuid-string? block-id)
+         (when (uuid-string? block-id)
            (first (array-seq (js/document.getElementsByClassName block-id))))))))
 
 #?(:cljs
@@ -897,7 +909,7 @@
    (do
      (defn include-windows-reserved-chars?
       [s]
-       (gp-util/safe-re-find windows-reserved-chars s))
+       (safe-re-find windows-reserved-chars s))
 
      (defn create-title-property?
        [s]
@@ -909,44 +921,18 @@
                 (string/includes? s "%")
                 (string/includes? s "#"))))))
 
-(defn remove-boundary-slashes
-  [s]
-  (when (string? s)
-    (let [s (if (= \/ (first s))
-              (subs s 1)
-              s)]
-      (if (= \/ (last s))
-        (subs s 0 (dec (count s)))
-        s))))
-
-(defn normalize
-  [s]
-  (.normalize s "NFC"))
-
 #?(:cljs
    (defn search-normalize
      "Normalize string for searching (loose)"
      [s]
      (removeAccents (.normalize (string/lower-case s) "NFKC"))))
 
-(defn page-name-sanity
-  "Sanitize the page-name."
-  ([page-name]
-   (page-name-sanity page-name false))
-  ([page-name replace-slash?]
-   (let [page (some-> page-name
-                      (remove-boundary-slashes)
-                      (normalize))]
-     (if replace-slash?
-       (string/replace page #"/" "%2A")
-       page))))
-
 #?(:cljs
    (defn file-name-sanity
      "Sanitize page-name for file name (strict), for file writing."
      [page-name]
      (some-> page-name
-             page-name-sanity
+             gp-util/page-name-sanity
              ;; for android filesystem compatiblity
              (string/replace #"[\\#|%]+" url-encode)
              ;; Windows reserved path characters
@@ -954,15 +940,16 @@
              (string/replace #"/" url-encode)
              (string/replace "*" "%2A"))))
 
-(defn page-name-sanity-lc
-  "Sanitize the query string for a page name (mandate for :block/name)"
-  [s]
-  (page-name-sanity (string/lower-case s)))
+#?(:cljs
+   (def page-name-sanity-lc
+     "Delegate to gp-util to loosely couple app usages to graph-parser"
+     gp-util/page-name-sanity-lc))
 
-(defn safe-page-name-sanity-lc
-  [s]
-  (if (string? s)
-    (page-name-sanity-lc s) s))
+#?(:cljs
+ (defn safe-page-name-sanity-lc
+   [s]
+   (if (string? s)
+     (page-name-sanity-lc s) s)))
 
 (defn get-page-original-name
   [page]
@@ -1242,17 +1229,6 @@
   [text]
   (string/join (replace regex-char-esc-smap text)))
 
-(defn split-namespace-pages
-  [title]
-  (let [parts (string/split title "/")]
-    (loop [others (rest parts)
-           result [(first parts)]]
-      (if (seq others)
-        (let [prev (last result)]
-          (recur (rest others)
-                 (conj result (str prev "/" (first others)))))
-        result))))
-
 (comment
   (re-matches (re-pattern (regex-escape "$u^8(d)+w.*[dw]d?")) "$u^8(d)+w.*[dw]d?"))
 
@@ -1260,14 +1236,6 @@
    (defn meta-key-name []
      (if mac? "Cmd" "Ctrl")))
 
-(defn wrapped-by-quotes?
-  [v]
-  (and (string? v) (>= (count v) 2) (= "\"" (first v) (last v))))
-
-(defn unquote-string
-  [v]
-  (string/trim (subs v 1 (dec (count v)))))
-
 #?(:cljs
    (defn right-click?
      [e]
@@ -1276,16 +1244,6 @@
        (or (= which 3)
            (= button 2)))))
 
-#?(:cljs
-   (defn url?
-     [s]
-     (and (string? s)
-          (try
-            (js/URL. s)
-            true
-            (catch js/Error _e
-              false)))))
-
 #?(:cljs
    (defn make-el-into-center-viewport
      [^js/HTMLElement el]

src/main/frontend/util/drawer.cljs

@@ -1,8 +1,8 @@
 (ns frontend.util.drawer
   (:require [clojure.string :as string]
             [frontend.util :as util]
-            [logseq.graph-parser.util :as gp-util]
-            [frontend.util.property :as property]
+            [logseq.graph-parser.mldoc :as gp-mldoc]
+            [logseq.graph-parser.property :as gp-property]
             [frontend.format.mldoc :as mldoc]))
 
 (defn drawer-start
@@ -23,7 +23,7 @@
 
 (defn get-drawer-ast
   [format content typ]
-  (let [ast (mldoc/->edn content (mldoc/default-config format))
+  (let [ast (mldoc/->edn content (gp-mldoc/default-config format))
         typ-drawer (ffirst (filter (fn [x]
                                      (mldoc/typ-drawer? x typ)) ast))]
     typ-drawer))
@@ -32,7 +32,7 @@
   [format content typ value]
   (when (string? content)
     (try
-      (let [ast (mldoc/->edn content (mldoc/default-config format))
+      (let [ast (mldoc/->edn content (gp-mldoc/default-config format))
             has-properties? (some (fn [x] (mldoc/properties? x)) ast)
             has-typ-drawer? (some (fn [x] (mldoc/typ-drawer? x typ)) ast)
             lines (string/split-lines content)
@@ -54,8 +54,8 @@
                         (if has-properties?
                           (cond
                             (= :org format)
-                            (let [prop-start-idx (.indexOf body-without-timestamps property/properties-start)
-                                  prop-end-idx (.indexOf body-without-timestamps property/properties-end)
+                            (let [prop-start-idx (.indexOf body-without-timestamps gp-property/properties-start)
+                                  prop-end-idx (.indexOf body-without-timestamps gp-property/properties-end)
                                   properties (subvec body-without-timestamps prop-start-idx (inc prop-end-idx))
                                   after (subvec body-without-timestamps (inc prop-end-idx))]
                               (string/join "\n" (concat [title] scheduled deadline properties [drawer] after)))
@@ -87,8 +87,8 @@
 
 (defn contains-logbook?
   [content]
-  (and (gp-util/safe-re-find (re-pattern (str "(?i)" logbook-start)) content)
-       (gp-util/safe-re-find (re-pattern (str "(?i)" drawer-end)) content)))
+  (and (util/safe-re-find (re-pattern (str "(?i)" logbook-start)) content)
+       (util/safe-re-find (re-pattern (str "(?i)" drawer-end)) content)))
 
 ;; TODO: DRY
 (defn remove-logbook

src/main/frontend/util/marker.cljs

@@ -1,7 +1,6 @@
 (ns frontend.util.marker
   (:require [clojure.string :as string]
-            [frontend.util :as util]
-            [logseq.graph-parser.util :as gp-util]))
+            [frontend.util :as util]))
 
 (defn marker-pattern [format]
   (re-pattern
@@ -21,7 +20,7 @@
         (if-let [matches (seq (util/re-pos new-line-re-pattern content))]
           (let [[start-pos content] (last matches)]
             (+ start-pos (count content)))
-          (count (gp-util/safe-re-find re-pattern content)))
+          (count (util/safe-re-find re-pattern content)))
         new-content
         (str (subs content 0 pos)
              (string/replace-first (subs content pos)
@@ -60,6 +59,6 @@
   (let [content    (string/triml content)
         new-marker (or new-marker
                        (cycle-marker-state (or marker
-                                               (last (gp-util/safe-re-find (marker-pattern format) content))) ; Returns the last matching group (last vec)
+                                               (last (util/safe-re-find (marker-pattern format) content))) ; Returns the last matching group (last vec)
                                            preferred-workflow))]
     [(add-or-update-marker content format new-marker) new-marker]))

src/main/frontend/util/priority.cljs

@@ -1,13 +1,12 @@
 (ns frontend.util.priority
   (:require [clojure.string :as string]
             [frontend.util :as util]
-            [logseq.graph-parser.util :as gp-util]
             [frontend.util.marker :as marker]))
 
 (defn cycle-priority-state
   [content]
   (let [priority-reg #"\[#([ABC]{1})\]\s{1}"
-        priority (last (gp-util/safe-re-find priority-reg content))
+        priority (last (util/safe-re-find priority-reg content))
         next-priority (case priority
                         "A" "B"
 
@@ -29,7 +28,7 @@
         (if-let [matches (seq (util/re-pos new-line-re-pattern content))]
           (let [[start-pos content] (last matches)]
             (+ start-pos (count content)))
-          (count (gp-util/safe-re-find re-pattern content)))
+          (count (util/safe-re-find re-pattern content)))
         skip-marker-pos
         (if-let [matches (seq (util/re-pos marker/bare-marker-pattern (subs content skip-hash-pos)))]
           (let [[start-pos content] (last matches)]

src/main/frontend/util/property.cljs

@@ -1,19 +1,17 @@
 (ns frontend.util.property
+  "Property fns needed by the rest of the app and not graph-parser"
   (:require [clojure.string :as string]
             [frontend.util :as util]
             [clojure.set :as set]
             [frontend.config :as config]
             [medley.core :as medley]
             [logseq.graph-parser.util :as gp-util]
+            [logseq.graph-parser.mldoc :as gp-mldoc]
+            [logseq.graph-parser.property :as gp-property :refer [properties-start properties-end]]
             [frontend.format.mldoc :as mldoc]
-            [frontend.text :as text]
+            [logseq.graph-parser.text :as text]
             [frontend.util.cursor :as cursor]))
 
-(defonce properties-start ":PROPERTIES:")
-(defonce properties-end ":END:")
-(defonce properties-end-pattern
-  (re-pattern (util/format "%s[\t\r ]*\n|(%s\\s*$)" properties-end properties-end)))
-
 (def built-in-extended-properties (atom #{}))
 (defn register-built-in-properties
   [props]
@@ -35,15 +33,9 @@
              built-in-properties-set (built-in-properties)]
          (every? built-in-properties-set ks))))
 
-(defn contains-properties?
-  [content]
-  (when content
-    (and (string/includes? content properties-start)
-         (gp-util/safe-re-find properties-end-pattern content))))
-
 (defn remove-empty-properties
   [content]
-  (if (contains-properties? content)
+  (if (gp-property/contains-properties? content)
     (string/replace content
                     (re-pattern ":PROPERTIES:\n+:END:\n*")
                     "")
@@ -53,28 +45,28 @@
   [line]
   (boolean
    (and (string? line)
-        (gp-util/safe-re-find #"^\s?[^ ]+:: " line))))
+        (util/safe-re-find #"^\s?[^ ]+:: " line))))
 
 (defn front-matter-property?
   [line]
   (boolean
    (and (string? line)
-        (gp-util/safe-re-find #"^\s*[^ ]+: " line))))
+        (util/safe-re-find #"^\s*[^ ]+: " line))))
 
 (defn get-property-key
   [line format]
   (and (string? line)
        (when-let [key (last
                        (if (= format :org)
-                         (gp-util/safe-re-find #"^\s*:([^: ]+): " line)
-                         (gp-util/safe-re-find #"^\s*([^ ]+):: " line)))]
+                         (util/safe-re-find #"^\s*:([^: ]+): " line)
+                         (util/safe-re-find #"^\s*([^ ]+):: " line)))]
          (keyword key))))
 
 (defn org-property?
   [line]
   (boolean
    (and (string? line)
-        (gp-util/safe-re-find #"^\s*:[^: ]+: " line)
+        (util/safe-re-find #"^\s*:[^: ]+: " line)
         (when-let [key (get-property-key line :org)]
           (not (contains? #{:PROPERTIES :END} key))))))
 
@@ -113,7 +105,7 @@
 (defn get-property-keys
   [format content]
   (cond
-    (contains-properties? content)
+    (gp-property/contains-properties? content)
     (get-org-property-keys content)
 
     (= :markdown format)
@@ -133,7 +125,7 @@
 (defn remove-properties
   [format content]
   (cond
-    (contains-properties? content)
+    (gp-property/contains-properties? content)
     (let [lines (string/split-lines content)
           [title-lines properties&body] (split-with #(-> (string/triml %)
                                                          string/upper-case
@@ -187,7 +179,7 @@
         properties (filter (fn [[k _v]] ((built-in-properties) k)) properties)]
     (if (seq properties)
       (let [lines (string/split-lines content)
-            ast (mldoc/->edn content (mldoc/default-config format))
+            ast (mldoc/->edn content (gp-mldoc/default-config format))
             [title body] (if (mldoc/block-with-title? (first (ffirst ast)))
                            [(first lines) (rest lines)]
                            [nil lines])
@@ -233,10 +225,15 @@
    (insert-property format content key value false))
   ([format content key value front-matter?]
    (when (string? content)
-     (let [ast (mldoc/->edn content (mldoc/default-config format))
+     (let [ast (mldoc/->edn content (gp-mldoc/default-config format))
            title? (mldoc/block-with-title? (ffirst (map first ast)))
            has-properties? (or (and title?
-                                    (mldoc/properties? (second ast)))
+                                    (or (mldoc/properties? (second ast))
+                                        (mldoc/properties? (second
+                                                            (remove
+                                                             (fn [[x _]]
+                                                               (= "Hiccup" (first x)))
+                                                             ast)))))
                                (mldoc/properties? (first ast)))
            lines (string/split-lines content)
            [title body] (if title?
@@ -342,7 +339,7 @@
      (let [format (or format :markdown)
            key (string/lower-case (name key))
            remove-f (if first? util/remove-first remove)]
-       (if (and (= format :org) (not (contains-properties? content)))
+       (if (and (= format :org) (not (gp-property/contains-properties? content)))
          content
          (let [lines (->> (string/split-lines content)
                           (remove-f (fn [line]
@@ -365,31 +362,6 @@
       (string/replace-first content (re-pattern ":PROPERTIES:\n:END:\n*") "")
       content)))
 
-(defn ->new-properties
-  "New syntax: key:: value"
-  [content]
-  (if (contains-properties? content)
-    (let [lines (string/split-lines content)
-          start-idx (.indexOf lines properties-start)
-          end-idx (.indexOf lines properties-end)]
-      (if (and (>= start-idx 0) (> end-idx 0) (> end-idx start-idx))
-        (let [before (subvec lines 0 start-idx)
-              middle (->> (subvec lines (inc start-idx) end-idx)
-                          (map (fn [text]
-                                 (let [[k v] (gp-util/split-first ":" (subs text 1))]
-                                   (if (and k v)
-                                     (let [k (string/replace k "_" "-")
-                                           compare-k (keyword (string/lower-case k))
-                                           k (if (contains? #{:id :custom_id :custom-id} compare-k) "id" k)
-                                           k (if (contains? #{:last-modified-at} compare-k) "updated-at" k)]
-                                       (str k ":: " (string/trim v)))
-                                     text)))))
-              after (subvec lines (inc end-idx))
-              lines (concat before middle after)]
-          (string/join "\n" lines))
-        content))
-    content))
-
 (defn add-page-properties
   [page-format properties-content properties]
   (let [properties (medley/map-keys name properties)
@@ -429,10 +401,3 @@
     (util/format
      (config/properties-wrapper-pattern page-format)
      (string/join "\n" lines))))
-
-(defn properties-ast?
-  [block]
-  (and
-   (vector? block)
-   (contains? #{"Property_Drawer" "Properties"}
-              (first block))))

src/main/frontend/util/thingatpt.cljs

@@ -1,10 +1,10 @@
 (ns frontend.util.thingatpt
   (:require [clojure.string :as string]
             [frontend.state :as state]
-            [frontend.util.property :as property-util]
             [frontend.util.cursor :as cursor]
             [frontend.config :as config]
-            [frontend.text :as text]
+            [logseq.graph-parser.text :as text]
+            [logseq.graph-parser.property :as gp-property]
             [cljs.reader :as reader]
             [goog.object :as gobj]))
 
@@ -69,8 +69,8 @@
   (when-let [properties
              (case (state/get-preferred-format) ;; TODO fix me to block's format
                :org (thing-at-point
-                     [property-util/properties-start
-                      property-util/properties-end]
+                     [gp-property/properties-start
+                      gp-property/properties-end]
                      input)
                (when-let [line (line-at-point input)]
                  (when (re-matches #"^[^\s.]+:: .*$" (:raw-content line))

src/main/logseq/graph_parser/block.cljc

@@ -0,0 +1,653 @@
+(ns ^:nbb-compatible logseq.graph-parser.block
+  ;; Disable clj linters since we don't support clj
+  #?(:clj {:clj-kondo/config {:linters {:unresolved-namespace {:level :off}
+                                        :unresolved-symbol {:level :off}}}})
+  "Block related code needed for graph-parser"
+  (:require [clojure.string :as string]
+            [clojure.walk :as walk]
+            [datascript.core :as d]
+            [logseq.graph-parser.text :as text]
+            [logseq.graph-parser.utf8 :as utf8]
+            [logseq.graph-parser.property :as gp-property]
+            [logseq.graph-parser.util :as gp-util]
+            [logseq.graph-parser.config :as gp-config]
+            [logseq.graph-parser.mldoc :as gp-mldoc]
+            [logseq.graph-parser.date-time-util :as date-time-util]
+            #?(:org.babashka/nbb [logseq.graph-parser.log :as log]
+               :default [lambdaisland.glogi :as log])))
+
+(defn heading-block?
+  [block]
+  (and
+   (vector? block)
+   (= "Heading" (first block))))
+
+(defn get-tag
+  [block]
+  (when-let [tag-value (and (vector? block)
+                            (= "Tag" (first block))
+                            (second block))]
+    (->> tag-value
+         (map (fn [[elem value]]
+                (case elem
+                  "Plain" value
+                  "Link" (:full_text value)
+                  "Nested_link" (:content value)
+                  "")))
+         (string/join))))
+
+(defn- get-page-reference
+  [block supported-formats]
+  (let [page (cond
+               (and (vector? block) (= "Link" (first block)))
+               (let [typ (first (:url (second block)))
+                     value (second (:url (second block)))]
+                 ;; {:url ["File" "file:../pages/hello_world.org"], :label [["Plain" "hello world"]], :title nil}
+                 (or
+                  (and
+                   (= typ "Page_ref")
+                   (and (string? value)
+                        (not (or (gp-config/local-asset? value)
+                                 (gp-config/draw? value))))
+                   value)
+
+                  (and
+                   (= typ "Search")
+                   (text/page-ref? value)
+                   (text/page-ref-un-brackets! value))
+
+                  (and
+                   (= typ "Search")
+                   (not (contains? #{\# \* \/ \[} (first value)))
+                   (let [ext (some-> (gp-util/get-file-ext value) keyword)]
+                     (when (and (not (string/starts-with? value "http:"))
+                                (not (string/starts-with? value "https:"))
+                                (not (string/starts-with? value "file:"))
+                                (not (gp-config/local-asset? value))
+                                (or (#{:excalidraw :tldr} ext)
+                                    (not (contains? supported-formats ext))))
+                       value)))
+
+                  (and
+                   (= typ "Complex")
+                   (= (:protocol value) "file")
+                   (:link value))
+
+                  (and
+                   (= typ "File")
+                   (second (first (:label (second block)))))))
+
+               (and (vector? block) (= "Nested_link" (first block)))
+               (let [content (:content (last block))]
+                 (subs content 2 (- (count content) 2)))
+
+               (and (vector? block)
+                    (= "Macro" (first block)))
+               (let [{:keys [name arguments]} (second block)
+                     argument (string/join ", " arguments)]
+                   (when (= name "embed")
+                     (text/page-ref-un-brackets! argument)))
+
+               (and (vector? block)
+                    (= "Tag" (first block)))
+               (let [text (get-tag block)]
+                 (text/page-ref-un-brackets! text))
+
+               :else
+               nil)]
+    (text/block-ref-un-brackets! page)))
+
+(defn- get-block-reference
+  [block]
+  (when-let [block-id (cond
+                        (and (vector? block)
+                             (= "Block_reference" (first block)))
+                        (last block)
+
+                        (and (vector? block)
+                             (= "Link" (first block))
+                             (map? (second block))
+                             (= "Block_ref" (first (:url (second block)))))
+                        (second (:url (second block)))
+
+                        (and (vector? block)
+                             (= "Macro" (first block)))
+                        (let [{:keys [name arguments]} (second block)]
+                          (when (and (= name "embed")
+                                     (string? (first arguments))
+                                     (string/starts-with? (first arguments) "((")
+                                     (string/ends-with? (first arguments) "))"))
+                            (subs (first arguments) 2 (- (count (first arguments)) 2))))
+
+                        (and (vector? block)
+                             (= "Link" (first block))
+                             (map? (second block)))
+                        (if (= "id" (:protocol (second (:url (second block)))))
+                          (:link (second (:url (second block))))
+                          (let [id (second (:url (second block)))]
+                            (text/block-ref-un-brackets! id)))
+
+                        :else
+                        nil)]
+    (when (and block-id
+               (gp-util/uuid-string? block-id))
+      block-id)))
+
+(defn- paragraph-block?
+  [block]
+  (and
+   (vector? block)
+   (= "Paragraph" (first block))))
+
+(defn timestamp-block?
+  [block]
+  (and
+   (vector? block)
+   (= "Timestamp" (first block))))
+
+;; TODO: we should move this to mldoc
+(defn extract-properties
+  [format properties user-config]
+  (when (seq properties)
+    (let [properties (seq properties)
+          page-refs (->>
+                     properties
+                     (remove (fn [[k _]]
+                               (contains? #{:background-color :background_color} (keyword k))))
+                     (map last)
+                     (map (fn [v]
+                            (when (and (string? v)
+                                       (not (gp-mldoc/link? format v)))
+                              (let [v (string/trim v)
+                                    result (text/split-page-refs-without-brackets v {:un-brackets? false})]
+                                (if (coll? result)
+                                  (map text/page-ref-un-brackets! result)
+                                  [])))))
+                     (apply concat)
+                     (remove string/blank?))
+          properties (->> properties
+                          (map (fn [[k v]]
+                                 (let [k (-> (string/lower-case (name k))
+                                             (string/replace " " "-")
+                                             (string/replace "_" "-"))
+                                       k (if (contains? #{"custom_id" "custom-id"} k)
+                                           "id"
+                                           k)
+                                       v (if (coll? v)
+                                           (remove string/blank? v)
+                                           (if (string/blank? v)
+                                             nil
+                                             (text/parse-property format k v user-config)))
+                                       k (keyword k)
+                                       v (if (and
+                                              (string? v)
+                                              (contains? #{:alias :aliases :tags} k))
+                                           (set [v])
+                                           v)
+                                       v (if (coll? v) (set v) v)]
+                                   [k v])))
+                          (remove #(nil? (second %))))]
+      {:properties (into {} properties)
+       :properties-order (map first properties)
+       :page-refs page-refs})))
+
+(defn- paragraph-timestamp-block?
+  [block]
+  (and (paragraph-block? block)
+       (or (timestamp-block? (first (second block)))
+           (timestamp-block? (second (second block))))))
+
+(defn- extract-timestamps
+  [block]
+  (some->>
+   (second block)
+   (filter timestamp-block?)
+   (map last)
+   (into {})))
+
+;; {"Deadline" {:date {:year 2020, :month 10, :day 20}, :wday "Tue", :time {:hour 8, :min 0}, :repetition [["DoublePlus"] ["Day"] 1], :active true}}
+(defn timestamps->scheduled-and-deadline
+  [timestamps]
+  (let [timestamps (gp-util/map-keys (comp keyword string/lower-case) timestamps)
+        m (some->> (select-keys timestamps [:scheduled :deadline])
+                   (map (fn [[k v]]
+                          (let [{:keys [date repetition]} v
+                                {:keys [year month day]} date
+                                day (js/parseInt (str year (gp-util/zero-pad month) (gp-util/zero-pad day)))]
+                            (cond->
+                             (case k
+                               :scheduled
+                               {:scheduled day}
+                               :deadline
+                               {:deadline day})
+                              repetition
+                              (assoc :repeated? true))))))]
+    (apply merge m)))
+
+(defn convert-page-if-journal
+  "Convert journal file name to user' custom date format"
+  [original-page-name date-formatter]
+  (when original-page-name
+    (let [page-name (gp-util/page-name-sanity-lc original-page-name)
+          day (date-time-util/journal-title->int page-name (date-time-util/safe-journal-title-formatters date-formatter))]
+     (if day
+       (let [original-page-name (date-time-util/int->journal-title day date-formatter)]
+         [original-page-name (gp-util/page-name-sanity-lc original-page-name) day])
+       [original-page-name page-name day]))))
+
+(defn page-name->map
+  "Create a page's map structure given a original page name (string).
+   map as input is supported for legacy compatibility.
+   with-timestamp?: assign timestampes to the map structure.
+    Useful when creating new pages from references or namespaces,
+    as there's no chance to introduce timestamps via editing in page"
+  [original-page-name with-id? db with-timestamp? date-formatter]
+  (cond
+    (and original-page-name (string? original-page-name))
+    (let [original-page-name (gp-util/remove-boundary-slashes original-page-name)
+          [original-page-name page-name journal-day] (convert-page-if-journal original-page-name date-formatter)
+          namespace? (and (not (boolean (text/get-nested-page-name original-page-name)))
+                          (text/namespace-page? original-page-name))
+          page-entity (some-> db (d/entity [:block/name page-name]))
+          original-page-name (or (:block/original-name page-entity) original-page-name)]
+      (merge
+       {:block/name page-name
+        :block/original-name original-page-name}
+       (when with-id?
+         (if page-entity
+           {:block/uuid (:block/uuid page-entity)}
+           {:block/uuid (d/squuid)}))
+       (when namespace?
+         (let [namespace (first (gp-util/split-last "/" original-page-name))]
+           (when-not (string/blank? namespace)
+             {:block/namespace {:block/name (gp-util/page-name-sanity-lc namespace)}})))
+       (when (and with-timestamp? (not page-entity)) ;; Only assign timestamp on creating new entity
+         (let [current-ms (date-time-util/time-ms)]
+           {:block/created-at current-ms
+            :block/updated-at current-ms}))
+       (if journal-day
+         {:block/journal? true
+          :block/journal-day journal-day}
+         {:block/journal? false})))
+
+    (and (map? original-page-name) (:block/uuid original-page-name))
+    original-page-name
+
+    (and (map? original-page-name) with-id?)
+    (assoc original-page-name :block/uuid (d/squuid))
+
+    :else
+    nil))
+
+(defn- with-page-refs
+  [{:keys [title body tags refs marker priority] :as block} with-id? supported-formats db date-formatter]
+  (let [refs (->> (concat tags refs [marker priority])
+                  (remove string/blank?)
+                  (distinct))
+        refs (atom refs)]
+    (walk/prewalk
+     (fn [form]
+       ;; skip custom queries
+       (when-not (and (vector? form)
+                      (= (first form) "Custom")
+                      (= (second form) "query"))
+         (when-let [page (get-page-reference form supported-formats)]
+           (swap! refs conj page))
+         (when-let [tag (get-tag form)]
+           (let [tag (text/page-ref-un-brackets! tag)]
+             (when (gp-util/tag-valid? tag)
+               (swap! refs conj tag))))
+         form))
+     (concat title body))
+    (let [refs (remove string/blank? @refs)
+          children-pages (->> (mapcat (fn [p]
+                                        (let [p (if (map? p)
+                                                  (:block/original-name p)
+                                                  p)]
+                                          (when (string? p)
+                                            (let [p (or (text/get-nested-page-name p) p)]
+                                              (when (text/namespace-page? p)
+                                                (gp-util/split-namespace-pages p))))))
+                                      refs)
+                              (remove string/blank?)
+                              (distinct))
+          refs (->> (distinct (concat refs children-pages))
+                    (remove nil?))
+          refs (map (fn [ref] (page-name->map ref with-id? db true date-formatter)) refs)]
+      (assoc block :refs refs))))
+
+(defn- with-block-refs
+  [{:keys [title body] :as block}]
+  (let [ref-blocks (atom nil)]
+    (walk/postwalk
+     (fn [form]
+       (when-let [block (get-block-reference form)]
+         (swap! ref-blocks conj block))
+       form)
+     (concat title body))
+    (let [ref-blocks (->> @ref-blocks
+                          (filter gp-util/uuid-string?))
+          ref-blocks (map
+                       (fn [id]
+                         [:block/uuid (uuid id)])
+                       ref-blocks)
+          refs (distinct (concat (:refs block) ref-blocks))]
+      (assoc block :refs refs))))
+
+(defn- block-keywordize
+  [block]
+  (gp-util/map-keys
+   (fn [k]
+     (if (namespace k)
+       k
+       (keyword "block" k)))
+   block))
+
+(defn- sanity-blocks-data
+  [blocks]
+  (map (fn [block]
+         (if (map? block)
+           (block-keywordize (gp-util/remove-nils block))
+           block))
+       blocks))
+
+(defn- with-path-refs
+  [blocks]
+  (loop [blocks blocks
+         acc []
+         parents []]
+    (if (empty? blocks)
+      acc
+      (let [block (first blocks)
+            cur-level (:block/level block)
+            level-diff (- cur-level
+                          (get (last parents) :block/level 0))
+            [path-refs parents]
+            (cond
+              (zero? level-diff)            ; sibling
+              (let [path-refs (mapcat :block/refs (drop-last parents))
+                    parents (conj (vec (butlast parents)) block)]
+                [path-refs parents])
+
+              (> level-diff 0)              ; child
+              (let [path-refs (mapcat :block/refs parents)]
+                [path-refs (conj parents block)])
+
+              (< level-diff 0)              ; new parent
+              (let [parents (vec (take-while (fn [p] (< (:block/level p) cur-level)) parents))
+                    path-refs (mapcat :block/refs parents)]
+                [path-refs (conj parents block)]))
+            path-ref-pages (->> path-refs
+                                (concat (:block/refs block))
+                                (map (fn [ref]
+                                       (cond
+                                         (map? ref)
+                                         (:block/name ref)
+
+                                         :else
+                                         ref)))
+                                (remove string/blank?)
+                                (map (fn [ref]
+                                       (if (string? ref)
+                                         {:block/name (gp-util/page-name-sanity-lc ref)}
+                                         ref)))
+                                (remove vector?)
+                                (remove nil?)
+                                (distinct))]
+        (recur (rest blocks)
+               (conj acc (assoc block :block/path-refs path-ref-pages))
+               parents)))))
+
+(defn- block-tags->pages
+  [{:keys [tags] :as block}]
+  (if (seq tags)
+    (assoc block :tags (map (fn [tag]
+                              (let [tag (text/page-ref-un-brackets! tag)]
+                                [:block/name (gp-util/page-name-sanity-lc tag)])) tags))
+    block))
+
+(defn- get-block-content
+  [utf8-content block format meta block-pattern]
+  (let [content (if-let [end-pos (:end_pos meta)]
+                  (utf8/substring utf8-content
+                                  (:start_pos meta)
+                                  end-pos)
+                  (utf8/substring utf8-content
+                                  (:start_pos meta)))
+        content (when content
+                  (let [content (text/remove-level-spaces content format block-pattern)]
+                    (if (or (:pre-block? block)
+                            (= (:format block) :org))
+                      content
+                      (gp-mldoc/remove-indentation-spaces content (inc (:level block)) false))))]
+    (if (= format :org)
+      content
+      (gp-property/->new-properties content))))
+
+(defn- get-custom-id-or-new-id
+  [properties]
+  (or (when-let [custom-id (or (get-in properties [:properties :custom-id])
+                               (get-in properties [:properties :custom_id])
+                               (get-in properties [:properties :id]))]
+        (let [custom-id (and (string? custom-id) (string/trim custom-id))]
+          (when (and custom-id (gp-util/uuid-string? custom-id))
+            (uuid custom-id))))
+      (d/squuid)))
+
+(defn get-page-refs-from-properties
+  [properties db date-formatter]
+  (let [page-refs (mapcat (fn [v] (cond
+                                   (coll? v)
+                                   v
+
+                                   (text/page-ref? v)
+                                   [(text/page-ref-un-brackets! v)]
+
+                                   :else
+                                   nil)) (vals properties))
+        page-refs (remove string/blank? page-refs)]
+    (map (fn [page] (page-name->map page true db true date-formatter)) page-refs)))
+
+(defn- with-page-block-refs
+  [block with-id? supported-formats db date-formatter]
+  (some-> block
+          (with-page-refs with-id? supported-formats db date-formatter)
+          with-block-refs
+          block-tags->pages
+          (update :refs (fn [col] (remove nil? col)))))
+
+(defn- with-pre-block-if-exists
+  [blocks body pre-block-properties encoded-content {:keys [supported-formats db date-formatter]}]
+  (let [first-block (first blocks)
+        first-block-start-pos (get-in first-block [:block/meta :start_pos])
+
+        ;; Add pre-block
+        blocks (if (or (> first-block-start-pos 0)
+                       (empty? blocks))
+                 (cons
+                  (merge
+                   (let [content (utf8/substring encoded-content 0 first-block-start-pos)
+                         {:keys [properties properties-order]} pre-block-properties
+                         id (get-custom-id-or-new-id {:properties properties})
+                         property-refs (->> (get-page-refs-from-properties properties db date-formatter)
+                                            (map :block/original-name))
+                         block {:uuid id
+                                :content content
+                                :level 1
+                                :properties properties
+                                :properties-order properties-order
+                                :refs property-refs
+                                :pre-block? true
+                                :unordered true
+                                :body body}
+                         block (with-page-block-refs block false supported-formats db date-formatter)]
+                     (block-keywordize block))
+                   (select-keys first-block [:block/format :block/page]))
+                  blocks)
+                 blocks)]
+    (with-path-refs blocks)))
+
+(defn- construct-block
+  [block properties timestamps body encoded-content format pos-meta with-id? {:keys [block-pattern supported-formats db date-formatter]}]
+  (let [id (get-custom-id-or-new-id properties)
+        ref-pages-in-properties (->> (:page-refs properties)
+                                     (remove string/blank?))
+        block (second block)
+        unordered? (:unordered block)
+        markdown-heading? (and (:size block) (= :markdown format))
+        block (if markdown-heading?
+                (assoc block
+                       :type :heading
+                       :level (if unordered? (:level block) 1)
+                       :heading-level (or (:size block) 6))
+                block)
+        block (cond->
+                (assoc block
+                       :uuid id
+                       :refs ref-pages-in-properties
+                       :format format
+                       :meta pos-meta)
+                (seq (:properties properties))
+                (assoc :properties (:properties properties))
+
+                (seq (:properties-order properties))
+                (assoc :properties-order (:properties-order properties)))
+        block (if (get-in block [:properties :collapsed])
+                (assoc block :collapsed? true)
+                block)
+        block (assoc block
+                     :content (get-block-content encoded-content block format pos-meta block-pattern))
+        block (if (seq timestamps)
+                (merge block (timestamps->scheduled-and-deadline timestamps))
+                block)
+        block (assoc block :body body)
+        block (with-page-block-refs block with-id? supported-formats db date-formatter)
+        {:keys [created-at updated-at]} (:properties properties)
+        block (cond-> block
+                (and created-at (integer? created-at))
+                (assoc :block/created-at created-at)
+
+                (and updated-at (integer? updated-at))
+                (assoc :block/updated-at updated-at))]
+    (dissoc block :title :body :anchor)))
+
+(defn extract-blocks
+  "Extract headings from mldoc ast.
+  Args:
+    `blocks`: mldoc ast.
+    `content`: markdown or org-mode text.
+    `with-id?`: If `with-id?` equals to true, all the referenced pages will have new db ids.
+    `format`: content's format, it could be either :markdown or :org-mode.
+    `options`: Options supported are :user-config, :block-pattern :supported-formats,
+     :date-formatter and :db"
+  [blocks content with-id? format {:keys [user-config] :as options}]
+  {:pre [(seq blocks) (string? content) (boolean? with-id?) (contains? #{:markdown :org} format)]}
+  (try
+    (let [encoded-content (utf8/encode content)
+          [blocks body pre-block-properties]
+          (loop [headings []
+                 blocks (reverse blocks)
+                 timestamps {}
+                 properties {}
+                 body []]
+            (if (seq blocks)
+              (let [[block pos-meta] (first blocks)
+                    ;; fix start_pos
+                    pos-meta (assoc pos-meta :end_pos
+                                    (if (seq headings)
+                                      (get-in (last headings) [:meta :start_pos])
+                                      nil))]
+                (cond
+                  (paragraph-timestamp-block? block)
+                  (let [timestamps (extract-timestamps block)
+                        timestamps' (merge timestamps timestamps)]
+                    (recur headings (rest blocks) timestamps' properties body))
+
+                  (gp-property/properties-ast? block)
+                  (let [properties (extract-properties format (second block) user-config)]
+                    (recur headings (rest blocks) timestamps properties body))
+
+                  (heading-block? block)
+                  (let [block (construct-block block properties timestamps body encoded-content format pos-meta with-id? options)]
+                    (recur (conj headings block) (rest blocks) {} {} []))
+
+                  :else
+                  (recur headings (rest blocks) timestamps properties (conj body block))))
+              [(-> (reverse headings)
+                   sanity-blocks-data)
+               body
+               properties]))
+          result (with-pre-block-if-exists blocks body pre-block-properties encoded-content options)]
+      (map #(dissoc % :block/meta) result))
+    (catch :default e
+      (log/error :extract-blocks-failure e))))
+
+(defn with-parent-and-left
+  [page-id blocks]
+  (loop [blocks (map (fn [block] (assoc block :block/level-spaces (:block/level block))) blocks)
+         parents [{:page/id page-id     ; db id or a map {:block/name "xxx"}
+                   :block/level 0
+                   :block/level-spaces 0}]
+         result []]
+    (if (empty? blocks)
+      (map #(dissoc % :block/level-spaces) result)
+      (let [[block & others] blocks
+            level-spaces (:block/level-spaces block)
+            {:block/keys [uuid level parent] :as last-parent} (last parents)
+            parent-spaces (:block/level-spaces last-parent)
+            [blocks parents result]
+            (cond
+              (= level-spaces parent-spaces)        ; sibling
+              (let [block (assoc block
+                                 :block/parent parent
+                                 :block/left [:block/uuid uuid]
+                                 :block/level level)
+                    parents' (conj (vec (butlast parents)) block)
+                    result' (conj result block)]
+                [others parents' result'])
+
+              (> level-spaces parent-spaces)         ; child
+              (let [parent (if uuid [:block/uuid uuid] (:page/id last-parent))
+                    block (cond->
+                            (assoc block
+                                  :block/parent parent
+                                  :block/left parent)
+                            ;; fix block levels with wrong order
+                            ;; For example:
+                            ;;   - a
+                            ;; - b
+                            ;; What if the input indentation is two spaces instead of 4 spaces
+                            (>= (- level-spaces parent-spaces) 1)
+                            (assoc :block/level (inc level)))
+                    parents' (conj parents block)
+                    result' (conj result block)]
+                [others parents' result'])
+
+              (< level-spaces parent-spaces)
+              (cond
+                (some #(= (:block/level-spaces %) (:block/level-spaces block)) parents) ; outdent
+                (let [parents' (vec (filter (fn [p] (<= (:block/level-spaces p) level-spaces)) parents))
+                      left (last parents')
+                      blocks (cons (assoc (first blocks)
+                                          :block/level (dec level)
+                                          :block/left [:block/uuid (:block/uuid left)])
+                                   (rest blocks))]
+                  [blocks parents' result])
+
+                :else
+                (let [[f r] (split-with (fn [p] (<= (:block/level-spaces p) level-spaces)) parents)
+                      left (first r)
+                      parent-id (if-let [block-id (:block/uuid (last f))]
+                                  [:block/uuid block-id]
+                                  page-id)
+                      block (cond->
+                              (assoc block
+                                     :block/parent parent-id
+                                     :block/left [:block/uuid (:block/uuid left)]
+                                     :block/level (:block/level left)
+                                     :block/level-spaces (:block/level-spaces left)))
+
+                      parents' (->> (concat f [block]) vec)
+                      result' (conj result block)]
+                  [others parents' result'])))]
+        (recur blocks parents result)))))

src/main/logseq/graph_parser/config.cljs

@@ -14,3 +14,11 @@
 (defn draw?
   [path]
   (string/starts-with? path default-draw-directory))
+
+;; TODO: rename
+(defonce mldoc-support-formats
+  #{:org :markdown :md})
+
+(defn mldoc-support?
+  [format]
+  (contains? mldoc-support-formats (keyword format)))

src/main/logseq/graph_parser/date_time_util.cljs

@@ -0,0 +1,51 @@
+(ns ^:nbb-compatible logseq.graph-parser.date-time-util
+  "cljs-time util fns for graph-parser"
+  (:require [cljs-time.coerce :as tc]
+            [cljs-time.core :as t]
+            [cljs-time.format :as tf]
+            [clojure.string :as string]
+            [logseq.graph-parser.util :as gp-util]))
+
+(defn time-ms
+  "Copy of util/time-ms. Too basic to couple this to main app"
+  []
+  (tc/to-long (t/now)))
+
+;; (tf/parse (tf/formatter "dd.MM.yyyy") "2021Q4") => 20040120T000000
+(defn safe-journal-title-formatters
+  [date-formatter]
+  (->> [date-formatter "MMM do, yyyy" "yyyy-MM-dd" "yyyy_MM_dd"]
+       (remove string/blank?)
+       distinct))
+
+(defn journal-title->
+  [journal-title then-fn formatters]
+  (when-not (string/blank? journal-title)
+    (when-let [time (->> (map
+                          (fn [formatter]
+                            (try
+                              (tf/parse (tf/formatter formatter) (gp-util/capitalize-all journal-title))
+                              (catch js/Error _e
+                                nil)))
+                          formatters)
+                         (filter some?)
+                         first)]
+      (then-fn time))))
+
+(defn journal-title->int
+  [journal-title formatters]
+  (when journal-title
+    (let [journal-title (gp-util/capitalize-all journal-title)]
+      (journal-title-> journal-title
+                       #(gp-util/parse-int (tf/unparse (tf/formatter "yyyyMMdd") %))
+                       formatters))))
+
+(defn format
+  [date date-formatter]
+  (when date-formatter
+    (tf/unparse (tf/formatter date-formatter) date)))
+
+(defn int->journal-title
+  [day date-formatter]
+  (when day
+    (format (tf/parse (tf/formatter "yyyyMMdd") (str day)) date-formatter)))

src/main/frontend/handler/extract.cljs → src/main/logseq/graph_parser/extract.cljc

@@ -1,27 +1,27 @@
-(ns frontend.handler.extract
-  "Extract helper."
+(ns ^:nbb-compatible logseq.graph-parser.extract
+  ;; Disable clj linters since we don't support clj
+  #?(:clj {:clj-kondo/config {:linters {:unresolved-namespace {:level :off}
+                                        :unresolved-symbol {:level :off}}}})
   (:require [clojure.set :as set]
             [clojure.string :as string]
             [clojure.walk :as walk]
-            [frontend.config :as config]
-            [frontend.db :as db]
-            [frontend.format :as format]
-            [frontend.format.block :as block]
-            [frontend.format.mldoc :as mldoc]
-            [frontend.state :as state]
-            [frontend.text :as text]
-            [frontend.util :as util]
+            [datascript.core :as d]
+            [logseq.graph-parser.text :as text]
             [logseq.graph-parser.util :as gp-util]
-            [frontend.util.property :as property]
-            [lambdaisland.glogi :as log]))
+            [logseq.graph-parser.mldoc :as gp-mldoc]
+            [logseq.graph-parser.block :as gp-block]
+            [logseq.graph-parser.property :as gp-property]
+            [logseq.graph-parser.config :as gp-config]
+            #?(:org.babashka/nbb [logseq.graph-parser.log :as log]
+               :default [lambdaisland.glogi :as log])))
 
-(defn get-page-name
-  [file ast]
+(defn- get-page-name
+  [file ast page-name-order]
   ;; headline
   (let [ast (map first ast)]
     (if (string/includes? file "pages/contents.")
       "Contents"
-      (let [first-block (last (first (filter block/heading-block? ast)))
+      (let [first-block (last (first (filter gp-block/heading-block? ast)))
             property-name (when (and (contains? #{"Properties" "Property_Drawer"} (ffirst ast))
                                      (not (string/blank? (:title (last (first ast))))))
                             (:title (last (first ast))))
@@ -31,24 +31,24 @@
                                     title))
             file-name (when-let [file-name (last (string/split file #"/"))]
                         (let [result (first (gp-util/split-last "." file-name))]
-                          (if (config/mldoc-support? (string/lower-case (util/get-file-ext file)))
-                            (util/url-decode (string/replace result "." "/"))
+                          (if (gp-config/mldoc-support? (string/lower-case (gp-util/get-file-ext file)))
+                            (js/decodeURIComponent (string/replace result "." "/"))
                             result)))]
         (or property-name
-            (if (= (state/page-name-order) "heading")
+            (if (= page-name-order "heading")
               (or first-block-name file-name)
               (or file-name first-block-name)))))))
 
 
 ;; TODO: performance improvement
 (defn- extract-pages-and-blocks
-  #_:clj-kondo/ignore
-  [repo-url format ast properties file content]
+  [format ast properties file content {:keys [date-formatter page-name-order db] :as options}]
   (try
-    (let [page (get-page-name file ast)
-          [_original-page-name page-name _journal-day] (block/convert-page-if-journal page)
-          blocks (->> (block/extract-blocks ast content false format)
-                      (block/with-parent-and-left {:block/name page-name}))
+    #_:clj-kondo/ignore ;;clj-kondo bug
+    (let [page (get-page-name file ast page-name-order)
+          [_original-page-name page-name _journal-day] (gp-block/convert-page-if-journal page date-formatter)
+          blocks (->> (gp-block/extract-blocks ast content false format (dissoc options :page-name-order))
+                      (gp-block/with-parent-and-left {:block/name page-name}))
           ref-pages (atom #{})
           ref-tags (atom #{})
           blocks (map (fn [block]
@@ -64,94 +64,95 @@
                                      :block/page [:block/name page-name]
                                      :block/refs block-ref-pages
                                      :block/path-refs block-path-ref-pages))))
-                   blocks)
+                      blocks)
           page-entity (let [alias (:alias properties)
                             alias (if (string? alias) [alias] alias)
                             aliases (and alias
-                                         (seq (remove #(or (= page-name (util/page-name-sanity-lc %))
+                                         (seq (remove #(or (= page-name (gp-util/page-name-sanity-lc %))
                                                            (string/blank? %)) ;; disable blank alias
                                                       alias)))
                             aliases (->>
                                      (map
-                                       (fn [alias]
-                                         (let [page-name (util/page-name-sanity-lc alias)
-                                               aliases (distinct
-                                                        (conj
-                                                         (remove #{alias} aliases)
-                                                         page))
-                                               aliases (when (seq aliases)
-                                                         (map
-                                                           (fn [alias]
-                                                             {:block/name (util/page-name-sanity-lc alias)})
-                                                           aliases))]
-                                           (if (seq aliases)
-                                             {:block/name page-name
-                                              :block/alias aliases}
-                                             {:block/name page-name})))
-                                       aliases)
+                                      (fn [alias]
+                                        (let [page-name (gp-util/page-name-sanity-lc alias)
+                                              aliases (distinct
+                                                       (conj
+                                                        (remove #{alias} aliases)
+                                                        page))
+                                              aliases (when (seq aliases)
+                                                        (map
+                                                         (fn [alias]
+                                                           {:block/name (gp-util/page-name-sanity-lc alias)})
+                                                         aliases))]
+                                          (if (seq aliases)
+                                            {:block/name page-name
+                                             :block/alias aliases}
+                                            {:block/name page-name})))
+                                      aliases)
                                      (remove nil?))]
                         (cond->
-                          (gp-util/remove-nils
-                           (assoc
-                            (block/page-name->map page false)
-                            :block/file {:file/path (gp-util/path-normalize file)}))
-                          (seq properties)
-                          (assoc :block/properties properties)
+                         (gp-util/remove-nils
+                          (assoc
+                           (gp-block/page-name->map page false db true date-formatter)
+                           :block/file {:file/path (gp-util/path-normalize file)}))
+                         (seq properties)
+                         (assoc :block/properties properties)
 
-                          (seq aliases)
-                          (assoc :block/alias aliases)
+                         (seq aliases)
+                         (assoc :block/alias aliases)
 
-                          (:tags properties)
-                          (assoc :block/tags (let [tags (:tags properties)
-                                                   tags (if (string? tags) [tags] tags)
-                                                   tags (remove string/blank? tags)]
-                                               (swap! ref-tags set/union (set tags))
-                                               (map (fn [tag] {:block/name (util/page-name-sanity-lc tag)
-                                                               :block/original-name tag})
-                                                 tags)))))
+                         (:tags properties)
+                         (assoc :block/tags (let [tags (:tags properties)
+                                                  tags (if (string? tags) [tags] tags)
+                                                  tags (remove string/blank? tags)]
+                                              (swap! ref-tags set/union (set tags))
+                                              (map (fn [tag] {:block/name (gp-util/page-name-sanity-lc tag)
+                                                              :block/original-name tag})
+                                                   tags)))))
           namespace-pages (let [page (:block/original-name page-entity)]
                             (when (text/namespace-page? page)
-                              (->> (util/split-namespace-pages page)
+                              (->> (gp-util/split-namespace-pages page)
                                    (map (fn [page]
-                                          (-> (block/page-name->map page true)
+                                          (-> (gp-block/page-name->map page true db true date-formatter)
                                               (assoc :block/format format)))))))
           pages (->> (concat
                       [page-entity]
                       @ref-pages
                       (map
-                        (fn [page]
-                          {:block/original-name page
-                           :block/name (util/page-name-sanity-lc page)})
-                        @ref-tags)
+                       (fn [page]
+                         {:block/original-name page
+                          :block/name (gp-util/page-name-sanity-lc page)})
+                       @ref-tags)
                       namespace-pages)
                      ;; remove block references
                      (remove vector?)
                      (remove nil?))
-          pages (util/distinct-by :block/name pages)
+          pages (gp-util/distinct-by :block/name pages)
           pages (remove nil? pages)
-          pages (map (fn [page] (assoc page :block/uuid (db/new-block-id))) pages)
+          pages (map (fn [page] (assoc page :block/uuid (d/squuid))) pages)
           blocks (->> (remove nil? blocks)
                       (map (fn [b] (dissoc b :block/title :block/body :block/level :block/children :block/meta :block/anchor))))]
       [pages blocks])
-    (catch js/Error e
+    (catch :default e
       (log/error :exception e))))
 
 (defn extract-blocks-pages
-  [repo-url file content]
+  [file content {:keys [user-config] :as options}]
   (if (string/blank? content)
     []
-    (let [format (format/get-format file)
+    (let [format (gp-util/get-format file)
           _ (println "Parsing start: " file)
-          ast (mldoc/->edn content (mldoc/default-config format
+          ast (gp-mldoc/->edn content (gp-mldoc/default-config format
                                                          ;; {:parse_outline_only? true}
-                                                         ))]
+                                                         )
+                           user-config)]
       (println "Parsing finished : " file)
       (let [first-block (ffirst ast)
-            properties (let [properties (and (property/properties-ast? first-block)
+            properties (let [properties (and (gp-property/properties-ast? first-block)
                                              (->> (last first-block)
                                                   (map (fn [[x y]]
                                                          [x (if (string? y)
-                                                              (text/parse-property format x y)
+                                                              (text/parse-property format x y user-config)
                                                               y)]))
                                                   (into {})
                                                   (walk/keywordize-keys)))]
@@ -162,17 +163,16 @@
                                        (string/replace (or v "") "\\" "")))
                              properties)))]
         (extract-pages-and-blocks
-         repo-url
          format ast properties
-         file content)))))
+         file content options)))))
 
-(defn with-block-uuid
+(defn- with-block-uuid
   [pages]
-  (->> (util/distinct-by :block/name pages)
+  (->> (gp-util/distinct-by :block/name pages)
        (map (fn [page]
               (if (:block/uuid page)
                 page
-                (assoc page :block/uuid (db/new-block-id)))))))
+                (assoc page :block/uuid (d/squuid)))))))
 
 (defn with-ref-pages
   [pages blocks]

src/main/logseq/graph_parser/log.cljs

@@ -0,0 +1,6 @@
+(ns logseq.graph-parser.log
+  "Minimal logging ns that implements basic lambdaisland.glogi fns. May use
+  glogi later if this ns is used more")
+
+(defn error [& msgs]
+  (apply js/console.error (map clj->js msgs)))

src/main/logseq/graph_parser/mldoc.cljc

@@ -0,0 +1,215 @@
+(ns ^:nbb-compatible logseq.graph-parser.mldoc
+  ;; Disable clj linters since we don't support clj
+  #?(:clj {:clj-kondo/config {:linters {:unresolved-namespace {:level :off}
+                                        :unresolved-symbol {:level :off}}}})
+  (:require #?(:org.babashka/nbb ["mldoc$default" :refer [Mldoc]]
+               :default ["mldoc" :refer [Mldoc]])
+            #?(:org.babashka/nbb [logseq.graph-parser.log :as log]
+               :default [lambdaisland.glogi :as log])
+            [goog.object :as gobj]
+            [cljs-bean.core :as bean]
+            [logseq.graph-parser.utf8 :as utf8]
+            [clojure.string :as string]
+            [linked.core :as linked]
+            [logseq.graph-parser.util :as gp-util]
+            [logseq.graph-parser.config :as gp-config]))
+
+(defonce parseJson (gobj/get Mldoc "parseJson"))
+(defonce parseInlineJson (gobj/get Mldoc "parseInlineJson"))
+(defonce astExportMarkdown (gobj/get Mldoc "astExportMarkdown"))
+
+(def default-references
+  (js/JSON.stringify
+   (clj->js {:embed_blocks []
+             :embed_pages []})))
+
+(defn- convert-export-md-remove-options [opts]
+  (->> opts
+       (mapv (fn [opt]
+               (case opt
+                 :page-ref ["Page_ref"]
+                 :emphasis ["Emphasis"]
+                 [])))
+       (remove empty?)))
+
+(defn parse-json
+  [content config]
+  (parseJson content config))
+
+(defn inline-parse-json
+  [text config]
+  (parseInlineJson text config))
+
+(defn ast-export-markdown
+  [ast config references]
+  (astExportMarkdown ast
+                     config
+                     (or references default-references)))
+
+(defn default-config
+  ([format]
+   (default-config format {:export-heading-to-list? false}))
+  ([format {:keys [export-heading-to-list? export-keep-properties? export-md-indent-style export-md-remove-options parse_outline_only?]}]
+   (let [format (string/capitalize (name (or format :markdown)))]
+     (->> {:toc false
+           :parse_outline_only (or parse_outline_only? false)
+           :heading_number false
+           :keep_line_break true
+           :format format
+           :heading_to_list (or export-heading-to-list? false)
+           :exporting_keep_properties export-keep-properties?
+           :export_md_indent_style export-md-indent-style
+           :export_md_remove_options
+           (convert-export-md-remove-options export-md-remove-options)}
+          (filter #(not (nil? (second %))))
+          (into {})
+          (bean/->js)
+          js/JSON.stringify))))
+
+(defn remove-indentation-spaces
+  [s level remove-first-line?]
+  (let [lines (string/split-lines s)
+        [f & r] lines
+        body (map (fn [line]
+                    (if (string/blank? (gp-util/safe-subs line 0 level))
+                      (gp-util/safe-subs line level)
+                      line))
+               (if remove-first-line? lines r))
+        content (if remove-first-line? body (cons f body))]
+    (string/join "\n" content)))
+
+(defn- update-src-full-content
+  [ast content]
+  (let [content (utf8/encode content)]
+    (map (fn [[block pos-meta]]
+          (if (and (vector? block)
+                   (= "Src" (first block)))
+            (let [{:keys [start_pos end_pos]} pos-meta
+                  content (utf8/substring content start_pos end_pos)
+                  spaces (re-find #"^[\t ]+" (first (string/split-lines content)))
+                  content (if spaces (remove-indentation-spaces content (count spaces) true)
+                              content)
+                  block ["Src" (assoc (second block) :full_content content)]]
+              [block pos-meta])
+            [block pos-meta])) ast)))
+
+(defn- ->vec
+  [s]
+  (if (string? s) [s] s))
+
+(defn- ->vec-concat
+  [& coll]
+  (->> (map ->vec coll)
+       (remove nil?)
+       (apply concat)
+       (distinct)))
+
+(defn collect-page-properties
+  [ast parse-property config-state]
+  (if (seq ast)
+    (let [original-ast ast
+          ast (map first ast)           ; without position meta
+          directive? (fn [[item _]] (= "directive" (string/lower-case (first item))))
+          grouped-ast (group-by directive? original-ast)
+          directive-ast (take-while directive? original-ast)
+          [properties-ast other-ast] (if (= "Property_Drawer" (ffirst ast))
+                                       [(last (first ast))
+                                        (rest original-ast)]
+                                       [(->> (map first directive-ast)
+                                             (map rest))
+                                        (get grouped-ast false)])
+          properties (->>
+                      properties-ast
+                      (map (fn [[k v]]
+                             (let [k (keyword (string/lower-case k))
+                                   v (if (contains? #{:title :description :filters :macro} k)
+                                       v
+                                       (parse-property k v config-state))]
+                               [k v]))))
+          properties (into (linked/map) properties)
+          macro-properties (filter (fn [x] (= :macro (first x))) properties)
+          macros (if (seq macro-properties)
+                   (->>
+                    (map
+                     (fn [[_ v]]
+                       (let [[k v] (gp-util/split-first " " v)]
+                         (mapv
+                          string/trim
+                          [k v])))
+                     macro-properties)
+                    (into {}))
+                   {})
+          properties (->> (remove (fn [x] (= :macro (first x))) properties)
+                          (into (linked/map)))
+          properties (cond-> properties
+                             (seq macros)
+                             (assoc :macros macros))
+          alias (:alias properties)
+          alias (when alias
+                  (if (coll? alias)
+                    (remove string/blank? alias)
+                    [alias]))
+          filetags (when-let [org-file-tags (:filetags properties)]
+                     (->> (string/split org-file-tags ":")
+                          (remove string/blank?)))
+          tags (:tags properties)
+          tags (->> (->vec-concat tags filetags)
+                    (remove string/blank?))
+          properties (assoc properties :tags tags :alias alias)
+          properties (-> properties
+                         (update :filetags (constantly filetags)))
+          properties (into (linked/map)
+                           (remove (fn [[_k v]]
+                                     (or (nil? v) (and (coll? v) (empty? v))))
+                                   properties))]
+      (if (seq properties)
+        (cons [["Properties" properties] nil] other-ast)
+        original-ast))
+    ast))
+
+(def parse-property nil)
+
+(defn ->edn
+  [content config config-state]
+  (if (string? content)
+    (try
+      (if (string/blank? content)
+        []
+        (-> content
+            (parse-json config)
+            (gp-util/json->clj)
+            (update-src-full-content content)
+            (collect-page-properties parse-property config-state)))
+      (catch :default e
+        (log/error :unexpected-error e)
+        []))
+    (log/error :edn/wrong-content-type content)))
+
+(defn inline->edn
+  [text config]
+  (try
+    (if (string/blank? text)
+      {}
+      (-> text
+          (inline-parse-json config)
+          (gp-util/json->clj)))
+    (catch :default _e
+      [])))
+
+(defn link?
+  [format link]
+  (when (string? link)
+    (let [[type link] (first (inline->edn link (default-config format)))
+          [ref-type ref-value] (:url link)]
+      (and (= "Link" type)
+           (or
+            ;; 1. url
+            (not (contains? #{"Page_ref" "Block_ref"} ref-type))
+
+            (and (contains? #{"Page_ref"} ref-type)
+                 (or
+                  ;; 2. excalidraw link
+                  (gp-config/draw? ref-value)
+
+                  ;; 3. local asset link
+                  (boolean (gp-config/local-asset? ref-value)))))))))

src/main/logseq/graph_parser/property.cljs

@@ -0,0 +1,48 @@
+(ns logseq.graph-parser.property
+  "Property fns needed by graph-parser"
+  (:require [logseq.graph-parser.util :as gp-util]
+            [clojure.string :as string]
+            [goog.string :as gstring]))
+
+(defn properties-ast?
+  [block]
+  (and
+   (vector? block)
+   (contains? #{"Property_Drawer" "Properties"}
+              (first block))))
+
+(defonce properties-start ":PROPERTIES:")
+(defonce properties-end ":END:")
+(defonce properties-end-pattern
+  (re-pattern (gstring/format "%s[\t\r ]*\n|(%s\\s*$)" properties-end properties-end)))
+
+(defn contains-properties?
+  [content]
+  (when content
+    (and (string/includes? content properties-start)
+         (gp-util/safe-re-find properties-end-pattern content))))
+
+(defn ->new-properties
+  "New syntax: key:: value"
+  [content]
+  (if (contains-properties? content)
+    (let [lines (string/split-lines content)
+          start-idx (.indexOf lines properties-start)
+          end-idx (.indexOf lines properties-end)]
+      (if (and (>= start-idx 0) (> end-idx 0) (> end-idx start-idx))
+        (let [before (subvec lines 0 start-idx)
+              middle (->> (subvec lines (inc start-idx) end-idx)
+                          (map (fn [text]
+                                 (let [[k v] (gp-util/split-first ":" (subs text 1))]
+                                   (if (and k v)
+                                     (let [k (string/replace k "_" "-")
+                                           compare-k (keyword (string/lower-case k))
+                                           k (if (contains? #{:id :custom_id :custom-id} compare-k) "id" k)
+                                           k (if (contains? #{:last-modified-at} compare-k) "updated-at" k)]
+                                       (str k ":: " (string/trim v)))
+                                     text)))))
+              after (subvec lines (inc end-idx))
+              lines (concat before middle after)]
+          (string/join "\n" lines))
+        content))
+    content))

src/main/frontend/text.cljs → src/main/logseq/graph_parser/text.cljs

@@ -1,11 +1,10 @@
-(ns frontend.text
-  (:require [frontend.config :as config]
-            [frontend.util :as util]
+(ns ^:nbb-compatible logseq.graph-parser.text
+  (:require ["path" :as path]
+            [goog.string :as gstring]
             [clojure.string :as string]
-            [frontend.format.mldoc :as mldoc]
             [clojure.set :as set]
-            [logseq.graph-parser.util :as gp-util]
-            [frontend.state :as state]))
+            [logseq.graph-parser.mldoc :as gp-mldoc]
+            [logseq.graph-parser.util :as gp-util]))
 
 (def page-ref-re-0 #"\[\[(.*)\]\]")
 (def org-page-ref-re #"\[\[(file:.*)\]\[.+?\]\]")
@@ -14,7 +13,8 @@
 (defn get-file-basename
   [path]
   (when-not (string/blank? path)
-    (util/node-path.name path)))
+    ;; Same as util/node-path.name
+    (.-name (path/parse (string/replace path "+" "/")))))
 
 (defn get-page-name
   [s]
@@ -123,14 +123,15 @@
   (string/split s #"(\"[^\"]*\")"))
 
 (def markdown-link #"\[([^\[]+)\](\(.*\))")
+
 (defn split-page-refs-without-brackets
   ([s]
    (split-page-refs-without-brackets s {}))
   ([s {:keys [un-brackets?]
        :or {un-brackets? true}}]
    (cond
-     (and (string? s) (util/wrapped-by-quotes? s))
-     (util/unquote-string s)
+     (and (string? s) (gp-util/wrapped-by-quotes? s))
+     (gp-util/unquote-string s)
 
      (and (string? s) (re-find markdown-link s))
      s
@@ -142,11 +143,11 @@
      (let [result (->> (sep-by-quotes s)
                        (mapcat
                         (fn [s]
-                          (when-not (util/wrapped-by-quotes? (string/trim s))
+                          (when-not (gp-util/wrapped-by-quotes? (string/trim s))
                             (string/split s page-ref-re-2))))
                        (mapcat (fn [s]
                                  (cond
-                                   (util/wrapped-by-quotes? s)
+                                   (gp-util/wrapped-by-quotes? s)
                                    nil
 
                                    (string/includes? (string/trimr s) "]],")
@@ -166,7 +167,7 @@
                        (remove string/blank?)
                        (mapcat (fn [s]
                                  (cond
-                                   (util/wrapped-by-quotes? s)
+                                   (gp-util/wrapped-by-quotes? s)
                                    nil
 
                                    (page-ref? s)
@@ -187,18 +188,9 @@
      :else
      s)))
 
-(defn extract-level-spaces
-  [text format]
-  (if-not (string/blank? text)
-    (let [pattern (util/format
-                   "^[%s]+\\s?"
-                   (config/get-block-pattern format))]
-      (gp-util/safe-re-find (re-pattern pattern) text))
-    ""))
-
 (defn- remove-level-space-aux!
   [text pattern space? trim-left?]
-  (let [pattern (util/format
+  (let [pattern (gstring/format
                  (if space?
                    "^[%s]+\\s+"
                    "^[%s]+\\s?")
@@ -207,11 +199,11 @@
     (string/replace-first text (re-pattern pattern) "")))
 
 (defn remove-level-spaces
-  ([text format]
-   (remove-level-spaces text format false true))
-  ([text format space?]
-   (remove-level-spaces text format space? true))
-  ([text format space? trim-left?]
+  ([text format block-pattern]
+   (remove-level-spaces text format block-pattern false true))
+  ([text format block-pattern space?]
+   (remove-level-spaces text format block-pattern space? true))
+  ([text format block-pattern space? trim-left?]
    (when format
      (cond
        (string/blank? text)
@@ -222,12 +214,12 @@
        text
 
        :else
-       (remove-level-space-aux! text (config/get-block-pattern format) space? trim-left?)))))
+       (remove-level-space-aux! text block-pattern space? trim-left?)))))
 
 (defn build-data-value
   [col]
   (let [items (map (fn [item] (str "\"" item "\"")) col)]
-    (util/format "[%s]"
+    (gstring/format "[%s]"
                  (string/join ", " items))))
 
 (defn media-link?
@@ -240,7 +232,7 @@
        (string/includes? p "/")
        (not (string/starts-with? p "../"))
        (not (string/starts-with? p "./"))
-       (not (util/url? p))))
+       (not (gp-util/url? p))))
 
 (defn add-timestamp
   [content key value]
@@ -340,16 +332,16 @@
   (atom #{"background-color" "background_color"}))
 
 (defn parse-property
-  ([k v]
-   (parse-property :markdown k v))
-  ([format k v]
+  ([k v config-state]
+   (parse-property :markdown k v config-state))
+  ([format k v config-state]
    (let [k (name k)
          v (if (or (symbol? v) (keyword? v)) (name v) (str v))
          v (string/trim v)]
      (cond
        (contains? (set/union
                    #{"title" "filters"}
-                   (get (state/get-config) :ignored-page-references-keywords)) k)
+                   (get config-state :ignored-page-references-keywords)) k)
        v
 
        (= v "true")
@@ -358,15 +350,15 @@
        false
 
        (and (not= k "alias") (gp-util/safe-re-find #"^\d+$" v))
-       (util/safe-parse-int v)
+       (gp-util/safe-parse-int v)
 
-       (util/wrapped-by-quotes? v) ; wrapped in ""
+       (gp-util/wrapped-by-quotes? v) ; wrapped in ""
        v
 
        (contains? @non-parsing-properties (string/lower-case k))
        v
 
-       (mldoc/link? format v)
+       (gp-mldoc/link? format v)
        v
 
        :else

src/main/frontend/utf8.cljs → src/main/logseq/graph_parser/utf8.cljs

@@ -1,4 +1,4 @@
-(ns frontend.utf8
+(ns ^:nbb-compatible logseq.graph-parser.utf8
   (:require [goog.object :as gobj]))
 
 (defonce encoder

src/main/logseq/graph_parser/util.cljs

@@ -8,6 +8,7 @@
 (defonce exactly-uuid-pattern (re-pattern (str "(?i)^" uuid-pattern "$")))
 
 (defn safe-re-find
+  "Copy of frontend.util/safe-re-find. Too basic to couple to main app"
   [pattern s]
   (when-not (string? s)
     ;; TODO: sentry
@@ -16,6 +17,7 @@
     (re-find pattern s)))
 
 (defn uuid-string?
+  "Copy of frontend.util/uuid-string?. Too basic to couple to main app"
   [s]
   (safe-re-find exactly-uuid-pattern s))
 
@@ -57,8 +59,131 @@
    (let [c (count s)]
      (subs s (min c start) (min c end)))))
 
+(defn unquote-string
+  [v]
+  (string/trim (subs v 1 (dec (count v)))))
+
+(defn wrapped-by-quotes?
+  [v]
+  (and (string? v) (>= (count v) 2) (= "\"" (first v) (last v))))
+
+(defn parse-int
+  "Copy of frontend.util/parse-int. Too basic to couple to main app"
+  [x]
+  (if (string? x)
+    (js/parseInt x)
+    x))
+
+(defn safe-parse-int
+  "Copy of frontend.util/safe-parse-int. Too basic to couple to main app"
+  [x]
+  (let [result (parse-int x)]
+    (if (js/isNaN result)
+      nil
+      result)))
+
+(defn url?
+  [s]
+  (and (string? s)
+       (try
+         (js/URL. s)
+         true
+         (catch js/Error _e
+           false))))
+
 (defn json->clj
   [json-string]
   (-> json-string
       (js/JSON.parse)
       (js->clj :keywordize-keys true)))
+
+;; TODO: Use update-keys once its available in cljs and nbb
+(defn map-keys
+  "Maps function `f` over the keys of map `m` to produce a new map."
+  [f m]
+  (reduce-kv
+   (fn [m_ k v]
+     (assoc m_ (f k) v)) {} m))
+
+(defn zero-pad
+  "Copy of frontend.util/zero-pad. Too basic to couple to main app"
+  [n]
+  (if (< n 10)
+    (str "0" n)
+    (str n)))
+
+(defn get-file-ext
+  "Copy of frontend.util/get-file-ext. Too basic to couple to main app"
+  [file]
+  (and
+   (string? file)
+   (string/includes? file ".")
+   (some-> (last (string/split file #"\.")) string/lower-case)))
+
+(defn remove-boundary-slashes
+  [s]
+  (when (string? s)
+    (let [s (if (= \/ (first s))
+              (subs s 1)
+              s)]
+      (if (= \/ (last s))
+        (subs s 0 (dec (count s)))
+        s))))
+
+(defn split-namespace-pages
+  [title]
+  (let [parts (string/split title "/")]
+    (loop [others (rest parts)
+           result [(first parts)]]
+      (if (seq others)
+        (let [prev (last result)]
+          (recur (rest others)
+                 (conj result (str prev "/" (first others)))))
+        result))))
+
+(defn page-name-sanity
+  "Sanitize the page-name."
+  ([page-name]
+   (page-name-sanity page-name false))
+  ([page-name replace-slash?]
+   (let [page (some-> page-name
+                      (remove-boundary-slashes)
+                      (path-normalize))]
+     (if replace-slash?
+       (string/replace page #"/" "%2A")
+       page))))
+
+(defn page-name-sanity-lc
+  "Sanitize the query string for a page name (mandate for :block/name)"
+  [s]
+  (page-name-sanity (string/lower-case s)))
+
+(defn capitalize-all
+  [s]
+  (some->> (string/split s #" ")
+           (map string/capitalize)
+           (string/join " ")))
+
+(defn distinct-by
+  "Copy of frontend.util/distinct-by. Too basic to couple to main app"
+  [f col]
+  (reduce
+   (fn [acc x]
+     (if (some #(= (f x) (f %)) acc)
+       acc
+       (vec (conj acc x))))
+   []
+   col))
+
+(defn normalize-format
+  [format]
+  (case (keyword format)
+    :md :markdown
+    :asciidoc :adoc
+    ;; default
+    (keyword format)))
+
+(defn get-format
+  [file]
+  (when file
+    (normalize-format (keyword (string/lower-case (last (string/split file #"\.")))))))

src/test/frontend/db/conn.clj

@@ -1,3 +0,0 @@
-(ns frontend.db.conn)
-
-

src/test/frontend/format/mldoc_test.cljs

@@ -1,39 +0,0 @@
-(ns frontend.format.mldoc-test
-  (:require [frontend.format.mldoc :as mldoc]
-            [cljs.test :refer [testing deftest are]]))
-
-(deftest test-link
-  (testing "non-link"
-    (are [x y] (= (mldoc/link? :markdown x) y)
-      "google.com" false))
-
-  (testing "plain links"
-    (are [x y] (= (mldoc/link? :markdown x) y)
-      "http://www.google.com" true
-      "http://google.com" true))
-
-  (testing "org links with labels"
-    (are [x y] (= (mldoc/link? :org x) y)
-      "[[http://www.google.com][google]]" true
-      "[[http://google.com][google]]" true
-      "[[https://www.google.com][google]]" true
-      "[[https://google.com][google]]" true))
-
-  (testing "org links without labels"
-    (are [x y] (= (mldoc/link? :org x) y)
-      "[[http://www.google.com]]" true
-      "[[https://www.google.com]]" true
-      "[[draws/2022-03-06-15-00-28.excalidraw]]" true
-      "[[assets/2022-03-06-15-00-28.pdf]]" true))
-
-  (testing "markdown links"
-    (are [x y] (= (mldoc/link? :markdown x) y)
-      "[google](http://www.google.com)" true
-      "[google](https://www.google.com)" true
-      "[[draws/2022-03-06-15-00-28.excalidraw]]" true
-      "![a pdf](assets/2022-03-06-15-00-28.pdf)" true))
-
-  ;; https://github.com/logseq/logseq/issues/4308
-  (testing "parsing links should be finished"
-    (are [x y] (= (mldoc/link? :markdown x) y)
-      "[YouTube](https://www.youtube.com/watch?v=-8ym7pyUs9gL) - [Vimeo](https://vimeo.com/677920303) {{youtube https://www.youtube.com/watch?v=-8ym7pyUs9g}}" true)))

src/test/frontend/handler/block_test.cljs

@@ -1,9 +0,0 @@
-(ns frontend.handler.block-test)
-
-
-
-(comment
-  (defn clip-block [x]
-    (map #(select-keys % [:block/parent :block/left :block/pre-block? :block/uuid :block/level
-                          :db/id])
-      x)))

src/test/frontend/handler/repo_test.cljs

@@ -1,44 +1,14 @@
 (ns frontend.handler.repo-test
   (:require [cljs.test :refer [deftest use-fixtures is testing]]
-            [clojure.string :as string]
-            ["fs" :as fs]
-            ["child_process" :as child-process]
             [frontend.handler.repo :as repo-handler]
             [frontend.test.helper :as test-helper]
+            [frontend.test.docs-graph-helper :as docs-graph-helper]
             [datascript.core :as d]
             [frontend.db.conn :as conn]))
 
 (use-fixtures :each {:before test-helper/start-test-db!
                      :after test-helper/destroy-test-db!})
 
-(defn- slurp
-  "Like clojure.core/slurp"
-  [file]
-  (str (fs/readFileSync file)))
-
-(defn- sh
-  "Run shell cmd synchronously and print to inherited streams by default. Aims
-    to be similar to babashka.tasks/shell"
-  [cmd opts]
-  (child-process/spawnSync (first cmd)
-                           (clj->js (rest cmd))
-                           (clj->js (merge {:stdio "inherit"} opts))))
-
-(defn- build-graph-files
-  [dir]
-  (let [files (->> (str (.-stdout (sh ["git" "ls-files"]
-                                      {:cwd dir :stdio nil})))
-                   string/split-lines
-                   (filter #(re-find #"^(pages|journals)" %))
-                   (map #(str dir "/" %)))]
-    (mapv #(hash-map :file/path % :file/content (slurp %)) files)))
-
-(defn- clone-docs-repo-if-not-exists
-  [dir]
-  (when-not (.existsSync fs dir)
-    (sh ["git" "clone" "--depth" "1" "-b" "v0.6.7" "-c" "advice.detachedHead=false"
-         "https://github.com/logseq/docs" dir] {})))
-
 (defn- get-top-block-properties
   [db]
   (->> (d/q '[:find (pull ?b [*])
@@ -67,8 +37,8 @@
 ;; Integration test that test parsing a large graph like docs
 (deftest ^:integration parse-and-load-files-to-db
   (let [graph-dir "src/test/docs"
-        _ (clone-docs-repo-if-not-exists graph-dir)
-        files (build-graph-files graph-dir)
+        _ (docs-graph-helper/clone-docs-repo-if-not-exists graph-dir)
+        files (docs-graph-helper/build-graph-files graph-dir)
         _ (repo-handler/parse-files-and-load-to-db! test-helper/test-db files {:re-render? false})
         db (conn/get-db test-helper/test-db)]
 

src/test/frontend/modules/outliner/core_test.cljs

@@ -8,7 +8,7 @@
             [frontend.db :as db]
             [frontend.db.model :as db-model]
             [clojure.walk :as walk]
-            [frontend.format.block :as block]
+            [logseq.graph-parser.block :as gp-block]
             [datascript.core :as d]
             [frontend.test.helper :as helper]))
 
@@ -65,7 +65,7 @@
 
 (defn- build-blocks
   [tree]
-  (block/with-parent-and-left 1 (build-node-tree tree)))
+  (gp-block/with-parent-and-left 1 (build-node-tree tree)))
 
 (defn transact-tree!
   [tree]

src/test/frontend/parser.cljs

@@ -1,28 +0,0 @@
-(ns frontend.parser
-  (:require [cljs.test :refer [is deftest]]
-            [frontend.format.mldoc :as mldoc :refer [->edn]]))
-
-(def md-config (mldoc/default-config :markdown))
-
-(deftest src-test
-  (is (=
-       (first (->edn "```
-: hello
-```" md-config))
-       [["Src"
-         {:lines [": hello" "\n"],
-          :pos_meta {:start_pos 4, :end_pos 12},
-          :full_content "```\n: hello\n```"}]
-        {:start_pos 0, :end_pos 15}])))
-
-(deftest name-definition-test
-  (is (=
-       (first (->edn "term
-: definition" md-config))
-       [["List"
-         [{:content [["Paragraph" [["Plain" "definition"]]]],
-           :items [],
-           :name [["Plain" "term"]],
-           :indent 0,
-           :ordered false}]]
-        {:start_pos 0, :end_pos 17}])))

src/test/frontend/react.cljc

@@ -1,67 +0,0 @@
-(ns frontend.react
-  "To facilitate testing, imitate the behavior of react.
-  Note: don't run component parallel"
-  #?(:cljs (:require-macros [frontend.react])))
-
-#_{:component-key {:result nil
-                   :watches []
-                   :root-info nil}}
-(def react-components (atom {}))
-(def ^:dynamic *with-key* nil)
-(def ^:dynamic *comp-key* nil)
-(def ^:dynamic *root-info* nil)
-
-(defn react
-  [react-ref]
-  (let [comp-key *comp-key*
-        component (get @react-components comp-key)]
-    (cond
-      (some? component)
-      (do
-        (when-not ((:watches component) react-ref)
-          (let [new-component (update component :watches conj react-ref)]
-            (swap! react-components assoc comp-key new-component)
-            (add-watch react-ref comp-key
-                       (fn [_key _atom old-state new-state]
-                         (when-not (= old-state new-state)
-                           (let [root-info (get-in @react-components [comp-key :root-info])
-                                 {:keys [f comp-key]} root-info]
-                             (binding [*with-key* comp-key
-                                       *root-info* root-info]
-                               (let [component (get @react-components comp-key)]
-                                 (reset! (:result component) (f))))))))))
-        @react-ref)
-
-      ;; Sometime react is not used in component by accident, return the val.
-      :else
-      @react-ref)))
-
-(defn set-comp-and-calc-result
-  [f]
-  (let [{result :result :as component} (get @react-components *comp-key*)]
-    (if component
-      (do (reset! result (f)) result)
-      (let [result (atom nil)]
-        (binding [*root-info* (if *root-info* *root-info* {:f f :comp-key *comp-key*})]
-          (let [component {:result result
-                           :watches #{}
-                           :root-info *root-info*}]
-            (swap! react-components assoc *comp-key* component))
-          (reset! result (f))
-          result)))))
-
-#?(:clj (defmacro defc
-          [sym args & body]
-          `(defn ~sym ~args
-             (let [f# (fn []
-                        (binding [*comp-key* *with-key*
-                                  ;; inner component should specify own *with-key*
-                                  *with-key* nil]
-                          ~@body))]
-               (binding [*comp-key* *with-key*]
-                 (set-comp-and-calc-result f#))))))
-
-#?(:clj (defmacro with-key
-          [key & body]
-          `(binding [*with-key* ~key]
-             ~@body)))

src/test/frontend/react_test.cljs

@@ -1,64 +0,0 @@
-(ns frontend.react-test
-  ;; namespace local config for r/defc tests
-  {:clj-kondo/config {:linters {:inline-def {:level :off}}}}
-  (:require [frontend.react :as r]
-            [cljs.test :refer [deftest is use-fixtures]]
-            [frontend.test.fixtures :as fixtures]))
-
-(use-fixtures :each
-  fixtures/react-components)
-
-(deftest simple-react-test
-  (let [react-ref (atom 1)]
-
-    (r/defc simple-component
-      []
-      (+ 2 (r/react react-ref)))
-
-    (let [result (r/with-key 1 (simple-component))]
-
-      (is (= 3 @result))
-      (reset! react-ref 2)
-      (is (= 4 @result)))))
-
-(deftest nest-component-test
-  (let [a (atom 1)
-        b (atom 2)]
-
-    (r/defc inner
-      []
-
-      (let [r (r/react b)]
-        r))
-
-    (r/defc out
-      []
-      (let [out (r/react a)
-            inner-result (r/with-key "1" (inner))]
-        (+ out @inner-result)))
-
-    (let [out-result (r/with-key "2" (out))]
-      (is (= 3 @out-result))
-      (reset! b 4)
-      (is (= 5 @out-result)))))
-
-#_(deftest defc-params-test
-  (let [a (atom 1)
-        b (atom 2)]
-
-    (r/defc inner-1
-      [c]
-      (+ c (r/react b)))
-
-    (r/defc out-1
-      []
-      (let [out (r/react a)
-            inner-result (r/with-key 1 (inner-1 5))]
-        (+ out @inner-result)))
-
-    (let [out-result (r/with-key 2 (out-1))]
-      (is (= 8 @out-result))
-
-      (reset! b 4)
-
-      (is (= 10 @out-result)))))

src/test/frontend/test/docs_graph_helper.cljs

@@ -0,0 +1,34 @@
+(ns ^:nbb-compatible frontend.test.docs-graph-helper
+  "Helper fns for running tests against docs graph"
+  (:require ["fs" :as fs]
+            ["child_process" :as child-process]
+            [clojure.string :as string]))
+
+
+(defn slurp
+  "Like clojure.core/slurp"
+  [file]
+  (str (fs/readFileSync file)))
+
+(defn- sh
+  "Run shell cmd synchronously and print to inherited streams by default. Aims
+    to be similar to babashka.tasks/shell"
+  [cmd opts]
+  (child-process/spawnSync (first cmd)
+                           (clj->js (rest cmd))
+                           (clj->js (merge {:stdio "inherit"} opts))))
+
+(defn build-graph-files
+  [dir]
+  (let [files (->> (str (.-stdout (sh ["git" "ls-files"]
+                                      {:cwd dir :stdio nil})))
+                   string/split-lines
+                   (filter #(re-find #"^(pages|journals)" %))
+                   (map #(str dir "/" %)))]
+    (mapv #(hash-map :file/path % :file/content (slurp %)) files)))
+
+(defn clone-docs-repo-if-not-exists
+  [dir]
+  (when-not (.existsSync fs dir)
+    (sh ["git" "clone" "--depth" "1" "-b" "v0.6.7" "-c" "advice.detachedHead=false"
+         "https://github.com/logseq/docs" dir] {})))

src/test/frontend/util/property_test.cljs

@@ -146,29 +146,6 @@ SCHEDULED: <2021-10-25 Mon>\n:PROPERTIES:\n:a: b\n:END:\nworld\n" "c" "d")
     (property/insert-properties :markdown "" {:foo "\"bar, baz\""})
     "foo:: \"bar, baz\""))
 
-(deftest test->new-properties
-  (are [x y] (= (property/->new-properties x) y)
-    ":PROPERTIES:\n:foo: bar\n:END:"
-    "foo:: bar"
-
-    "hello\n:PROPERTIES:\n:foo: bar\n:END:"
-    "hello\nfoo:: bar"
-
-    "hello\n:PROPERTIES:\n:foo: bar\n:nice: bingo\n:END:"
-    "hello\nfoo:: bar\nnice:: bingo"
-
-    "hello\n:PROPERTIES:\n:foo: bar\n:nice: bingo\n:END:"
-    "hello\nfoo:: bar\nnice:: bingo"
-
-    "hello\n:PROPERTIES:\n:foo: bar\n:nice: bingo\n:END:\nnice"
-    "hello\nfoo:: bar\nnice:: bingo\nnice"
-
-    "hello\n:PROPERTIES:\n:foo: bar\n:nice:\n:END:\nnice"
-    "hello\nfoo:: bar\nnice:: \nnice"
-
-    "hello\n:PROPERTIES:\n:foo: bar\n:nice\n:END:\nnice"
-    "hello\nfoo:: bar\n:nice\nnice"))
-
 (deftest test-build-properties-str
   (are [x y] (= (property/build-properties-str :mardown x) y)
     {:title "a"}

src/test/frontend/format/block_test.cljs → src/test/logseq/graph_parser/block_test.cljs

@@ -1,9 +1,9 @@
-(ns frontend.format.block-test
-  (:require [frontend.format.block :as block]
+(ns logseq.graph-parser.block-test
+  (:require [logseq.graph-parser.block :as gp-block]
             [cljs.test :refer [deftest are]]))
 
 (deftest test-extract-properties
-  (are [x y] (= (:properties (block/extract-properties :markdown x)) y)
+  (are [x y] (= (:properties (gp-block/extract-properties :markdown x {})) y)
     [["year" "1000"]] {:year 1000}
     [["year" "\"1000\""]] {:year "\"1000\""}
     [["background-color" "#000000"]] {:background-color "#000000"}
@@ -23,7 +23,7 @@
     [["foo" "bar, [[baz, test]]"]] {:foo #{"bar" "baz, test"}}
     [["foo" "bar, [[baz, test, [[nested]]]]"]] {:foo #{"bar" "baz, test, [[nested]]"}})
 
-  (are [x y] (= (vec (:page-refs (block/extract-properties :markdown x))) y)
+  (are [x y] (= (vec (:page-refs (gp-block/extract-properties :markdown x {}))) y)
     [["year" "1000"]] []
     [["year" "\"1000\""]] []
     [["foo" "[[bar]] test"]] ["bar" "test"]

src/test/frontend/handler/extract_test.cljs → src/test/logseq/graph_parser/extract_test.cljs

@@ -1,17 +1,17 @@
-(ns frontend.handler.extract-test
+(ns logseq.graph-parser.extract-test
   (:require [cljs.test :refer [async deftest is]]
-            [frontend.handler.extract :as extract]
-            [frontend.util :as util]
+            [logseq.graph-parser.extract :as extract]
+            [clojure.pprint :as pprint]
             [promesa.core :as p]))
 
 (defn- extract
   [text]
-  (p/let [result (extract/extract-blocks-pages "repo" "a.md" text)
+  (p/let [result (extract/extract-blocks-pages "a.md" text {:block-pattern "-"})
           result (last result)
           lefts (map (juxt :block/parent :block/left) result)]
     (if (not= (count lefts) (count (distinct lefts)))
       (do
-        (util/pprint (map (fn [x] (select-keys x [:block/uuid :block/level :block/content :block/left])) result))
+        (pprint/pprint (map (fn [x] (select-keys x [:block/uuid :block/level :block/content :block/left])) result))
         (throw (js/Error. ":block/parent && :block/left conflicts")))
       (mapv :block/content result))))
 

src/test/logseq/graph_parser/mldoc_test.cljs

@@ -0,0 +1,130 @@
+(ns logseq.graph-parser.mldoc-test
+  (:require [logseq.graph-parser.mldoc :as gp-mldoc]
+            [clojure.string :as string]
+            [frontend.test.docs-graph-helper :as docs-graph-helper]
+            [cljs.test :refer [testing deftest are is]]))
+
+(deftest test-link
+  (testing "non-link"
+    (are [x y] (= (gp-mldoc/link? :markdown x) y)
+      "google.com" false))
+
+  (testing "plain links"
+    (are [x y] (= (gp-mldoc/link? :markdown x) y)
+      "http://www.google.com" true
+      "http://google.com" true))
+
+  (testing "org links with labels"
+    (are [x y] (= (gp-mldoc/link? :org x) y)
+      "[[http://www.google.com][google]]" true
+      "[[http://google.com][google]]" true
+      "[[https://www.google.com][google]]" true
+      "[[https://google.com][google]]" true))
+
+  (testing "org links without labels"
+    (are [x y] (= (gp-mldoc/link? :org x) y)
+      "[[http://www.google.com]]" true
+      "[[https://www.google.com]]" true
+      "[[draws/2022-03-06-15-00-28.excalidraw]]" true
+      "[[assets/2022-03-06-15-00-28.pdf]]" true))
+
+  (testing "markdown links"
+    (are [x y] (= (gp-mldoc/link? :markdown x) y)
+      "[google](http://www.google.com)" true
+      "[google](https://www.google.com)" true
+      "[[draws/2022-03-06-15-00-28.excalidraw]]" true
+      "![a pdf](assets/2022-03-06-15-00-28.pdf)" true))
+
+  ;; https://github.com/logseq/logseq/issues/4308
+  (testing "parsing links should be finished"
+    (are [x y] (= (gp-mldoc/link? :markdown x) y)
+      "[YouTube](https://www.youtube.com/watch?v=-8ym7pyUs9gL) - [Vimeo](https://vimeo.com/677920303) {{youtube https://www.youtube.com/watch?v=-8ym7pyUs9g}}" true)))
+
+(def md-config (gp-mldoc/default-config :markdown))
+
+(deftest src-test
+  (is (= [["Src"
+           {:lines [": hello" "\n"],
+            :pos_meta {:start_pos 4, :end_pos 12},
+            :full_content "```\n: hello\n```"}]
+          {:start_pos 0, :end_pos 15}]
+         (first (gp-mldoc/->edn "```
+: hello
+```" md-config {})))
+      "Basic src example")
+
+  (is (= [["Src"
+            {:lines ["  hello" "\n" "  world" "\n"],
+             :pos_meta {:start_pos 7, :end_pos 25},
+             :full_content "```\nhello\nworld\n```"}]
+           {:start_pos 1, :end_pos 29}]
+         (second (gp-mldoc/->edn "
+  ```
+  hello
+  world
+  ```
+" md-config {})))
+      "Src example with leading whitespace"))
+
+(deftest properties-test
+  (are [x y] (= [["Properties" y] nil]
+                (first (gp-mldoc/->edn x md-config {})))
+
+       ;; comma separates values
+       "property:: foo, bar"
+       {:property #{"foo" "bar"}}
+
+       ;; alias property
+       "alias:: foo,, bar"
+       {:alias ["foo" "bar"]}
+
+       ;; tags property
+       "tags:: foo,bar,foo"
+       {:tags ["foo" "bar"]}
+
+       ;; title property
+       "title:: comma, is ok"
+       {:title "comma, is ok"}))
+
+(deftest name-definition-test
+  (is (= [["List"
+           [{:content [["Paragraph" [["Plain" "definition"]]]],
+             :items [],
+             :name [["Plain" "term"]],
+             :indent 0,
+             :ordered false}]]
+          {:start_pos 0, :end_pos 17}]
+         (first (gp-mldoc/->edn "term
+: definition" md-config {})))))
+
+(deftest ^:integration test->edn
+  (let [graph-dir "src/test/docs"
+        _ (docs-graph-helper/clone-docs-repo-if-not-exists graph-dir)
+        files (docs-graph-helper/build-graph-files graph-dir)
+        asts-by-file (->> files
+                          (map (fn [{:file/keys [path content]}]
+                                 (let [format (if (string/ends-with? path ".org")
+                                                :org :markdown)]
+                                   [path
+                                    (gp-mldoc/->edn content
+                                                    (gp-mldoc/default-config format)
+                                                    {})])))
+                          (into {}))]
+    (is (= {"CommentBlock" 1,
+            "Custom" 41,
+            "Displayed_Math" 1,
+            "Drawer" 1,
+            "Example" 20,
+            "Footnote_Definition" 2,
+            "Heading" 3493,
+            "Hiccup" 15,
+            "List" 36,
+            "Paragraph" 411,
+            "Properties" 104,
+            "Property_Drawer" 188,
+            "Quote" 9,
+            "Raw_Html" 12,
+            "Src" 56,
+            "Table" 4}
+           (->> asts-by-file (mapcat val) (map ffirst) frequencies))
+        "AST node type counts")))