Merge pull request #1444 from logseq/feat/export-md

feat(export): export markdown

package.json

@@ -72,7 +72,7 @@
         "gulp-cached": "^1.1.1",
         "ignore": "^5.1.8",
         "jszip": "^3.5.0",
-        "mldoc": "^0.5.1",
+        "mldoc": "0.5.3",
         "mousetrap": "^1.6.5",
         "path": "^0.12.7",
         "react": "^17.0.1",

src/main/frontend/components/header.cljs

@@ -131,6 +131,12 @@
           :options {:on-click (fn []
                                 (export/export-repo-as-html! current-repo))}
           :icon nil})
+
+       (when current-repo
+         {:title (t :export-markdown)
+          :options {:on-click (fn []
+                                (export/export-repo-as-markdown! current-repo))}})
+
        (when current-repo
          {:title (t :import)
           :options {:href (rfe/href :import)}
@@ -215,4 +221,5 @@
                        :default-home default-home})
 
        [:a#download-as-html.hidden]
-       [:a#download-as-zip.hidden]])))
+       [:a#download-as-zip.hidden]
+       [:a#export-as-markdown.hidden]])))

src/main/frontend/dicts.cljs

@@ -311,6 +311,7 @@ title: How to take dummy notes?
         :close "Close"
         :re-index "Re-index"
         :export-json "Export as JSON"
+        :export-markdown "Export as Markdown"
         :unlink "unlink"
         :search (if config/publishing?
                   "Search"
@@ -1033,6 +1034,7 @@ title: How to take dummy notes?
            :cancel "取消"
            :re-index "重新建立索引"
            :export-json "以 JSON 格式导出"
+           :export-markdown "以 Markdown 格式导出"
            :unlink "解除绑定"
            :search (if config/publishing?
                      "搜索"
@@ -1297,6 +1299,7 @@ title: How to take dummy notes?
              :cancel "取消"
              :re-index "重新建立索引"
              :export-json "以 JSON 格式導出"
+             :export-markdown "以 Markdown 格式導出"
              :unlink "解除綁定"
              :search (if config/publishing?
                        "搜索"

src/main/frontend/format.cljs

@@ -33,8 +33,10 @@
 
 ;; html
 (defn get-default-config
-  [format]
-  (mldoc/default-config format))
+  ([format]
+   (mldoc/default-config format))
+  ([format heading-to-list?]
+   (mldoc/default-config format heading-to-list?)))
 
 (defn to-html
   ([content format]

src/main/frontend/format/adoc.cljs

@@ -26,4 +26,6 @@
   (lazyLoad [this ok-handler]
     (loader/load
      "https://cdnjs.cloudflare.com/ajax/libs/asciidoctor.js/1.5.9/asciidoctor.min.js"
-     ok-handler)))
+     ok-handler))
+  (exportMarkdown [this content config references]
+    (throw "not support")))

src/main/frontend/format/mldoc.cljs

@@ -14,16 +14,26 @@
 (defonce parseInlineJson (gobj/get Mldoc "parseInlineJson"))
 (defonce parseHtml (gobj/get Mldoc "parseHtml"))
 (defonce anchorLink (gobj/get Mldoc "anchorLink"))
+(defonce parseAndExportMarkdown (gobj/get Mldoc "parseAndExportMarkdown"))
 
 (defn default-config
-  [format]
-  (let [format (string/capitalize (name (or format :markdown)))]
-    (js/JSON.stringify
-     (bean/->js
-      (assoc {:toc false
-              :heading_number false
-              :keep_line_break true}
-             :format format)))))
+  ([format]
+   (default-config format false))
+  ([format export-heading-to-list?]
+   (let [format (string/capitalize (name (or format :markdown)))]
+     (js/JSON.stringify
+      (bean/->js
+       {:toc false
+        :heading_number false
+        :keep_line_break true
+        :format format
+        :heading_to_list export-heading-to-list?})))))
+
+(def default-references
+  (js/JSON.stringify
+   (clj->js {:embed_blocks []
+             :embed_pages []
+             :refer_blocks []})))
 
 (defn parse-json
   [content config]
@@ -33,6 +43,12 @@
   [text config]
   (parseInlineJson text (or config default-config)))
 
+(defn parse-export-markdown
+  [content config references]
+  (parseAndExportMarkdown content
+                          (or config default-config)
+                          (or references default-references)))
+
 ;; Org-roam
 (defn get-tags-from-definition
   [ast]
@@ -178,7 +194,10 @@
   (loaded? [this]
     true)
   (lazyLoad [this ok-handler]
-    true))
+    true)
+  (exportMarkdown [this content config references]
+    (parse-export-markdown content config references))
+  )
 
 (defn plain->text
   [plains]

src/main/frontend/format/protocol.cljs

@@ -4,4 +4,5 @@
   (toEdn [this content config])
   (toHtml [this content config])
   (loaded? [this])
-  (lazyLoad [this ok-handler]))
+  (lazyLoad [this ok-handler])
+  (exportMarkdown [this content config references]))

src/main/frontend/handler/export.cljs

@@ -1,6 +1,9 @@
 (ns frontend.handler.export
   (:require [frontend.state :as state]
             [frontend.db :as db]
+            [frontend.format.protocol :as fp]
+            [frontend.format :as f]
+            [datascript.core :as d]
             [frontend.util :as util]
             [cljs-bean.core :as bean]
             [clojure.string :as string]
@@ -102,3 +105,118 @@
           (.setAttribute anchor "href" (js/window.URL.createObjectURL zipfile))
           (.setAttribute anchor "download" (.-name zipfile))
           (.click anchor))))))
+
+
+
+(defn- get-file-contents-with-suffix
+  [repo]
+  (let [conn (db/get-conn repo)]
+    (->>
+     (filterv (fn [[path _]]
+                (or (string/ends-with? path ".md")))
+              (db/get-file-contents repo))
+     (mapv (fn [[path content]] {:path path :content content
+                                 :names (d/q '[:find [?n ?n2]
+                                               :in $ ?p
+                                               :where [?e :file/path ?p]
+                                               [?e2 :page/file ?e]
+                                               [?e2 :page/name ?n]
+                                               [?e2 :page/original-name ?n2]] conn path)
+                                 :format (f/get-format path)})))))
+
+(defn- get-embed-and-refs-blocks-pages-aux
+  [repo page-or-block is-block? exclude-blocks exclude-pages]
+  (let [[ref-blocks ref-pages]
+        (->> (if is-block?
+               [page-or-block]
+               (db/get-page-blocks
+                repo page-or-block {:use-cache? false
+                                    :pull-keys '[:block/ref-pages :block/ref-blocks]}))
+             (filterv #(or (:block/ref-blocks %) (:block/ref-pages %)))
+             (mapv (fn [b] [(:block/ref-blocks b), (:block/ref-pages b)]))
+             (apply mapv vector)
+             (mapv #(vec (distinct (flatten (remove nil? %))))))
+        ref-block-ids
+        (->> ref-blocks
+             (#(remove (fn [b] (contains? exclude-blocks (:db/id b))) %))
+             (mapv #(:db/id %)))
+        ref-page-ids
+        (->> ref-pages
+             (#(remove (fn [b] (contains? exclude-pages (:db/id b))) %))
+             (mapv #(:db/id %)))
+        ref-blocks
+        (->> ref-block-ids
+             (db/pull-many repo '[*])
+             (flatten))
+        ref-pages
+        (->> ref-page-ids
+             (db/pull-many repo '[*])
+             (flatten))
+        [next-ref-blocks1 next-ref-pages1]
+        (->> ref-blocks
+             (mapv #(get-embed-and-refs-blocks-pages-aux repo % true
+                                                         (set (concat ref-block-ids exclude-blocks)) exclude-pages))
+             (apply mapv vector))
+        [next-ref-blocks2 next-ref-pages2]
+        (->> ref-pages
+             (mapv #(get-embed-and-refs-blocks-pages-aux repo (:page/name %) false
+                                                         exclude-blocks (set (concat ref-page-ids exclude-pages))))
+             (apply mapv vector))]
+    [(->> (concat ref-block-ids next-ref-blocks1 next-ref-blocks2)
+          (flatten)
+          (distinct))
+     (->> (concat ref-page-ids next-ref-pages1 next-ref-pages2)
+          (flatten)
+          (distinct))]))
+
+
+(defn- get-embed-and-refs-blocks-pages
+  [repo page]
+  (let [[block-ids page-ids]
+        (get-embed-and-refs-blocks-pages-aux repo page false #{} #{})
+        blocks
+        (db/pull-many repo '[*] block-ids)
+        pages-name-and-content
+        (->> page-ids
+             (d/q '[:find ?n (pull ?p [:file/path])
+                    :in $ [?e ...]
+                    :where
+                    [?e :page/file ?p]
+                    [?e :page/name ?n]] (db/get-conn repo))
+             (mapv (fn [[page-name file-path]] [page-name (:file/path file-path)]))
+             (d/q '[:find ?n ?c
+                    :in $ [[?n ?p] ...]
+                    :where
+                    [?e :file/path ?p]
+                    [?e :file/content ?c]] @(db/get-files-conn repo)))
+        embed-blocks
+        (mapv (fn [b] [(str (:block/uuid b))
+                       [(apply str
+                               (mapv #(:block/content %)
+                                     (db/get-block-and-children repo (:block/uuid b))))
+                        (:block/title b)]])
+              blocks)]
+    {:embed_blocks embed-blocks
+     :embed_pages pages-name-and-content}))
+
+
+(defn export-repo-as-markdown!
+  [repo]
+  (when-let [repo (state/get-current-repo)]
+    (when-let [files (get-file-contents-with-suffix repo)]
+      (let [heading-to-list? (state/export-heading-to-list?)
+            files
+            (->> files
+                 (mapv (fn [{:keys [path content names format]}]
+                         (when (first names)
+                           [path (fp/exportMarkdown f/mldoc-record content
+                                                    (f/get-default-config format heading-to-list?)
+                                                    (js/JSON.stringify
+                                                     (clj->js (get-embed-and-refs-blocks-pages repo (first names)))))])))
+                 (remove nil?))
+            zip-file-name (str repo "_markdown_" (quot (util/time-ms) 1000))]
+        (p/let [zipfile (zip/make-zip zip-file-name files)]
+          (when-let [anchor (gdom/getElement "export-as-markdown")]
+            (.setAttribute anchor "href" (js/window.URL.createObjectURL zipfile))
+            (.setAttribute anchor "download" (.-name zipfile))
+            (.click anchor)))))))

src/main/frontend/state.cljs

@@ -198,6 +198,11 @@
   (not (false? (:feature/enable-journals?
                 (get (sub-config) repo)))))
 
+(defn export-heading-to-list?
+  []
+  (not (false? (:export/heading-to-list?
+                (get (sub-config) (get-current-repo))))))
+
 (defn enable-encryption?
   [repo]
   (:feature/enable-encryption?