|
@@ -5,74 +5,24 @@
|
|
|
[clojure.set :as s]
|
|
|
[clojure.string :as string]
|
|
|
[clojure.walk :as walk]
|
|
|
- [datascript.core :as d]
|
|
|
[frontend.config :as config]
|
|
|
[frontend.db :as db]
|
|
|
[frontend.extensions.zip :as zip]
|
|
|
[frontend.external.roam-export :as roam-export]
|
|
|
- [frontend.format.mldoc :as mldoc]
|
|
|
[frontend.handler.notification :as notification]
|
|
|
[frontend.mobile.util :as mobile-util]
|
|
|
- [frontend.modules.file.core :as outliner-file]
|
|
|
- [frontend.modules.outliner.tree :as outliner-tree]
|
|
|
[logseq.publishing.html :as publish-html]
|
|
|
[frontend.state :as state]
|
|
|
[frontend.util :as util]
|
|
|
- [frontend.handler.property.file :as property-file]
|
|
|
[goog.dom :as gdom]
|
|
|
[lambdaisland.glogi :as log]
|
|
|
- [logseq.graph-parser.property :as gp-property]
|
|
|
- [logseq.common.util.block-ref :as block-ref]
|
|
|
- [logseq.common.util.page-ref :as page-ref]
|
|
|
[promesa.core :as p]
|
|
|
- [frontend.persist-db :as persist-db])
|
|
|
+ [frontend.persist-db :as persist-db]
|
|
|
+ [cljs-bean.core :as bean]
|
|
|
+ [frontend.handler.export.common :as export-common-handler])
|
|
|
(:import
|
|
|
[goog.string StringBuffer]))
|
|
|
|
|
|
-(defn- get-page-content
|
|
|
- [repo page]
|
|
|
- (outliner-file/tree->file-content
|
|
|
- (outliner-tree/blocks->vec-tree
|
|
|
- (db/get-page-blocks-no-cache repo page) page) {:init-level 1}))
|
|
|
-
|
|
|
-(defn- get-file-content
|
|
|
- [repo file-path]
|
|
|
- (if-let [page-name
|
|
|
- (ffirst (d/q '[:find ?pn
|
|
|
- :in $ ?path
|
|
|
- :where
|
|
|
- [?p :block/file ?f]
|
|
|
- [?p :block/name ?pn]
|
|
|
- [?f :file/path ?path]]
|
|
|
- (db/get-db repo) file-path))]
|
|
|
- (get-page-content repo page-name)
|
|
|
- (ffirst
|
|
|
- (d/q '[:find ?content
|
|
|
- :in $ ?path
|
|
|
- :where
|
|
|
- [?f :file/path ?path]
|
|
|
- [?f :file/content ?content]]
|
|
|
- (db/get-db repo) file-path))))
|
|
|
-
|
|
|
-(defn- get-blocks-contents
|
|
|
- [repo root-block-uuid]
|
|
|
- (->
|
|
|
- (db/get-block-and-children repo root-block-uuid)
|
|
|
- (outliner-tree/blocks->vec-tree (str root-block-uuid))
|
|
|
- (outliner-file/tree->file-content {:init-level 1})))
|
|
|
-
|
|
|
-(defn download-file!
|
|
|
- [file-path]
|
|
|
- (when-let [repo (state/get-current-repo)]
|
|
|
- (when-let [content (get-file-content repo file-path)]
|
|
|
- (let [data (js/Blob. ["\ufeff" (array content)] ; prepend BOM
|
|
|
- (clj->js {:type "text/plain;charset=utf-8,"}))
|
|
|
- anchor (gdom/getElement "download")
|
|
|
- url (js/window.URL.createObjectURL data)]
|
|
|
- (.setAttribute anchor "href" url)
|
|
|
- (.setAttribute anchor "download" file-path)
|
|
|
- (.click anchor)))))
|
|
|
-
|
|
|
(defn download-repo-as-html!
|
|
|
"download public pages as html"
|
|
|
[repo]
|
|
@@ -98,28 +48,12 @@
|
|
|
(.setAttribute anchor "download" "index.html")
|
|
|
(.click anchor))))))
|
|
|
|
|
|
-(defn- get-file-contents
|
|
|
- ([repo]
|
|
|
- (get-file-contents repo {:init-level 1}))
|
|
|
- ([repo file-opts]
|
|
|
- (let [db (db/get-db repo)]
|
|
|
- (->> (d/q '[:find ?n ?fp
|
|
|
- :where
|
|
|
- [?e :block/file ?f]
|
|
|
- [?f :file/path ?fp]
|
|
|
- [?e :block/name ?n]] db)
|
|
|
- (mapv (fn [[page-name file-path]]
|
|
|
- [file-path
|
|
|
- (outliner-file/tree->file-content
|
|
|
- (outliner-tree/blocks->vec-tree
|
|
|
- (db/get-page-blocks-no-cache page-name) page-name)
|
|
|
- file-opts)]))))))
|
|
|
-
|
|
|
(defn export-repo-as-zip!
|
|
|
[repo]
|
|
|
- (let [files (get-file-contents repo)
|
|
|
- [owner repo-name] (util/get-git-owner-and-repo repo)
|
|
|
- repo-name (str owner "-" repo-name)]
|
|
|
+ (p/let [files (export-common-handler/<get-file-contents repo "md")
|
|
|
+ [owner repo-name] (util/get-git-owner-and-repo repo)
|
|
|
+ repo-name (str owner "-" repo-name)
|
|
|
+ files (map (fn [{:keys [path content]}] [path content]) files)]
|
|
|
(when (seq files)
|
|
|
(p/let [zipfile (zip/make-zip repo-name files repo)]
|
|
|
(when-let [anchor (gdom/getElement "download")]
|
|
@@ -127,137 +61,6 @@
|
|
|
(.setAttribute anchor "download" (.-name zipfile))
|
|
|
(.click anchor))))))
|
|
|
|
|
|
-(defn- get-embed-pages-from-ast [ast]
|
|
|
- (let [result (transient #{})]
|
|
|
- (doseq [item ast]
|
|
|
- (walk/prewalk (fn [i]
|
|
|
- (cond
|
|
|
- (and (vector? i)
|
|
|
- (= "Macro" (first i))
|
|
|
- (= "embed" (some-> (:name (second i))
|
|
|
- (string/lower-case)))
|
|
|
- (some-> (:arguments (second i))
|
|
|
- first
|
|
|
- page-ref/page-ref?))
|
|
|
- (let [arguments (:arguments (second i))
|
|
|
- page-ref (first arguments)
|
|
|
- page-name (-> page-ref
|
|
|
- (subs 2)
|
|
|
- (#(subs % 0 (- (count %) 2)))
|
|
|
- (string/lower-case))]
|
|
|
- (conj! result page-name)
|
|
|
- i)
|
|
|
- :else
|
|
|
- i))
|
|
|
- item))
|
|
|
- (persistent! result)))
|
|
|
-
|
|
|
-(defn- get-embed-blocks-from-ast [ast]
|
|
|
- (let [result (transient #{})]
|
|
|
- (doseq [item ast]
|
|
|
- (walk/prewalk (fn [i]
|
|
|
- (cond
|
|
|
- (and (vector? i)
|
|
|
- (= "Macro" (first i))
|
|
|
- (= "embed" (some-> (:name (second i))
|
|
|
- (string/lower-case)))
|
|
|
- (some-> (:arguments (second i))
|
|
|
- (first)
|
|
|
- block-ref/string-block-ref?))
|
|
|
- (let [arguments (:arguments (second i))
|
|
|
- block-uuid (block-ref/get-string-block-ref-id (first arguments))]
|
|
|
- (conj! result block-uuid)
|
|
|
- i)
|
|
|
- :else
|
|
|
- i)) item))
|
|
|
- (persistent! result)))
|
|
|
-
|
|
|
-(defn- get-block-refs-from-ast [ast]
|
|
|
- (let [result (transient #{})]
|
|
|
- (doseq [item ast]
|
|
|
- (walk/prewalk (fn [i]
|
|
|
- (cond
|
|
|
- (and (vector? i)
|
|
|
- (= "Block_ref" (first i))
|
|
|
- (some? (second i)))
|
|
|
- (let [block-uuid (second i)]
|
|
|
- (conj! result block-uuid)
|
|
|
- i)
|
|
|
- :else
|
|
|
- i)) item))
|
|
|
- (persistent! result)))
|
|
|
-
|
|
|
-(declare get-page-page&block-refs)
|
|
|
-(defn get-block-page&block-refs [repo block-uuid embed-pages embed-blocks block-refs]
|
|
|
- (let [block (db/entity [:block/uuid (uuid block-uuid)])
|
|
|
- block-content (get-blocks-contents repo (:block/uuid block))
|
|
|
- format (:block/format block)
|
|
|
- ast (mldoc/->edn block-content format)
|
|
|
- embed-pages-new (get-embed-pages-from-ast ast)
|
|
|
- embed-blocks-new (get-embed-blocks-from-ast ast)
|
|
|
- block-refs-new (get-block-refs-from-ast ast)
|
|
|
- embed-pages-diff (s/difference embed-pages-new embed-pages)
|
|
|
- embed-blocks-diff (s/difference embed-blocks-new embed-blocks)
|
|
|
- block-refs-diff (s/difference block-refs-new block-refs)
|
|
|
- embed-pages* (s/union embed-pages-new embed-pages)
|
|
|
- embed-blocks* (s/union embed-blocks-new embed-blocks)
|
|
|
- block-refs* (s/union block-refs-new block-refs)
|
|
|
- [embed-pages-1 embed-blocks-1 block-refs-1]
|
|
|
- (->>
|
|
|
- (mapv (fn [page-name]
|
|
|
- (let [{:keys [embed-pages embed-blocks block-refs]}
|
|
|
- (get-page-page&block-refs repo page-name embed-pages* embed-blocks* block-refs*)]
|
|
|
- [embed-pages embed-blocks block-refs])) embed-pages-diff)
|
|
|
- (apply mapv vector) ; [[1 2 3] [4 5 6] [7 8 9]] -> [[1 4 7] [2 5 8] [3 6 9]]
|
|
|
- (mapv #(apply s/union %)))
|
|
|
- [embed-pages-2 embed-blocks-2 block-refs-2]
|
|
|
- (->>
|
|
|
- (mapv (fn [block-uuid]
|
|
|
- (let [{:keys [embed-pages embed-blocks block-refs]}
|
|
|
- (get-block-page&block-refs repo block-uuid embed-pages* embed-blocks* block-refs*)]
|
|
|
- [embed-pages embed-blocks block-refs])) (s/union embed-blocks-diff block-refs-diff))
|
|
|
- (apply mapv vector)
|
|
|
- (mapv #(apply s/union %)))]
|
|
|
- {:embed-pages (s/union embed-pages-1 embed-pages-2 embed-pages*)
|
|
|
- :embed-blocks (s/union embed-blocks-1 embed-blocks-2 embed-blocks*)
|
|
|
- :block-refs (s/union block-refs-1 block-refs-2 block-refs*)}))
|
|
|
-
|
|
|
-
|
|
|
-(defn get-page-page&block-refs [repo page-name embed-pages embed-blocks block-refs]
|
|
|
- (let [page-name* (util/page-name-sanity-lc page-name)
|
|
|
- page-content (get-page-content repo page-name*)
|
|
|
- format (:block/format (db/entity [:block/name page-name*]))
|
|
|
- ast (mldoc/->edn page-content format)
|
|
|
- embed-pages-new (get-embed-pages-from-ast ast)
|
|
|
- embed-blocks-new (get-embed-blocks-from-ast ast)
|
|
|
- block-refs-new (get-block-refs-from-ast ast)
|
|
|
- embed-pages-diff (s/difference embed-pages-new embed-pages)
|
|
|
- embed-blocks-diff (s/difference embed-blocks-new embed-blocks)
|
|
|
- block-refs-diff (s/difference block-refs-new block-refs)
|
|
|
- embed-pages* (s/union embed-pages-new embed-pages)
|
|
|
- embed-blocks* (s/union embed-blocks-new embed-blocks)
|
|
|
- block-refs* (s/union block-refs-new block-refs)
|
|
|
- [embed-pages-1 embed-blocks-1 block-refs-1]
|
|
|
- (->>
|
|
|
- (mapv (fn [page-name]
|
|
|
- (let [{:keys [embed-pages embed-blocks block-refs]}
|
|
|
- (get-page-page&block-refs repo page-name embed-pages* embed-blocks* block-refs*)]
|
|
|
- [embed-pages embed-blocks block-refs])) embed-pages-diff)
|
|
|
- (apply mapv vector)
|
|
|
- (mapv #(apply s/union %)))
|
|
|
- [embed-pages-2 embed-blocks-2 block-refs-2]
|
|
|
- (->>
|
|
|
- (mapv (fn [block-uuid]
|
|
|
- (let [{:keys [embed-pages embed-blocks block-refs]}
|
|
|
- (get-block-page&block-refs repo block-uuid embed-pages* embed-blocks* block-refs*)]
|
|
|
- [embed-pages embed-blocks block-refs])) (s/union embed-blocks-diff block-refs-diff))
|
|
|
- (apply mapv vector)
|
|
|
- (mapv #(apply s/union %)))]
|
|
|
- {:embed-pages (s/union embed-pages-1 embed-pages-2 embed-pages*)
|
|
|
- :embed-blocks (s/union embed-blocks-1 embed-blocks-2 embed-blocks*)
|
|
|
- :block-refs (s/union block-refs-1 block-refs-2 block-refs*)}))
|
|
|
-
|
|
|
-
|
|
|
(defn- export-file-on-mobile [data path]
|
|
|
(p/catch
|
|
|
(.writeFile Filesystem (clj->js {:path path
|
|
@@ -294,56 +97,21 @@
|
|
|
x))
|
|
|
vec-tree))
|
|
|
|
|
|
-(defn- safe-keywordize
|
|
|
- [block]
|
|
|
- (update block :block/properties
|
|
|
- (fn [properties]
|
|
|
- (when (seq properties)
|
|
|
- (->> (filter (fn [[k _v]]
|
|
|
- (gp-property/valid-property-name? (str k))) properties)
|
|
|
- (into {}))))))
|
|
|
-
|
|
|
-(defn- blocks
|
|
|
- [repo db]
|
|
|
- {:version 1
|
|
|
- :blocks
|
|
|
- (->> (d/q '[:find (pull ?b [*])
|
|
|
- :in $
|
|
|
- :where
|
|
|
- [?b :block/original-name]
|
|
|
- [?b :block/name]] db)
|
|
|
-
|
|
|
- (map (fn [[{:block/keys [name] :as page}]]
|
|
|
- (let [whiteboard? (contains? (set (:block/type page)) "whiteboard")
|
|
|
- blocks (db/get-page-blocks-no-cache
|
|
|
- (state/get-current-repo)
|
|
|
- name
|
|
|
- {:transform? false})
|
|
|
- blocks' (if whiteboard?
|
|
|
- blocks
|
|
|
- (map (fn [b]
|
|
|
- (let [b' (if (seq (:block/properties b))
|
|
|
- (update b :block/content
|
|
|
- (fn [content]
|
|
|
- (property-file/remove-properties-when-file-based
|
|
|
- repo (:block/format b) content)))
|
|
|
- b)]
|
|
|
- (safe-keywordize b'))) blocks))
|
|
|
- children (if whiteboard?
|
|
|
- blocks'
|
|
|
- (outliner-tree/blocks->vec-tree blocks' name))
|
|
|
- page' (safe-keywordize page)]
|
|
|
- (assoc page' :block/children children))))
|
|
|
- (nested-select-keys
|
|
|
- [:block/id
|
|
|
- :block/type
|
|
|
- :block/page-name
|
|
|
- :block/properties
|
|
|
- :block/format
|
|
|
- :block/children
|
|
|
- :block/content
|
|
|
- :block/created-at
|
|
|
- :block/updated-at]))})
|
|
|
+(defn- <build-blocks
|
|
|
+ [repo]
|
|
|
+ (p/let [pages (export-common-handler/<get-all-pages repo)]
|
|
|
+ {:version 1
|
|
|
+ :blocks
|
|
|
+ (nested-select-keys [:block/id
|
|
|
+ :block/type
|
|
|
+ :block/page-name
|
|
|
+ :block/properties
|
|
|
+ :block/format
|
|
|
+ :block/children
|
|
|
+ :block/content
|
|
|
+ :block/created-at
|
|
|
+ :block/updated-at]
|
|
|
+ pages)}))
|
|
|
|
|
|
(defn- file-name [repo extension]
|
|
|
(-> (string/replace repo config/local-db-prefix "")
|
|
@@ -351,25 +119,27 @@
|
|
|
(str "_" (quot (util/time-ms) 1000))
|
|
|
(str "." (string/lower-case (name extension)))))
|
|
|
|
|
|
-(defn- export-repo-as-edn-str [repo]
|
|
|
- (when-let [db (db/get-db repo)]
|
|
|
+(defn- <export-repo-as-edn-str [repo]
|
|
|
+ (p/let [result (<build-blocks repo)]
|
|
|
+ (prn :debug :result result)
|
|
|
(let [sb (StringBuffer.)]
|
|
|
- (pprint/pprint (blocks repo db) (StringBufferWriter. sb))
|
|
|
+ (pprint/pprint result (StringBufferWriter. sb))
|
|
|
(str sb))))
|
|
|
|
|
|
-(defn export-repo-as-edn-v2!
|
|
|
+(defn export-repo-as-edn!
|
|
|
[repo]
|
|
|
- (when-let [edn-str (export-repo-as-edn-str repo)]
|
|
|
- (let [data-str (some->> edn-str
|
|
|
- js/encodeURIComponent
|
|
|
- (str "data:text/edn;charset=utf-8,"))
|
|
|
- filename (file-name repo :edn)]
|
|
|
- (if (mobile-util/native-platform?)
|
|
|
- (export-file-on-mobile edn-str filename)
|
|
|
- (when-let [anchor (gdom/getElement "download-as-edn-v2")]
|
|
|
- (.setAttribute anchor "href" data-str)
|
|
|
- (.setAttribute anchor "download" filename)
|
|
|
- (.click anchor))))))
|
|
|
+ (p/let [edn-str (<export-repo-as-edn-str repo)]
|
|
|
+ (when edn-str
|
|
|
+ (let [data-str (some->> edn-str
|
|
|
+ js/encodeURIComponent
|
|
|
+ (str "data:text/edn;charset=utf-8,"))
|
|
|
+ filename (file-name repo :edn)]
|
|
|
+ (if (mobile-util/native-platform?)
|
|
|
+ (export-file-on-mobile edn-str filename)
|
|
|
+ (when-let [anchor (gdom/getElement "download-as-edn-v2")]
|
|
|
+ (.setAttribute anchor "href" data-str)
|
|
|
+ (.setAttribute anchor "download" filename)
|
|
|
+ (.click anchor)))))))
|
|
|
|
|
|
(defn- nested-update-id
|
|
|
[vec-tree]
|
|
@@ -380,23 +150,22 @@
|
|
|
x))
|
|
|
vec-tree))
|
|
|
|
|
|
-(defn export-repo-as-json-v2!
|
|
|
+(defn export-repo-as-json!
|
|
|
[repo]
|
|
|
- (when-let [db (db/get-db repo)]
|
|
|
- (let [json-str
|
|
|
- (-> (blocks repo db)
|
|
|
- nested-update-id
|
|
|
- clj->js
|
|
|
- js/JSON.stringify)
|
|
|
+ (p/let [result (<build-blocks repo)
|
|
|
+ json-str (-> result
|
|
|
+ nested-update-id
|
|
|
+ clj->js
|
|
|
+ js/JSON.stringify)
|
|
|
filename (file-name repo :json)
|
|
|
data-str (str "data:text/json;charset=utf-8,"
|
|
|
(js/encodeURIComponent json-str))]
|
|
|
- (if (mobile-util/native-platform?)
|
|
|
- (export-file-on-mobile json-str filename)
|
|
|
- (when-let [anchor (gdom/getElement "download-as-json-v2")]
|
|
|
- (.setAttribute anchor "href" data-str)
|
|
|
- (.setAttribute anchor "download" filename)
|
|
|
- (.click anchor))))))
|
|
|
+ (if (mobile-util/native-platform?)
|
|
|
+ (export-file-on-mobile json-str filename)
|
|
|
+ (when-let [anchor (gdom/getElement "download-as-json-v2")]
|
|
|
+ (.setAttribute anchor "href" data-str)
|
|
|
+ (.setAttribute anchor "download" filename)
|
|
|
+ (.click anchor)))))
|
|
|
|
|
|
(defn export-repo-as-sqlite-db!
|
|
|
[repo]
|
|
@@ -415,39 +184,25 @@
|
|
|
|
|
|
;; https://roamresearch.com/#/app/help/page/Nxz8u0vXU
|
|
|
;; export to roam json according to above spec
|
|
|
-(defn- roam-json [db]
|
|
|
- (->> (d/q '[:find (pull ?b [*])
|
|
|
- :in $
|
|
|
- :where
|
|
|
- [?b :block/file]
|
|
|
- [?b :block/original-name]
|
|
|
- [?b :block/name]] db)
|
|
|
-
|
|
|
- (map (fn [[{:block/keys [name] :as page}]]
|
|
|
- (assoc page
|
|
|
- :block/children
|
|
|
- (outliner-tree/blocks->vec-tree
|
|
|
- (db/get-page-blocks-no-cache
|
|
|
- (state/get-current-repo)
|
|
|
- name
|
|
|
- {:transform? false}) name))))
|
|
|
-
|
|
|
- (roam-export/traverse
|
|
|
- [:page/title
|
|
|
- :block/string
|
|
|
- :block/uid
|
|
|
- :block/children])))
|
|
|
+(defn- <roam-data [repo]
|
|
|
+ (p/let [pages (export-common-handler/<get-all-pages repo)]
|
|
|
+ (let [non-empty-pages (remove #(empty? (:block/children %)) pages)]
|
|
|
+ (roam-export/traverse
|
|
|
+ [:page/title
|
|
|
+ :block/string
|
|
|
+ :block/uid
|
|
|
+ :block/children]
|
|
|
+ non-empty-pages))))
|
|
|
|
|
|
(defn export-repo-as-roam-json!
|
|
|
[repo]
|
|
|
- (when-let [db (db/get-db repo)]
|
|
|
- (let [json-str
|
|
|
- (-> (roam-json db)
|
|
|
- clj->js
|
|
|
- js/JSON.stringify)
|
|
|
+ (p/let [data (<roam-data repo)
|
|
|
+ json-str (-> data
|
|
|
+ bean/->js
|
|
|
+ js/JSON.stringify)
|
|
|
data-str (str "data:text/json;charset=utf-8,"
|
|
|
(js/encodeURIComponent json-str))]
|
|
|
- (when-let [anchor (gdom/getElement "download-as-roam-json")]
|
|
|
- (.setAttribute anchor "href" data-str)
|
|
|
- (.setAttribute anchor "download" (file-name (str repo "_roam") :json))
|
|
|
- (.click anchor)))))
|
|
|
+ (when-let [anchor (gdom/getElement "download-as-roam-json")]
|
|
|
+ (.setAttribute anchor "href" data-str)
|
|
|
+ (.setAttribute anchor "download" (file-name (str repo "_roam") :json))
|
|
|
+ (.click anchor))))
|