db_import.cljs 5.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138
  1. (ns db-import
  2. "Imports given file(s) to a db graph. This script is primarily for
  3. developing the import feature and for engineers who want to customize
  4. the import process"
  5. (:require [clojure.string :as string]
  6. [datascript.core :as d]
  7. ["path" :as node-path]
  8. ["os" :as os]
  9. ["fs" :as fs]
  10. ["fs/promises" :as fsp]
  11. [nbb.core :as nbb]
  12. [babashka.cli :as cli]
  13. [logseq.graph-parser.exporter :as gp-exporter]
  14. [logseq.common.graph :as common-graph]
  15. #_:clj-kondo/ignore
  16. [logseq.outliner.cli :as outliner-cli]
  17. [promesa.core :as p]))
  18. (defn- build-graph-files
  19. "Given a file graph directory, return all files including assets and adds relative paths
  20. on ::rpath since paths are absolute by default and exporter needs relative paths for
  21. some operations"
  22. [dir*]
  23. (let [dir (node-path/resolve dir*)]
  24. (->> (common-graph/get-files dir)
  25. (concat (when (fs/existsSync (node-path/join dir* "assets"))
  26. (common-graph/readdir (node-path/join dir* "assets"))))
  27. (mapv #(hash-map :path %
  28. ::rpath (node-path/relative dir* %))))))
  29. (defn- <read-file
  30. [file]
  31. (p/let [s (fsp/readFile (:path file))]
  32. (str s)))
  33. (defn- <copy-asset-file [file db-graph-dir file-graph-dir]
  34. (p/let [parent-dir (node-path/dirname
  35. (node-path/join db-graph-dir (node-path/relative file-graph-dir (:path file))))
  36. _ (fsp/mkdir parent-dir #js {:recursive true})]
  37. (fsp/copyFile (:path file) (node-path/join parent-dir (node-path/basename (:path file))))))
  38. (defn- notify-user [m]
  39. (println (:msg m))
  40. (println "Ex-data:" (pr-str (dissoc (:ex-data m) :error)))
  41. (println "Stacktrace:")
  42. (if-let [stack (some-> (get-in m [:ex-data :error]) ex-data :sci.impl/callstack deref)]
  43. (println (string/join
  44. "\n"
  45. (map
  46. #(str (:file %) (when (:line %) (str ":" (:line %)))
  47. " calls #'"
  48. (str (get-in % [:sci.impl/f-meta :ns]) "/" (get-in % [:sci.impl/f-meta :name])))
  49. stack)))
  50. (println (.-stack (get-in m [:ex-data :error])))))
  51. (def default-export-options
  52. {;; common options
  53. :rpath-key ::rpath
  54. :notify-user notify-user
  55. :<read-file <read-file
  56. ;; :set-ui-state prn
  57. ;; config file options
  58. ;; TODO: Add actual default
  59. :default-config {}})
  60. (defn- import-file-graph-to-db
  61. "Import a file graph dir just like UI does. However, unlike the UI the
  62. exporter receives file maps containing keys :path and ::rpath since :path
  63. are full paths"
  64. [file-graph-dir db-graph-dir conn options]
  65. (let [*files (build-graph-files file-graph-dir)
  66. config-file (first (filter #(string/ends-with? (:path %) "logseq/config.edn") *files))
  67. _ (assert config-file "No 'logseq/config.edn' found for file graph dir")
  68. options (merge options
  69. default-export-options
  70. ;; asset file options
  71. {:<copy-asset (fn copy-asset [file]
  72. (<copy-asset-file file db-graph-dir file-graph-dir))})]
  73. (gp-exporter/export-file-graph conn conn config-file *files options)))
  74. (defn- resolve-path
  75. "If relative path, resolve with $ORIGINAL_PWD"
  76. [path]
  77. (if (node-path/isAbsolute path)
  78. path
  79. (node-path/join (or js/process.env.ORIGINAL_PWD ".") path)))
  80. (defn- import-files-to-db
  81. "Import specific doc files for dev purposes"
  82. [file conn {:keys [files] :as options}]
  83. (let [doc-options (gp-exporter/build-doc-options conn {:macros {}} (merge options default-export-options))
  84. files' (mapv #(hash-map :path %)
  85. (into [file] (map resolve-path files)))]
  86. (gp-exporter/export-doc-files conn files' <read-file doc-options)))
  87. (def spec
  88. "Options spec"
  89. {:help {:alias :h
  90. :desc "Print help"}
  91. :verbose {:alias :v
  92. :desc "Verbose mode"}
  93. :tag-classes {:alias :t
  94. :coerce []
  95. :desc "List of tags to convert to classes"}
  96. :files {:alias :f
  97. :coerce []
  98. :desc "Additional files to import"}
  99. :property-classes {:alias :p
  100. :coerce []
  101. :desc "List of properties whose values convert to classes"}
  102. :property-parent-classes
  103. {:alias :P
  104. :coerce []
  105. :desc "List of properties whose values convert to a parent class"}})
  106. (defn -main [args]
  107. (let [[file-graph db-graph-dir] args
  108. options (cli/parse-opts args {:spec spec})
  109. _ (when (or (< (count args) 2) (:help options))
  110. (println (str "Usage: $0 FILE-GRAPH DB-GRAPH [OPTIONS]\nOptions:\n"
  111. (cli/format-opts {:spec spec})))
  112. (js/process.exit 1))
  113. [dir db-name] (if (string/includes? db-graph-dir "/")
  114. (let [graph-dir' (resolve-path db-graph-dir)]
  115. ((juxt node-path/dirname node-path/basename) graph-dir'))
  116. [(node-path/join (os/homedir) "logseq" "graphs") db-graph-dir])
  117. file-graph' (resolve-path file-graph)
  118. conn (outliner-cli/init-conn dir db-name)
  119. directory? (.isDirectory (fs/statSync file-graph'))]
  120. (p/do!
  121. (if directory?
  122. (import-file-graph-to-db file-graph' (node-path/join dir db-name) conn (merge options {:graph-name db-name}))
  123. (import-files-to-db file-graph' conn (merge options {:graph-name db-name})))
  124. (when (:verbose options) (println "Transacted" (count (d/datoms @conn :eavt)) "datoms"))
  125. (println "Created graph" (str db-name "!")))))
  126. (when (= nbb/*file* (:file (meta #'-main)))
  127. (-main *command-line-args*))