enhance: add support for importing page and block tags

Allows users to specify which tags are tag classes and
the rest are just normal pages. Fixes LOG-2984

deps/db/src/logseq/db/frontend/content.cljs

@@ -56,3 +56,19 @@
     content
     tags)
    (string/trim)))
+
+(defn replace-tags-with-page-refs
+  "Replace tags in content with page-ref ids"
+  [content tags]
+  (reduce
+   (fn [content tag]
+     (string/replace content
+                     (str "#" (:block/original-name tag))
+                     (str page-ref/left-brackets
+                                ;; TODO: Use uuid when it becomes available
+                                ;; page-ref-special-chars
+                                ;; (:block/uuid tag)
+                          (:block/original-name tag)
+                          page-ref/right-brackets)))
+   content
+   tags))

deps/db/src/logseq/db/frontend/property.cljs

@@ -24,6 +24,10 @@
           :schema {:type :page
                    :cardinality :many
                    :classes #{:logseq.class}}}
+   :pagetags {:original-name "pageTags"
+              :visible true
+              :schema {:type :page
+                       :cardinality :many}}
    :background-color {:schema {:type :default :hide? true}
                       :visible true}
    :background-image {:schema {:type :default :hide? true}

deps/graph-parser/src/logseq/graph_parser.cljs

@@ -135,23 +135,60 @@ Options available:
                 (assoc :block/created-at updated-at))]
     block))
 
-(defn- update-block-with-invalid-tags
-  [block]
+(defn- update-page-tags
+  [block tag-classes names-uuids page-tags-uuid]
+  (if (seq (:block/tags block))
+    (let [page-tags (->> (:block/tags block)
+                         (remove #(contains? tag-classes (:block/name %)))
+                         (map #(or (get names-uuids (:block/name %))
+                                   (throw (ex-info (str "No uuid found for tag " (pr-str (:block/name %)))
+                                                   {:tag %}))))
+                         set)]
+      (cond-> block
+        true
+        (update :block/tags
+                (fn [tags]
+                  (keep #(when (contains? tag-classes (:block/name %))
+                           (-> %
+                               add-missing-timestamps
+                               ;; don't use build-new-class b/c of timestamps
+                               (merge {:block/journal? false
+                                       :block/format :markdown
+                                       :block/type "class"
+                                       :block/uuid (d/squuid)})))
+                        tags)))
+        (seq page-tags)
+        (assoc :block/properties {page-tags-uuid page-tags})))
+    block))
+
+(defn- update-block-tags
+  [block tag-classes]
   (if (seq (:block/tags block))
-    (update block :block/tags
+    (-> block
+        (update :block/content
+                db-content/content-without-tags
+                (->> (:block/tags block)
+                     (filter #(tag-classes (:block/name %)))
+                     (map :block/original-name)))
+        (update :block/content
+                db-content/replace-tags-with-page-refs
+                (remove #(tag-classes (:block/name %))
+                        (:block/tags block)))
+        (update :block/tags
             (fn [tags]
-              (mapv #(-> %
-                         add-missing-timestamps
-                         ;; don't use build-new-class b/c of timestamps
-                         (merge {:block/journal? false
-                                 :block/format :markdown
-                                 :block/type "class"
-                                 :block/uuid (d/squuid)}))
-                    tags)))
+              (keep #(when (contains? tag-classes (:block/name %))
+                       (-> %
+                           add-missing-timestamps
+                           ;; don't use build-new-class b/c of timestamps
+                           (merge {:block/journal? false
+                                   :block/format :markdown
+                                   :block/type "class"
+                                   :block/uuid (d/squuid)})))
+                    tags))))
     block))
 
 (defn- update-imported-block
-  [conn block]
+  [conn block tag-classes]
   (prn ::block block)
   (let [remove-keys (fn [m pred] (into {} (remove (comp pred key) m)))]
     (-> block
@@ -180,14 +217,7 @@ Options available:
                                                new-key
                                                k)))
                               (remove-keys keyword?)))))))
-        update-block-with-invalid-tags
-        ((fn [block']
-           (if (seq (:block/tags block'))
-             (update block :block/content
-                     db-content/content-without-tags
-                     (map :block/original-name (:block/tags block')))
-             block)))
-
+        (update-block-tags tag-classes)
         ((fn [block']
            (if (seq (:block/refs block'))
              (update block' :block/refs
@@ -209,9 +239,10 @@ Options available:
 
 (defn import-file-to-db-graph
   "Parse file and save parsed data to the given db graph."
-  [conn file content {:keys [extract-options]
+  [conn file content {:keys [extract-options user-options page-tags-uuid]
                       :as options}]
   (let [format (common-util/get-format file)
+        tag-classes (set (map string/lower-case (:tag-classes user-options)))
         {:keys [tx ast]}
         (let [extract-options' (merge {:block-pattern (common-config/get-block-pattern format)
                                        :date-formatter "MMM do, yyyy"
@@ -242,6 +273,8 @@ Options available:
                                   (seq))
                  ;; To prevent "unique constraint" on datascript
               block-ids (set/union (set block-ids) (set block-refs-ids))
+              pages* (extract/with-ref-pages pages blocks)
+              names-uuids (into {} (map (juxt :block/name :block/uuid) pages*))
               pages (map #(-> (merge {:block/journal? false} %)
                               ;; Fix pages missing :block/original-name. Shouldn't happen
                               ((fn [m]
@@ -253,10 +286,10 @@ Options available:
                               (assoc :block/format :markdown)
                               (dissoc :block/properties-text-values :block/properties-order :block/invalid-properties
                                       :block/whiteboard?)
-                              update-block-with-invalid-tags
                               ;; FIXME: Remove when properties are supported
-                              (assoc :block/properties {}))
-                         (extract/with-ref-pages pages blocks))
+                              (assoc :block/properties {})
+                              (update-page-tags tag-classes names-uuids page-tags-uuid))
+                         pages*)
 
               ;; post-handling
               whiteboard-pages (->> pages
@@ -267,7 +300,7 @@ Options available:
                                                       :block/format :markdown
                                                       ;; fixme: missing properties
                                                       :block/properties {(get-pid @conn :ls-type) :whiteboard-page})))))
-              blocks (map #(update-imported-block conn %) blocks)
+              blocks (map #(update-imported-block conn % tag-classes) blocks)
               pages-index (map #(select-keys % [:block/name]) pages)]
 
           {:tx (concat refs whiteboard-pages pages-index pages block-ids blocks)

src/main/frontend/components/imports.cljs

@@ -35,7 +35,8 @@
             [promesa.core :as p]
             [rum.core :as rum]
             [logseq.common.config :as common-config]
-            [lambdaisland.glogi :as log]))
+            [lambdaisland.glogi :as log]
+            [frontend.handler.db-based.property.util :as db-pu]))
 
 ;; Can't name this component as `frontend.components.import` since shadow-cljs
 ;; will complain about it.
@@ -165,10 +166,10 @@
       (ui/button "Submit"
                  {:on-click on-submit})]]))
 
-
 (defn- import-from-doc-files!
-  [db-conn repo config doc-files]
-  (let [imported-chan (async/promise-chan)]
+  [db-conn repo config doc-files user-options]
+  (let [imported-chan (async/promise-chan)
+        page-tags-uuid (db-pu/get-built-in-property-uuid repo :pagetags)]
     (try
       (let [docs-chan (async/to-chan! (medley/indexed doc-files))]
         (state/set-state! [:graph/importing-state :total] (count doc-files))
@@ -189,7 +190,13 @@
                                   (p/then (fn [m]
                                             ;; Write to frontend first as writing to worker first is poor ux with slow streaming changes
                                             (let [{:keys [tx-report]}
-                                                  (graph-parser/import-file-to-db-graph db-conn (:file/path m) (:file/content m) {:extract-options extract-options})]
+                                                  (graph-parser/import-file-to-db-graph
+                                                   db-conn
+                                                   (:file/path m)
+                                                   (:file/content m)
+                                                   {:extract-options extract-options
+                                                    :user-options user-options
+                                                    :page-tags-uuid page-tags-uuid})]
                                               (db-browser/transact! @db-browser/*worker repo (:tx-data tx-report) (:tx-meta tx-report)))
                                             m)))))
               (recur))
@@ -286,11 +293,11 @@
                 (outliner-core/insert-blocks! repo db-conn (build-hidden-favorites-page-blocks page-block-uuid-coll)
                                               page-entity {}))))))))))
 
-
-(rum/defc confirm-graph-name-dialog
+(rum/defc import-file-graph-dialog
   [initial-name on-graph-name-confirmed]
-  (let [[input set-input!] (rum/use-state initial-name)
-        on-submit #(do (on-graph-name-confirmed input)
+  (let [[graph-input set-graph-input!] (rum/use-state initial-name)
+        [tags-input set-tags-input!] (rum/use-state "")
+        on-submit #(do (on-graph-name-confirmed {:graph-name graph-input :tags tags-input})
                        (state/close-modal!))]
     [:div.container
      [:div.sm:flex.sm:items-start
@@ -300,19 +307,32 @@
 
      [:input.form-input.block.w-full.sm:text-sm.sm:leading-5.my-2.mb-4
       {:auto-focus true
-       :default-value input
+       :default-value graph-input
        :on-change (fn [e]
-                    (set-input! (util/evalue e)))
+                    (set-graph-input! (util/evalue e)))
+       :on-key-down (fn [e]
+                      (when (= "Enter" (util/ekey e))
+                        (on-submit)))}]
+     [:div.sm:flex.sm:items-start
+      [:div.mt-3.text-center.sm:mt-0.sm:text-left
+       [:h3#modal-headline.leading-6.font-medium
+        "(Optional) Tags to import as tag classes:"]
+       [:span.text-xs
+        "Tags are case insensitive and separated by commas"]]]
+     [:input.form-input.block.w-full.sm:text-sm.sm:leading-5.my-2.mb-4
+      {:default-value tags-input
+       :on-change (fn [e]
+                    (set-tags-input! (util/evalue e)))
        :on-key-down (fn [e]
                       (when (= "Enter" (util/ekey e))
                         (on-submit)))}]
 
      [:div.mt-5.sm:mt-4.flex
-      (ui/button "Confirm"
+      (ui/button "Submit"
                  {:on-click on-submit})]]))
 
 (defn- import-file-graph
-  [*files graph-name config-file]
+  [*files {:keys [graph-name tags]} config-file]
   (state/set-state! :graph/importing :folder)
   (state/set-state! [:graph/importing-state :current-page] (str graph-name " Assets"))
   (async/go
@@ -325,7 +345,8 @@
           asset-files (filter #(string/starts-with? (:rpath %) "assets/") files)]
       (async/<! (p->c (import-logseq-files (filter #(string/starts-with? (:rpath %) "logseq/") files))))
       (async/<! (import-from-asset-files! asset-files))
-      (async/<! (import-from-doc-files! db-conn repo config doc-files))
+      (async/<! (import-from-doc-files! db-conn repo config doc-files
+                                        {:tag-classes (set (string/split tags #",\s*"))}))
       (async/<! (p->c (import-favorites-from-config-edn! db-conn repo config-file)))
       (state/set-state! :graph/importing nil)
       (state/set-state! :graph/importing-state nil)
@@ -335,14 +356,14 @@
                             :info false))
       (finished-cb))))
 
-(defn graph-folder-to-db-import-handler
+(defn import-file-to-db-handler
   "Import from a graph folder as a DB-based graph.
 
 - Page name, journal name creation"
   [ev _opts]
   (let [^js file-objs (array-seq (.-files (.-target ev)))
         original-graph-name (string/replace (.-webkitRelativePath (first file-objs)) #"/.*" "")
-        import-graph-fn (fn [graph-name]
+        import-graph-fn (fn [user-inputs]
                           (let [files (->> file-objs
                                            (map #(hash-map :file-object %
                                                            :rpath (path/trim-dir-prefix original-graph-name (.-webkitRelativePath %))))
@@ -350,24 +371,24 @@
                                                          ;; TODO: Update this when supporting more formats as this aggressively excludes most formats
                                                          (fs-util/ignored-path? original-graph-name (.-webkitRelativePath (:file-object %))))))]
                             (if-let [config-file (first (filter #(= (:rpath %) "logseq/config.edn") files))]
-                              (import-file-graph files graph-name config-file)
+                              (import-file-graph files user-inputs config-file)
                               (notification/show! "Import failed as the file 'logseq/config.edn' was not found for a Logseq graph."
                                                   :error))))]
     (state/set-modal!
-     #(confirm-graph-name-dialog original-graph-name
-                                 (fn [graph-name]
-                                   (cond
-                                     (repo/invalid-graph-name? graph-name)
-                                     (repo/invalid-graph-name-warning)
+     #(import-file-graph-dialog original-graph-name
+                                (fn [{:keys [graph-name] :as user-inputs}]
+                                  (cond
+                                    (repo/invalid-graph-name? graph-name)
+                                    (repo/invalid-graph-name-warning)
 
-                                     (string/blank? graph-name)
-                                     (notification/show! "Empty graph name." :error)
+                                    (string/blank? graph-name)
+                                    (notification/show! "Empty graph name." :error)
 
-                                     (repo-handler/graph-already-exists? graph-name)
-                                     (notification/show! "Please specify another name as another graph with this name already exists!" :error)
+                                    (repo-handler/graph-already-exists? graph-name)
+                                    (notification/show! "Please specify another name as another graph with this name already exists!" :error)
 
-                                     :else
-                                     (import-graph-fn graph-name)))))))
+                                    :else
+                                    (import-graph-fn user-inputs)))))))
 
 
   (rum/defc importer < rum/reactive
@@ -410,11 +431,11 @@
             [[:strong "File to DB graph"]
              [:small  "Import a file-based Logseq graph folder into a new DB graph"]]]
            [:input.absolute.hidden
-            {:id        "import-graph-folder"
+            {:id        "import-file-graph"
              :type      "file"
              :webkitdirectory "true"
              :on-change (debounce (fn [e]
-                                    (graph-folder-to-db-import-handler e {}))
+                                    (import-file-to-db-handler e {}))
                                   1000)}]])
 
          [:label.action-input.flex.items-center.mx-2.my-2