enhance: improved config.edn for new db graphs

New db config.edn don't include any of the file graph keys or comments.
Previously all comments were confusingly left lying around.
Also moved conversion of imported config.edn so that CLI can use it.
Also temporarily moved one dep to
https://github.com/nextjournal/nbb-test-runner/pull/2 until the PR is
merged

deps/common/nbb.edn

@@ -1,4 +1,5 @@
 {:paths ["src" "resources"]
  :deps
- {io.github.nextjournal/nbb-test-runner
-  {:git/sha "60ed57aa04bca8d604f5ba6b28848bd887109347"}}}
+ ;; TODO: Remove fork when https://github.com/nextjournal/nbb-test-runner/pull/2 is merged
+ {io.github.colin-p-hill/nbb-test-runner
+  {:git/sha "def2cbdb5b3a0e1612b28bf64f5d869c27c733d3"}}}

deps/common/resources/templates/config.edn

@@ -1,5 +1,7 @@
 {:meta/version 1
 
+ ;; == FILE GRAPH CONFIG ==
+ ;;
  ;; Set the preferred format.
  ;; This is _only_ for file graphs.
  ;; Available options:
@@ -42,6 +44,102 @@
  ;; Default value: "yyyy_MM_dd"
  ;; :journal/file-name-format "yyyy_MM_dd"
 
+ ;; Set the default location for storing notes.
+ ;; This is _only_ for file graphs.
+ ;; Default value: "pages"
+ ;; :pages-directory "pages"
+
+ ;; Set the default location for storing journals.
+ ;; This is _only_ for file graphs.
+ ;; Default value: "journals"
+ ;; :journals-directory "journals"
+
+ ;; Set the default location for storing whiteboards.
+ ;; This is _only_ for file graphs.
+ ;; Default value: "whiteboards"
+ ;; :whiteboards-directory "whiteboards"
+
+ ;; Enabling this option converts
+ ;; [[Grant Ideas]] to [[file:./grant_ideas.org][Grant Ideas]] for org-mode.
+ ;; For more information, visit https://github.com/logseq/logseq/issues/672
+ ;; This is _only_ for file graphs.
+ ;; :org-mode/insert-file-link? false
+
+;; Favorites to list on the left sidebar
+ ;; This is _only_ for file graphs.
+ :favorites []
+
+ ;; Set flashcards interval.
+ ;; This is _only_ for file graphs.
+ ;; Expected value:
+ ;; - Float between 0 and 1
+ ;; higher values result in faster changes to the next review interval.
+ ;; Default value: 0.5
+ ;; :srs/learning-fraction 0.5
+
+ ;; Set the initial interval after the first successful review of a card.
+ ;; This is _only_ for file graphs.
+ ;; Default value: 4
+ ;; :srs/initial-interval 4
+
+ ;; Hide specific block properties.
+ ;; This is _only_ for file graphs.
+ ;; Example usage:
+ ;; :block-hidden-properties #{:public :icon}
+
+ ;; Create a page for all properties.
+ ;; This is _only_ for file graphs.
+ ;; Default value: true
+ :property-pages/enabled? true
+
+ ;; Properties to exclude from having property pages
+ ;; This is _only_ for file graphs.
+ ;; Example usage:
+ ;; :property-pages/excludelist #{:duration :author}
+
+ ;; By default, property value separated by commas will not be treated as
+ ;; page references. You can add properties to enable it.
+ ;; This is _only_ for file graphs.
+ ;; Example usage:
+ ;; :property/separated-by-commas #{:alias :tags}
+
+ ;; Properties that are ignored when parsing property values for references
+ ;; This is _only_ for file graphs.
+ ;; Example usage:
+ ;; :ignored-page-references-keywords #{:author :website}
+
+ ;; logbook configuration.
+ ;; This is _only_ for file graphs.
+ ;; :logbook/settings
+ ;; {:with-second-support? false ;limit logbook to minutes, seconds will be eliminated
+ ;;  :enabled-in-all-blocks true ;display logbook in all blocks after timetracking
+ ;;  :enabled-in-timestamped-blocks false ;don't display logbook at all
+ ;; }
+
+ ;; File sync options
+ ;; Ignore these files when syncing, regexp is supported.
+ ;; This is _only_ for file graphs.
+ ;; :file-sync/ignore-files []
+
+ ;; Configure the escaping method for special characters in page titles.
+ ;; This is _only_ for file graphs.
+ ;; Warning:
+ ;;   This is a dangerous operation. To modify the setting,
+ ;;   you'll need to manually rename all affected files and
+ ;;   re-index them on all clients after synchronization.
+ ;;   Incorrect handling may result in messy page titles.
+ ;; Available options:
+ ;;   - :triple-lowbar (default)
+ ;;      ;use triple underscore `___` for slash `/` in page title
+ ;;      ;use Percent-encoding for other invalid characters
+ :file/name-format :triple-lowbar
+ ;; == END OF FILE GRAPH CONFIG ==
+
+ ;; Hide empty block properties
+ ;; This is _only_ for DB graphs.
+ ;; Default value: false
+ ;; :ui/hide-empty-properties? false
+
  ;; Enable tooltip preview on hover.
  ;; Default value: true
  :ui/enable-tooltip? true
@@ -58,11 +156,6 @@
  ;; Default value: true
  :ui/auto-expand-block-refs? true
 
- ;; Hide empty block properties
- ;; This is _only_ for DB graphs.
- ;; Default value: false
- ;; :ui/hide-empty-properties? false
-
  ;; Disable accent marks when searching.
  ;; After changing this setting, rebuild the search index by pressing (^C ^S).
  ;; Default value: true
@@ -137,27 +230,6 @@
  ;; 3. Set "home" as the home page and display multiple pages in the right sidebar:
  ;; :default-home {:page "home", :sidebar ["Page A" "Page B"]}
 
- ;; Set the default location for storing notes.
- ;; This is _only_ for file graphs.
- ;; Default value: "pages"
- ;; :pages-directory "pages"
-
- ;; Set the default location for storing journals.
- ;; This is _only_ for file graphs.
- ;; Default value: "journals"
- ;; :journals-directory "journals"
-
- ;; Set the default location for storing whiteboards.
- ;; This is _only_ for file graphs.
- ;; Default value: "whiteboards"
- ;; :whiteboards-directory "whiteboards"
-
- ;; Enabling this option converts
- ;; [[Grant Ideas]] to [[file:./grant_ideas.org][Grant Ideas]] for org-mode.
- ;; For more information, visit https://github.com/logseq/logseq/issues/672
- ;; This is _only_ for file graphs.
- ;; :org-mode/insert-file-link? false
-
  ;; Configure custom shortcuts.
  ;; Syntax:
  ;; 1. + indicates simultaneous key presses, e.g., `Ctrl+Shift+a`.
@@ -274,58 +346,6 @@
  ;;  :charge-strength -600   ; Default value: -600
  ;;  :charge-range    600}   ; Default value: 600
 
-
- ;; Favorites to list on the left sidebar
- ;; This is _only_ for file graphs.
- :favorites []
-
- ;; Set flashcards interval.
- ;; This is _only_ for file graphs.
- ;; Expected value:
- ;; - Float between 0 and 1
- ;; higher values result in faster changes to the next review interval.
- ;; Default value: 0.5
- ;; :srs/learning-fraction 0.5
-
- ;; Set the initial interval after the first successful review of a card.
- ;; This is _only_ for file graphs.
- ;; Default value: 4
- ;; :srs/initial-interval 4
-
- ;; Hide specific block properties.
- ;; This is _only_ for file graphs.
- ;; Example usage:
- ;; :block-hidden-properties #{:public :icon}
-
- ;; Create a page for all properties.
- ;; This is _only_ for file graphs.
- ;; Default value: true
- :property-pages/enabled? true
-
- ;; Properties to exclude from having property pages
- ;; This is _only_ for file graphs.
- ;; Example usage:
- ;; :property-pages/excludelist #{:duration :author}
-
- ;; By default, property value separated by commas will not be treated as
- ;; page references. You can add properties to enable it.
- ;; This is _only_ for file graphs.
- ;; Example usage:
- ;; :property/separated-by-commas #{:alias :tags}
-
- ;; Properties that are ignored when parsing property values for references
- ;; This is _only_ for file graphs.
- ;; Example usage:
- ;; :ignored-page-references-keywords #{:author :website}
-
- ;; logbook configuration.
- ;; This is _only_ for file graphs.
- ;; :logbook/settings
- ;; {:with-second-support? false ;limit logbook to minutes, seconds will be eliminated
- ;;  :enabled-in-all-blocks true ;display logbook in all blocks after timetracking
- ;;  :enabled-in-timestamped-blocks false ;don't display logbook at all
- ;; }
-
  ;; Mobile photo upload configuration.
  ;; :mobile/photo
  ;; {:allow-editing? true
@@ -375,11 +395,6 @@
  ;;  :redirect-page? false          ;; Default value: false
  ;;  :default-page "quick capture"} ;; Default page: "quick capture"
 
- ;; File sync options
- ;; Ignore these files when syncing, regexp is supported.
- ;; This is _only_ for file graphs.
- ;; :file-sync/ignore-files []
-
  ;; Configure the Enter key behavior for
  ;; context-aware editing with DWIM (Do What I Mean).
  ;; context-aware Enter key behavior implies that pressing Enter will
@@ -393,16 +408,4 @@
  ;;  :page-ref?        true        ;; Default value: true
  ;;  :properties?      true        ;; Default value: true
  ;;  :list?            false}      ;; Default value: false
-
- ;; Configure the escaping method for special characters in page titles.
- ;; This is _only_ for file graphs.
- ;; Warning:
- ;;   This is a dangerous operation. To modify the setting,
- ;;   you'll need to manually rename all affected files and
- ;;   re-index them on all clients after synchronization.
- ;;   Incorrect handling may result in messy page titles.
- ;; Available options:
- ;;   - :triple-lowbar (default)
- ;;      ;use triple underscore `___` for slash `/` in page title
- ;;      ;use Percent-encoding for other invalid characters
- :file/name-format :triple-lowbar}
+ }

deps/common/src/logseq/common/config.cljs

@@ -1,5 +1,5 @@
 (ns logseq.common.config
-  "Common config and constants that are shared between deps and app"
+  "Common config constants and fns that are shared between deps and app"
   (:require [clojure.string :as string]
             [goog.object :as gobj]))
 
@@ -115,3 +115,43 @@
       "*"
 
       "-")))
+
+(defn create-config-for-db-graph
+  "Given a new config.edn file string, creates a config.edn for use with only DB graphs"
+  [config]
+  (string/replace config #"(?m)[\s]*;; == FILE GRAPH CONFIG ==(?:.|\n)*?;; == END OF FILE GRAPH CONFIG ==\n?" ""))
+
+(def file-only-config
+  "File only config keys that are deprecated in DB graphs along with
+  descriptions for their deprecation."
+  (merge
+   (zipmap
+    [:file/name-format
+     :file-sync/ignore-files
+     :hidden
+     :ignored-page-references-keywords
+     :journal/file-name-format
+     :journal/page-title-format
+     :journals-directory
+     :logbook/settings
+     :org-mode/insert-file-link?
+     :pages-directory
+     :preferred-workflow
+     :property/separated-by-commas
+     :property-pages/excludelist
+     :srs/learning-fraction
+     :srs/initial-interval
+     :whiteboards-directory]
+    (repeat "is not used in DB graphs"))
+   {:preferred-format
+    "is not used in DB graphs as there is only markdown mode."
+    :property-pages/enabled?
+    "is not used in DB graphs as all properties have pages"
+    :block-hidden-properties
+    "is not used in DB graphs as hiding a property is done in its configuration"
+    :feature/enable-block-timestamps?
+    "is not used in DB graphs as it is always enabled"
+    :favorites
+    "is not stored in config for DB graphs"
+    :default-templates
+    "is replaced by #Template and the `Apply template to tags` property"}))

deps/common/test/logseq/common/config_test.cljc

@@ -0,0 +1,45 @@
+(ns logseq.common.config-test
+  (:require [cljs.test :refer [deftest is]]
+            [clojure.string :as string]
+            [logseq.common.config :as common-config]
+            #?(:org.babashka/nbb [nbb.classpath :as cp])
+            ["fs" :as fs]
+            ["path" :as node-path]))
+
+(deftest remove-hidden-files
+  (let [files ["pages/foo.md" "pages/bar.md"
+               "script/README.md" "script/config.edn"
+               "dev/README.md" "dev/config.edn"]]
+    (is (= ["pages/foo.md" "pages/bar.md"]
+           #_:clj-kondo/ignore ;; buggy unresolved var
+           (common-config/remove-hidden-files
+            files
+            {:hidden ["script" "/dev"]}
+            identity))
+        "Removes hidden relative files")
+
+    (is (= ["/pages/foo.md" "/pages/bar.md"]
+           (common-config/remove-hidden-files
+            (map #(str "/" %) files)
+            {:hidden ["script" "/dev"]}
+            identity))
+        "Removes hidden files if they start with '/'")))
+
+(defn find-on-classpath [classpath rel-path]
+  (some (fn [dir]
+          (let [f (node-path/join dir rel-path)]
+            (when (fs/existsSync f) f)))
+        (string/split classpath #":")))
+
+#?(:org.babashka/nbb
+   (deftest create-config-for-db-graph
+     (let [original-config (some-> (find-on-classpath (cp/get-classpath) "templates/config.edn") fs/readFileSync str)
+           _ (assert original-config "config.edn must not be blank")
+           migrated-config (common-config/create-config-for-db-graph original-config)
+           forbidden-kws-regex (re-pattern (str (string/join "|" (keys common-config/file-only-config))))]
+      ;;  (println migrated-config)
+       (is (not (string/includes? migrated-config "== FILE ONLY CONFIG"))
+           "No longer includes file config header")
+       (assert (re-find forbidden-kws-regex original-config) "File config keys present in original config")
+       (is (not (re-find forbidden-kws-regex migrated-config))
+           "File config keys no longer present in migrated config"))))

deps/common/test/logseq/common/config_test.cljs

@@ -1,21 +0,0 @@
-(ns logseq.common.config-test
-  (:require [logseq.common.config :as common-config]
-            [cljs.test :refer [deftest is]]))
-
-(deftest remove-hidden-files
-  (let [files ["pages/foo.md" "pages/bar.md"
-               "script/README.md" "script/config.edn"
-               "dev/README.md" "dev/config.edn"]]
-    (is (= ["pages/foo.md" "pages/bar.md"]
-           (common-config/remove-hidden-files
-            files
-            {:hidden ["script" "/dev"]}
-            identity))
-        "Removes hidden relative files")
-
-    (is (= ["/pages/foo.md" "/pages/bar.md"]
-           (common-config/remove-hidden-files
-            (map #(str "/" %) files)
-            {:hidden ["script" "/dev"]}
-            identity))
-        "Removes hidden files if they start with '/'")))

deps/graph-parser/src/logseq/graph_parser/exporter.cljs

@@ -764,6 +764,7 @@
     {:block block'' :properties-tx properties-tx}))
 
 (defn- pretty-print-dissoc
+  "Remove list of keys from a given map string while preserving whitespace"
   [s dissoc-keys]
   (-> (reduce rewrite/dissoc
               (rewrite/parse-string s)
@@ -1496,7 +1497,11 @@
                                              <save-file default-save-file}}]
   (-> (<read-file config-file)
       (p/then #(p/do!
-                (<save-file repo-or-conn "logseq/config.edn" %)
+                (<save-file repo-or-conn
+                            "logseq/config.edn"
+                            ;; Converts a file graph config.edn for use with DB graphs. Unlike common-config/create-config-for-db-graph,
+                            ;; manually dissoc deprecated keys for config to be valid
+                            (pretty-print-dissoc % (keys common-config/file-only-config)))
                 (let [config (edn/read-string %)]
                   (when-let [title-format (or (:journal/page-title-format config) (:date-formatter config))]
                     (ldb/transact! repo-or-conn [{:db/ident :logseq.class/Journal

deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs

@@ -693,7 +693,7 @@
 
 (deftest-async export-config-file-sets-title-format
   (p/let [conn (db-test/create-conn)
-          read-file #(p/do! (pr-str {:journal/page-title-format "yyyy-MM-dd"}))
+          read-file #(p/do! "{:journal/page-title-format \"yyyy-MM-dd\"}")
           _ (gp-exporter/export-config-file conn "logseq/config.edn" read-file {})]
     (is (= "yyyy-MM-dd"
            (:logseq.property.journal/title-format (d/entity @conn :logseq.class/Journal)))

src/main/frontend/components/imports.cljs

@@ -1,6 +1,7 @@
 (ns frontend.components.imports
   "Import data into Logseq."
-  (:require [cljs-time.core :as t]
+  (:require [borkdude.rewrite-edn :as rewrite]
+            [cljs-time.core :as t]
             [cljs.pprint :as pprint]
             [clojure.string :as string]
             [frontend.components.onboarding.setups :as setups]
@@ -34,7 +35,8 @@
             [logseq.shui.hooks :as hooks]
             [logseq.shui.ui :as shui]
             [promesa.core :as p]
-            [rum.core :as rum]))
+            [rum.core :as rum]
+            [logseq.common.config :as common-config]))
 
 ;; Can't name this component as `frontend.components.import` since shadow-cljs
 ;; will complain about it.
@@ -347,6 +349,15 @@
                    (fs/mkdir-if-not-exists parent-dir)
                    (fs/write-plain-text-file! repo repo-dir (:path file) content {:skip-transact? true})))))))
 
+(defn- convert-file-config-for-db-graph
+  "Converts a file graph config.edn for use with DB graphs. Unlike common-config/create-config-for-db-graph,
+   have to manually dissoc deprecated keys for config to be valid"
+  [content]
+  (-> (reduce rewrite/dissoc
+              (rewrite/parse-string (str content))
+              (keys common-config/file-only-config))
+      str))
+
 (defn- import-file-graph
   [*files
    {:keys [graph-name tag-classes property-classes property-parent-classes] :as user-options}
@@ -370,7 +381,7 @@
                    ;; config file options
                    :default-config config/config-default-content
                    :<save-config-file (fn save-config-file [_ path content]
-                                        (let [migrated-content (repo-handler/migrate-db-config content)]
+                                        (let [migrated-content (convert-file-config-for-db-graph content)]
                                           (db-editor-handler/save-file! path migrated-content)))
                    ;; logseq file options
                    :<save-logseq-file (fn save-logseq-file [_ path content]

src/main/frontend/handler/common/config_edn.cljs

@@ -7,7 +7,8 @@
             [lambdaisland.glogi :as log]
             [frontend.handler.notification :as notification]
             [goog.string :as gstring]
-            [reitit.frontend.easy :as rfe]))
+            [reitit.frontend.easy :as rfe]
+            [logseq.common.config :as common-config]))
 
 (defn- humanize-more
   "Make error maps from me/humanize more readable for users. Doesn't try to handle
@@ -86,40 +87,6 @@ nested keys or positional errors e.g. tuples"
       :else
       (validate-config-map parsed-body schema path))))
 
-(def file-only-config
-  "File only config that is deprecated in DB graphs"
-  (merge
-   (zipmap
-    [:file/name-format
-     :file-sync/ignore-files
-     :hidden
-     :ignored-page-references-keywords
-     :journal/file-name-format
-     :journal/page-title-format
-     :journals-directory
-     :logbook/settings
-     :org-mode/insert-file-link?
-     :pages-directory
-     :preferred-workflow
-     :property/separated-by-commas
-     :property-pages/excludelist
-     :srs/learning-fraction
-     :srs/initial-interval
-     :whiteboards-directory]
-    (repeat "is not used in DB graphs"))
-   {:preferred-format
-    "is not used in DB graphs as there is only markdown mode."
-    :property-pages/enabled?
-    "is not used in DB graphs as all properties have pages"
-    :block-hidden-properties
-    "is not used in DB graphs as hiding a property is done in its configuration"
-    :feature/enable-block-timestamps?
-    "is not used in DB graphs as it is always enabled"
-    :favorites
-    "is not stored in config for DB graphs"
-    :default-templates
-    "is replaced by #Template and the `Apply template to tags` property"}))
-
 (defn detect-deprecations
   "Detects config keys that will or have been deprecated"
   [path content {:keys [db-graph?]}]
@@ -132,7 +99,7 @@ nested keys or positional errors e.g. tuples"
                    "is no longer supported."}
                    db-graph?
                    (merge
-                    file-only-config))]
+                    common-config/file-only-config))]
     (cond
       (= body ::failed-to-detect)
       (log/info :msg "Skip deprecation check since config is not valid edn")

src/main/frontend/handler/repo.cljs

@@ -1,8 +1,7 @@
 (ns frontend.handler.repo
   "System-component-like ns that manages user's repos/graphs"
   (:refer-clojure :exclude [clone])
-  (:require [borkdude.rewrite-edn :as rewrite]
-            [cljs-bean.core :as bean]
+  (:require [cljs-bean.core :as bean]
             [clojure.string :as string]
             [electron.ipc :as ipc]
             [frontend.config :as config]
@@ -11,7 +10,6 @@
             [frontend.db.persist :as db-persist]
             [frontend.db.react :as react]
             [frontend.db.restore :as db-restore]
-            [frontend.handler.common.config-edn :as config-edn-common-handler]
             [frontend.handler.global-config :as global-config-handler]
             [frontend.handler.graph :as graph-handler]
             [frontend.handler.notification :as notification]
@@ -27,7 +25,8 @@
             [frontend.util :as util]
             [frontend.util.fs :as util-fs]
             [frontend.util.text :as text-util]
-            [promesa.core :as p]))
+            [promesa.core :as p]
+            [logseq.common.config :as common-config]))
 
 ;; Project settings should be checked in two situations:
 ;; 1. User changes the config.edn directly in logseq.com (fn: alter-file)
@@ -177,16 +176,9 @@
   (let [full-graph-name (string/lower-case (str config/db-version-prefix graph-name))]
     (some #(= (some-> (:url %) string/lower-case) full-graph-name) (state/get-repos))))
 
-(defn migrate-db-config
-  [content]
-  (-> (reduce rewrite/dissoc
-              (rewrite/parse-string (str content))
-              (keys config-edn-common-handler/file-only-config))
-      str))
-
 (defn- create-db [full-graph-name {:keys [file-graph-import?]}]
   (->
-   (p/let [config (migrate-db-config config/config-default-content)
+   (p/let [config (common-config/create-config-for-db-graph config/config-default-content)
            _ (persist-db/<new full-graph-name
                               (cond-> {:config config}
                                 file-graph-import? (assoc :import-type :file-graph)))