Fix parse-graph namespace bugs

parse-graph didn't have all the extract-options that the app does.
Adding them fixed the bugs.
Also fix publishing bug where old :all-pages-public? option wasn't
respected

deps/graph-parser/src/logseq/graph_parser/cli.cljs

@@ -6,6 +6,7 @@
             [clojure.string :as string]
             [logseq.graph-parser :as graph-parser]
             [logseq.graph-parser.config :as gp-config]
+            [logseq.graph-parser.util :as gp-util]
             [logseq.db :as ldb]))
 
 (defn slurp
@@ -46,13 +47,20 @@ TODO: Fail fast when process exits 1"
 (defn- parse-files
   [conn files {:keys [config] :as options}]
   (let [extract-options (merge {:date-formatter (gp-config/get-date-formatter config)
-                                :user-config config}
+                                :user-config config
+                                :supported-formats (gp-config/supported-formats)
+                                :filename-format (or (:file/name-format config) :legacy)
+                                :extracted-block-ids (atom #{})}
                                (select-keys options [:verbose]))]
     (mapv
      (fn [{:file/keys [path content]}]
        (let [{:keys [ast]}
-             (graph-parser/parse-file conn path content (merge {:extract-options extract-options}
-                                                               (:parse-file-options options)))]
+             (let [parse-file-options
+                   (merge {:extract-options
+                           (assoc extract-options
+                                  :block-pattern (gp-config/get-block-pattern (gp-util/get-format path)))}
+                          (:parse-file-options options))]
+               (graph-parser/parse-file conn path content parse-file-options))]
          {:file path :ast ast}))
      files)))
 

deps/graph-parser/src/logseq/graph_parser/test/docs_graph_helper.cljs

@@ -94,8 +94,7 @@
                 (into {})))
         "Task marker counts")
 
-    (is (= {:markdown 5495 :org 457} ;; 2 pages for namespaces are not parsed
-           (get-block-format-counts db))
+    (is (= {:markdown 5499 :org 457} (get-block-format-counts db))
         "Block format counts")
 
     (is (= {:description 81, :updated-at 46, :tags 5, :logseq.macro-arguments 104
@@ -129,11 +128,17 @@
         "Counts for blocks with common block attributes")
 
     (is (= #{"term" "setting" "book" "templates" "Query table" "page"
-             "Whiteboard" "Whiteboard/Tool" "Community" "Tweet"}
+             "Whiteboard" "Whiteboard/Tool" "Whiteboard/Tool/Shape" "Whiteboard/Object"
+             "Whiteboard/Property" "Community" "Tweet"}
            (->> (d/q '[:find (pull ?n [*]) :where [?b :block/namespace ?n]] db)
                 (map (comp :block/original-name first))
                 set))
-        "Has correct namespaces")))
+        "Has correct namespaces")
+
+    (is (empty? (->> (d/q '[:find ?n :where [?b :block/name ?n]] db)
+                     (map first)
+                     (filter #(string/includes? % "___"))))
+        "Block names don't have the slash/triple-lowbar delimiter")))
 
 (defn docs-graph-assertions
   "These are common assertions that should pass in both graph-parser and main
@@ -145,7 +150,7 @@
   ;; only increase over time as the docs graph rarely has deletions
   (testing "Counts"
     (is (= 306 (count files)) "Correct file count")
-    (is (= 69679 (count (d/datoms db :eavt))) "Correct datoms count")
+    (is (= 69508 (count (d/datoms db :eavt))) "Correct datoms count")
 
     (is (= 5866
            (ffirst

deps/publishing/src/logseq/publishing/html.cljs

@@ -132,7 +132,10 @@ necessary db filtering"
   "Given the graph's db, filters the db using the given options and returns the
 generated index.html string and assets used by the html"
   [db* {:keys [app-state repo-config html-options]}]
-  (let [[db asset-filenames'] (if (:publishing/all-pages-public? repo-config)
+  (let [all-pages-public? (if-let [val (:publishing/all-pages-public? repo-config)]
+                            val
+                            (:all-pages-public? repo-config))
+        [db asset-filenames'] (if all-pages-public?
                                 (db/clean-export! db*)
                                 (db/filter-only-public-pages-and-blocks db*))
         asset-filenames (remove nil? asset-filenames')

src/main/frontend/handler/common/file.cljs

@@ -54,6 +54,7 @@
          options (merge (dissoc options :verbose)
                         {:new? new?
                          :delete-blocks-fn (partial validate-and-get-blocks-to-delete repo-url)
+                         ;; Options here should also be present in gp-cli/parse-graph
                          :extract-options (merge
                                            {:user-config (state/get-config)
                                             :date-formatter (state/get-date-formatter)

src/test/frontend/handler/repo_test.cljs

@@ -2,20 +2,31 @@
   (:require [cljs.test :refer [deftest use-fixtures testing is]]
             [frontend.handler.repo :as repo-handler]
             [frontend.test.helper :as test-helper :refer [load-test-files]]
+            [frontend.state :as state]
             [logseq.graph-parser.cli :as gp-cli]
             [logseq.graph-parser.test.docs-graph-helper :as docs-graph-helper]
             [logseq.graph-parser.util.block-ref :as block-ref]
             [frontend.db.model :as model]
-            [frontend.db.conn :as conn]))
+            [frontend.db.conn :as conn]
+            [clojure.edn :as edn]
+            ["path" :as node-path]
+            ["fs" :as fs]))
 
-(use-fixtures :each {:before test-helper/start-test-db!
-                     :after test-helper/destroy-test-db!})
+(use-fixtures :each {:before (fn []
+                               ;; Set current-repo explicitly since it's not the default
+                               (state/set-current-repo! test-helper/test-db)
+                               (test-helper/start-test-db!))
+                     :after (fn []
+                              (state/set-current-repo! nil)
+                              (test-helper/destroy-test-db!))})
 
 (deftest ^:integration parse-and-load-files-to-db
   (let [graph-dir "src/test/docs-0.9.2"
         _ (docs-graph-helper/clone-docs-repo-if-not-exists graph-dir "v0.9.2")
         files (gp-cli/build-graph-files graph-dir)
-        _ (repo-handler/parse-files-and-load-to-db! test-helper/test-db files {:re-render? false :verbose false})
+        repo-config (edn/read-string (str (fs/readFileSync (node-path/join graph-dir "logseq/config.edn"))))
+        _ (test-helper/with-config repo-config
+            (repo-handler/parse-files-and-load-to-db! test-helper/test-db files {:re-render? false :verbose false}))
         db (conn/get-db test-helper/test-db)]
 
     (docs-graph-helper/docs-graph-assertions db (map :file/path files))))