Finish up logseq.graph-parser

- Parser now parses all graph files like the app does, not just pages and journals.
  This required extracting another fn from repo-handler
- Add and tweak CI steps that are specific to graph-parser. All
  namespaces in this library are checked for nbb compatibility
- Cleaned up parser cli API so only one fn is needed for scripts
- Tests were updated to match new parsing behavior
- large_vars.clj can run with a smaller max-line-count after only refactoring two fns
- Add docs

.github/workflows/build.yml

@@ -81,7 +81,7 @@ jobs:
 
       # In this job because it depends on an npm package
       - name: Load nbb compatible namespaces
-        run: bb test:load-nbb-compatible-namespaces
+        run: bb test:load-namespaces-with-nbb
 
   lint:
     runs-on: ubuntu-latest

.github/workflows/graph-parser.yml

@@ -1,6 +1,7 @@
 name: logseq graph-parser CI
 
 on:
+  # Path filters ensure jobs only kick off if a change is made to graph-parser
   push:
     branches: [master]
     paths:
@@ -47,10 +48,10 @@ jobs:
         with:
           cli: ${{ env.CLOJURE_VERSION }}
 
-      # - name: Setup Babashka
-      #   uses: turtlequeue/[email protected]
-      #   with:
-      #     babashka-version: ${{ env.BABASHKA_VERSION }}
+      - name: Setup Babashka
+        uses: turtlequeue/[email protected]
+        with:
+          babashka-version: ${{ env.BABASHKA_VERSION }}
 
       - name: Clojure cache
         uses: actions/cache@v2
@@ -64,20 +65,20 @@ jobs:
 
       - name: Fetch Clojure deps
         if: steps.clojure-deps.outputs.cache-hit != 'true'
-        run: clojure -A:test -P
+        run: cd deps/graph-parser && clojure -A:test -P
 
       - name: Fetch yarn deps
         run: cd deps/graph-parser && yarn install --frozen-lockfile
 
       - name: Run ClojureScript tests
-        run: clojure -M:test
+        run: cd deps/graph-parser && clojure -M:test
 
       - name: Run nbb-logseq tests
         run: cd deps/graph-parser && yarn nbb-logseq -cp src:test -m logseq.graph-parser.nbb-test-runner/run-tests
 
-      # # In this job because it depends on an npm package
-      # - name: Load nbb compatible namespaces
-      #   run: bb test:load-nbb-compatible-namespaces
+      # In this job because it depends on an npm package
+      - name: Load namespaces into nbb-logseq
+        run: bb test:load-all-namespaces-with-nbb deps/graph-parser src
 
   lint:
     runs-on: ubuntu-latest
@@ -105,8 +106,8 @@ jobs:
       - name: Run clj-kondo lint
         run: cd deps/graph-parser && clojure -M:clj-kondo --parallel --lint src test
 
-      - name: Lint for vars that are too large
-        run: scripts/large_vars.clj deps/graph-parser/src
-
       - name: Carve lint for unused vars
         run: cd deps/graph-parser && ../../scripts/carve.clj
+
+      - name: Lint for vars that are too large
+        run: scripts/large_vars.clj deps/graph-parser/src '{:max-lines-count 75}'

CODEBASE_OVERVIEW.md

@@ -46,7 +46,11 @@ After cloning the [Logseq repository](https://github.com/logseq/logseq), there a
 
   - `src/main/frontend/` contains code that powers the Logseq editor. Folders and files inside are organized by features or functions. For example, `components` contains all the UI components and `handler` contains all the event-handling code. You can explore on your own interest.
 
-  - `src/main/logseq/` contains the api used by plugins and the graph-parser.
+  - `src/main/logseq/` contains the api used by plugins.
+
+- `deps/` contains dependencies or libraries used by the frontend.
+
+  - `deps/graph-parser/` is a library that parses a Logseq graph and saves it to a database.
 
 ## Data Flow
 

bb.edn

@@ -26,9 +26,12 @@
   dev:lint
   logseq.tasks.dev/lint
 
-  test:load-nbb-compatible-namespaces
+  test:load-namespaces-with-nbb
   logseq.tasks.nbb/load-compatible-namespaces
 
+  test:load-all-namespaces-with-nbb
+  logseq.tasks.nbb/load-all-namespaces
+
   lang:list
   logseq.tasks.lang/list-langs
 

deps/graph-parser/.carve/ignore

@@ -1,5 +1,5 @@
 ;; For CLI
-logseq.graph-parser.cli/parse
+logseq.graph-parser.cli/parse-graph
 ;; For CLI
 logseq.graph-parser.db/start-conn
 ;; For CLI

deps/graph-parser/.clj-kondo/config.edn

@@ -16,5 +16,4 @@
              logseq.graph-parser.property gp-property
              logseq.graph-parser.config gp-config
              logseq.graph-parser.date-time-util date-time-util}}}
- :lint-as {promesa.core/let clojure.core/let}
  :skip-comments true}

deps/graph-parser/README.md

@@ -0,0 +1,63 @@
+## Description
+
+This library parses a logseq graph directory and returns it as a datascript
+database connection. This library powers the Logseq app and also runs from the
+commandline, _independent_ of the app. This is powerful as this can run anywhere
+that a Node.js script has access to a Logseq graph e.g. on CI processes like
+Github Actions. This library is compatible with ClojureScript and with
+[nbb-logseq](https://github.com/logseq/nbb-logseq) to respectively provide
+frontend and commandline functionality.
+
+## API
+
+This library is under the parent namespace `logseq.graph-parser`. This library
+provides two main namespaces for parsing, `logseq.graph-parser` and
+`logseq.graph-parser.cli`. `logseq.graph-parser/parse-file` is the main fn for
+the frontend. `logseq.graph-parser.cli/parse-graph` is the main fn for node.js
+CLIs.
+
+## Usage
+
+See `logseq.graph-parser.cli-test` for now. A real world example is coming soon.
+
+## Dev
+
+This follows the practices that [the Logseq frontend
+follows](/docs/dev-practices.md). Most of the same linters are used, with
+configurations that are specific to this library. See [this library's CI
+file](/.github/workflows/graph-parser.yml) for linting examples.
+
+### Setup
+
+To run linters and tests, you'll want to install yarn dependencies once:
+```
+yarn install
+```
+
+This step is not needed if you're just running the application.
+
+### Testing
+
+Since this file is compatible with cljs and nbb-logseq, tests are run against both languages.
+
+ClojureScript tests use https://github.com/Olical/cljs-test-runner. To run tests:
+```
+clojure -M:test
+```
+
+To see available options that can run specific tests or namespaces: `clojure -M:test --help`
+
+To run nbb-logseq tests:
+```
+yarn nbb-logseq -cp src:test -m logseq.graph-parser.nbb-test-runner/run-tests
+```
+
+### Managing dependencies
+
+The package.json dependencies are just for testing and should be updated if there is
+new behavior to test.
+
+The deps.edn dependecies are used by both ClojureScript and nbb-logseq. Their
+versions should be backwards compatible with each other with priority given to
+the frontend. _No new dependency_ should be introduced to this library without
+an understanding of the tradeoffs of adding this to nbb-logseq.

deps/graph-parser/deps.edn

@@ -11,9 +11,9 @@
   cljs-bean/cljs-bean {:mvn/version "1.5.0"}}
 
  :aliases
- ;; This runs tests with nodejs. Would be nice to run this with a headless env since
- ;; this is how its normally run in the app but this requires more setup with
- ;; karma and shadow-cljs.edn
+ ;; This runs tests with nodejs. Would be nice to run this with in a browser env
+ ;; since this is how its normally run in the app but this requires more setup
+ ;; with karma, shadow-cljs.edn and headless mode on CI
  {:test {:extra-paths ["test"]
          :extra-deps {olical/cljs-test-runner {:mvn/version "3.8.0"}
                       org.clojure/clojurescript {:mvn/version "1.11.54"}}

deps/graph-parser/src/logseq/graph_parser.cljs

@@ -1,38 +1,37 @@
-(ns ^:nbb-compatible logseq.graph-parser
-  "Main ns for parsing graph from source files"
+(ns logseq.graph-parser
+  "Main ns used by logseq app to parse graph from source files"
   (:require [datascript.core :as d]
             [logseq.graph-parser.extract :as extract]
             [logseq.graph-parser.util :as gp-util]
             [logseq.graph-parser.date-time-util :as date-time-util]
             [logseq.graph-parser.config :as gp-config]
+            [clojure.string :as string]
             [clojure.set :as set]))
 
 (defn- db-set-file-content!
   "Modified copy of frontend.db.model/db-set-file-content!"
-  [db path content]
+  [conn path content]
   (let [tx-data {:file/path path
                  :file/content content}]
-    (d/transact! db [tx-data] {:skip-refresh? true})))
+    (d/transact! conn [tx-data] {:skip-refresh? true})))
 
 (defn parse-file
-  "Parse file and save parsed data to the given db"
-  [db file content {:keys [new? delete-blocks-fn new-graph? extract-options]
+  "Parse file and save parsed data to the given db. Main parse fn used by logseq app"
+  [conn file content {:keys [new? delete-blocks-fn new-graph? extract-options]
                     :or {new? true
                          new-graph? false
                          delete-blocks-fn (constantly [])}}]
-  (db-set-file-content! db file content)
+  (db-set-file-content! conn file content)
   (let [format (gp-util/get-format file)
         file-content [{:file/path file}]
         tx (if (contains? gp-config/mldoc-support-formats format)
              (let [extract-options' (merge {:block-pattern (gp-config/get-block-pattern format)
                                             :date-formatter "MMM do, yyyy"
                                             :supported-formats (gp-config/supported-formats)}
-                                           extract-options)
+                                           extract-options
+                                           {:db @conn})
                    [pages blocks]
-                   (extract/extract-blocks-pages
-                    file
-                    content
-                    (merge extract-options' {:db @db}))
+                   (extract/extract-blocks-pages file content extract-options')
                    delete-blocks (delete-blocks-fn (first pages) file)
                    block-ids (map (fn [block] {:block/uuid (:block/uuid block)}) blocks)
                    block-refs-ids (->> (mapcat :block/refs blocks)
@@ -51,13 +50,21 @@
                                new?
                                ;; TODO: use file system timestamp?
                                (assoc :file/created-at (date-time-util/time-ms)))])]
-    (d/transact! db (gp-util/remove-nils tx) (when new-graph? {:new-graph? true}))))
+    (d/transact! conn (gp-util/remove-nils tx) (when new-graph? {:new-graph? true}))))
 
-(defn parse
-  "Main parse fn"
-  ([db files]
-   (parse db files {}))
-  ([db files {:keys [config]}]
-   (let [extract-options {:date-formatter (gp-config/get-date-formatter config)}]
-     (doseq [{:file/keys [path content]} files]
-       (parse-file db path content {:extract-options extract-options})))))
+(defn filter-files
+  "Filters files in preparation for parsing. Only includes files that are
+  supported by parser"
+  [files]
+  (let [support-files (filter
+                       (fn [file]
+                         (let [format (gp-util/get-format (:file/path file))]
+                           (contains? (set/union #{:edn :css} gp-config/mldoc-support-formats) format)))
+                       files)
+        support-files (sort-by :file/path support-files)
+        {journals true non-journals false} (group-by (fn [file] (string/includes? (:file/path file) "journals/")) support-files)
+        {built-in true others false} (group-by (fn [file]
+                                                 (or (string/includes? (:file/path file) "contents.")
+                                                     (string/includes? (:file/path file) ".edn")
+                                                     (string/includes? (:file/path file) "custom.css"))) non-journals)]
+    (concat (reverse journals) built-in others)))

deps/graph-parser/src/logseq/graph_parser/block.cljc

@@ -1,4 +1,4 @@
-(ns ^:nbb-compatible logseq.graph-parser.block
+(ns logseq.graph-parser.block
   ;; Disable clj linters since we don't support clj
   #?(:clj {:clj-kondo/config {:linters {:unresolved-namespace {:level :off}
                                         :unresolved-symbol {:level :off}}}})

deps/graph-parser/src/logseq/graph_parser/cli.cljs

@@ -1,20 +1,66 @@
 (ns logseq.graph-parser.cli
-  "Ns only for use by CLIs as it uses node.js libraries"
+  "Primary ns to parse graphs with node.js based CLIs"
   (:require ["fs" :as fs]
+            ["child_process" :as child-process]
             [clojure.edn :as edn]
-            [logseq.graph-parser :as graph-parser]))
+            [clojure.string :as string]
+            [logseq.graph-parser :as graph-parser]
+            [logseq.graph-parser.config :as gp-config]
+            [logseq.graph-parser.db :as gp-db]))
+
+(defn- slurp
+  "Like clojure.core/slurp"
+  [file]
+  (str (fs/readFileSync file)))
+
+(defn- sh
+  "Run shell cmd synchronously and print to inherited streams by default. Aims
+    to be similar to babashka.tasks/shell
+TODO: Fail fast when process exits 1"
+  [cmd opts]
+  (child-process/spawnSync (first cmd)
+                           (clj->js (rest cmd))
+                           (clj->js (merge {:stdio "inherit"} opts))))
+
+(defn build-graph-files
+  "Given a git graph directory, returns allowed file paths and their contents in
+  preparation for parsing"
+  [dir]
+  (let [files (->> (str (.-stdout (sh ["git" "ls-files"]
+                                      {:cwd dir :stdio nil})))
+                   string/split-lines
+                   (map #(hash-map :file/path (str dir "/" %)))
+                   graph-parser/filter-files)]
+    (mapv #(assoc % :file/content (slurp (:file/path %))) files)))
 
 (defn- read-config
   "Commandline version of frontend.handler.common/read-config without graceful
   handling of broken config. Config is assumed to be at $dir/logseq/config.edn "
   [dir]
-  (if (fs/existsSync (str dir "/logseq/config.edn"))
-    (-> (str dir "/logseq/config.edn") fs/readFileSync str edn/read-string)
-    {}))
+  (let [config-file (str dir "/" gp-config/app-name "/config.edn")]
+    (if (fs/existsSync config-file)
+     (-> config-file fs/readFileSync str edn/read-string)
+     {})))
+
+(defn- parse-files
+  [conn files {:keys [config] :as options}]
+  (let [extract-options (merge {:date-formatter (gp-config/get-date-formatter config)}
+                               (select-keys options [:verbose]))]
+    (doseq [{:file/keys [path content]} files]
+      (graph-parser/parse-file conn path content {:extract-options extract-options}))))
 
-(defn parse
-  "Main entry point for parsing"
-  [dir db files]
-  (graph-parser/parse db
-                      files
-                      {:config (read-config dir)}))
+(defn parse-graph
+  "Parses a given graph directory and returns a datascript connection and all
+  files that were processed. The directory is parsed as if it were a new graph
+  as it can't assume that the metadata in logseq/ is up to date. Directory is
+  assumed to be using git"
+  ([dir]
+   (parse-graph dir {}))
+  ([dir options]
+   (let [files (build-graph-files dir)
+         conn (gp-db/start-conn)
+         config (read-config dir)]
+     (println "Parsing" (count files) "files...")
+     (parse-files conn files (merge options {:config config}))
+     {:conn conn
+      :files (map :file/path files)})))

deps/graph-parser/src/logseq/graph_parser/config.cljs

@@ -1,9 +1,13 @@
-(ns ^:nbb-compatible logseq.graph-parser.config
+(ns logseq.graph-parser.config
   "Config that is shared between graph-parser and rest of app"
   (:require [logseq.graph-parser.util :as gp-util]
             [clojure.set :as set]
             [clojure.string :as string]))
 
+(def app-name
+  "Copy of frontend.config/app-name. Too small to couple to main app"
+  "logseq")
+
 (defonce local-assets-dir "assets")
 
 (defn local-asset?

deps/graph-parser/src/logseq/graph_parser/date_time_util.cljs

@@ -1,4 +1,4 @@
-(ns ^:nbb-compatible logseq.graph-parser.date-time-util
+(ns logseq.graph-parser.date-time-util
   "cljs-time util fns for graph-parser"
   (:require [cljs-time.coerce :as tc]
             [cljs-time.core :as t]

deps/graph-parser/src/logseq/graph_parser/db/default.cljs

@@ -1,4 +1,4 @@
-(ns ^:nbb-compatible logseq.graph-parser.db.default
+(ns logseq.graph-parser.db.default
   (:require [clojure.string :as string]))
 
 (defonce built-in-pages-names

deps/graph-parser/src/logseq/graph_parser/db/schema.cljs

@@ -1,4 +1,4 @@
-(ns ^:nbb-compatible logseq.graph-parser.db.schema)
+(ns logseq.graph-parser.db.schema)
 
 (defonce version 1)
 (defonce ast-version 1)

deps/graph-parser/src/logseq/graph_parser/extract.cljc

@@ -1,4 +1,4 @@
-(ns ^:nbb-compatible logseq.graph-parser.extract
+(ns logseq.graph-parser.extract
   ;; Disable clj linters since we don't support clj
   #?(:clj {:clj-kondo/config {:linters {:unresolved-namespace {:level :off}
                                         :unresolved-symbol {:level :off}}}})
@@ -39,6 +39,52 @@
               (or first-block-name file-name)
               (or file-name first-block-name)))))))
 
+(defn- build-page-entity
+  [properties file page-name page ref-tags {:keys [date-formatter db]}]
+  (let [alias (:alias properties)
+        alias (if (string? alias) [alias] alias)
+        aliases (and alias
+                     (seq (remove #(or (= page-name (gp-util/page-name-sanity-lc %))
+                                       (string/blank? %)) ;; disable blank alias
+                                  alias)))
+        aliases (->>
+                 (map
+                  (fn [alias]
+                    (let [page-name (gp-util/page-name-sanity-lc alias)
+                          aliases (distinct
+                                   (conj
+                                    (remove #{alias} aliases)
+                                    page))
+                          aliases (when (seq aliases)
+                                    (map
+                                     (fn [alias]
+                                       {:block/name (gp-util/page-name-sanity-lc alias)})
+                                     aliases))]
+                      (if (seq aliases)
+                        {:block/name page-name
+                         :block/alias aliases}
+                        {:block/name page-name})))
+                  aliases)
+                 (remove nil?))]
+    (cond->
+     (gp-util/remove-nils
+      (assoc
+       (gp-block/page-name->map page false db true date-formatter)
+       :block/file {:file/path (gp-util/path-normalize file)}))
+     (seq properties)
+     (assoc :block/properties properties)
+
+     (seq aliases)
+     (assoc :block/alias aliases)
+
+     (:tags properties)
+     (assoc :block/tags (let [tags (:tags properties)
+                              tags (if (string? tags) [tags] tags)
+                              tags (remove string/blank? tags)]
+                          (swap! ref-tags set/union (set tags))
+                          (map (fn [tag] {:block/name (gp-util/page-name-sanity-lc tag)
+                                          :block/original-name tag})
+                               tags))))))
 
 ;; TODO: performance improvement
 (defn- extract-pages-and-blocks
@@ -64,50 +110,7 @@
                                      :block/refs block-ref-pages
                                      :block/path-refs block-path-ref-pages))))
                       blocks)
-          page-entity (let [alias (:alias properties)
-                            alias (if (string? alias) [alias] alias)
-                            aliases (and alias
-                                         (seq (remove #(or (= page-name (gp-util/page-name-sanity-lc %))
-                                                           (string/blank? %)) ;; disable blank alias
-                                                      alias)))
-                            aliases (->>
-                                     (map
-                                      (fn [alias]
-                                        (let [page-name (gp-util/page-name-sanity-lc alias)
-                                              aliases (distinct
-                                                       (conj
-                                                        (remove #{alias} aliases)
-                                                        page))
-                                              aliases (when (seq aliases)
-                                                        (map
-                                                         (fn [alias]
-                                                           {:block/name (gp-util/page-name-sanity-lc alias)})
-                                                         aliases))]
-                                          (if (seq aliases)
-                                            {:block/name page-name
-                                             :block/alias aliases}
-                                            {:block/name page-name})))
-                                      aliases)
-                                     (remove nil?))]
-                        (cond->
-                         (gp-util/remove-nils
-                          (assoc
-                           (gp-block/page-name->map page false db true date-formatter)
-                           :block/file {:file/path (gp-util/path-normalize file)}))
-                         (seq properties)
-                         (assoc :block/properties properties)
-
-                         (seq aliases)
-                         (assoc :block/alias aliases)
-
-                         (:tags properties)
-                         (assoc :block/tags (let [tags (:tags properties)
-                                                  tags (if (string? tags) [tags] tags)
-                                                  tags (remove string/blank? tags)]
-                                              (swap! ref-tags set/union (set tags))
-                                              (map (fn [tag] {:block/name (gp-util/page-name-sanity-lc tag)
-                                                              :block/original-name tag})
-                                                   tags)))))
+          page-entity (build-page-entity properties file page-name page ref-tags options)
           namespace-pages (let [page (:block/original-name page-entity)]
                             (when (text/namespace-page? page)
                               (->> (gp-util/split-namespace-pages page)
@@ -136,16 +139,16 @@
       (log/error :exception e))))
 
 (defn extract-blocks-pages
-  [file content {:keys [user-config] :as options}]
+  [file content {:keys [user-config verbose] :or {verbose true} :as options}]
   (if (string/blank? content)
     []
     (let [format (gp-util/get-format file)
-          _ (println "Parsing start: " file)
+          _ (when verbose (println "Parsing start: " file))
           ast (gp-mldoc/->edn content (gp-mldoc/default-config format
                                                          ;; {:parse_outline_only? true}
                                                          )
                            user-config)]
-      (println "Parsing finished : " file)
+      (when verbose (println "Parsing finished: " file))
       (let [first-block (ffirst ast)
             properties (let [properties (and (gp-property/properties-ast? first-block)
                                              (->> (last first-block)

deps/graph-parser/src/logseq/graph_parser/mldoc.cljc

@@ -1,4 +1,4 @@
-(ns ^:nbb-compatible logseq.graph-parser.mldoc
+(ns logseq.graph-parser.mldoc
   ;; Disable clj linters since we don't support clj
   #?(:clj {:clj-kondo/config {:linters {:unresolved-namespace {:level :off}
                                         :unresolved-symbol {:level :off}}}})

deps/graph-parser/src/logseq/graph_parser/test/docs_graph_helper.cljs

@@ -1,18 +1,14 @@
-(ns ^:nbb-compatible logseq.graph-parser.test.docs-graph-helper
+(ns logseq.graph-parser.test.docs-graph-helper
   "Helper fns for setting up and running tests against docs graph"
   (:require ["fs" :as fs]
             ["child_process" :as child-process]
             [cljs.test :refer [is testing]]
             [clojure.string :as string]
+            [logseq.graph-parser.config :as gp-config]
             [datascript.core :as d]))
 
 ;; Helper fns for test setup
 ;; =========================
-(defn slurp
-  "Like clojure.core/slurp"
-  [file]
-  (str (fs/readFileSync file)))
-
 (defn- sh
   "Run shell cmd synchronously and print to inherited streams by default. Aims
     to be similar to babashka.tasks/shell"
@@ -21,15 +17,6 @@
                            (clj->js (rest cmd))
                            (clj->js (merge {:stdio "inherit"} opts))))
 
-(defn build-graph-files
-  [dir]
-  (let [files (->> (str (.-stdout (sh ["git" "ls-files"]
-                                      {:cwd dir :stdio nil})))
-                   string/split-lines
-                   (filter #(re-find #"^(pages|journals)" %))
-                   (map #(str dir "/" %)))]
-    (mapv #(hash-map :file/path % :file/content (slurp %)) files)))
-
 (defn clone-docs-repo-if-not-exists
   [dir]
   (when-not (.existsSync fs dir)
@@ -64,42 +51,29 @@
        (apply merge-with +)
        (into {})))
 
-(defn docs-graph-assertions
-  "These are common assertions that should pass in both graph-parser and main
-  logseq app. It is important to run these in both contexts to ensure that the
-  functionality in frontend.handler.repo and logseq.graph-parser remain the
-  same"
-  [db files]
-  ;; Counts assertions help check for no major regressions. These counts should
-  ;; only increase over time as the docs graph rarely has deletions
-  (testing "Counts"
-    (is (= 206 (count files)) "Correct file count")
-    (is (= 40888 (count (d/datoms db :eavt))) "Correct datoms count")
-
-    (is (= 3597
-           (ffirst
-            (d/q '[:find (count ?b)
-                   :where [?b :block/path-refs ?bp] [?bp :block/name]] db)))
-        "Correct referenced blocks count")
-    (is (= 21
-           (ffirst
-            (d/q '[:find (count ?b)
-                   :where [?b :block/content ?content]
-                   [(clojure.string/includes? ?content "+BEGIN_QUERY")]]
-                 db)))
-        "Advanced query count"))
+(defn- get-block-format-counts
+  [db]
+  (->> (d/q '[:find (pull ?b [*]) :where [?b :block/format]] db)
+       (map first)
+       (group-by :block/format)
+       (map (fn [[k v]] [k (count v)]))
+       (into {})))
 
+(defn- query-assertions
+  [db files]
   (testing "Query based stats"
-    (is (= (set (map :file/path files))
+    (is (= (->> files
+                ;; logseq files aren't saved under :block/file
+                (remove #(string/includes? % (str "/" gp-config/app-name "/")))
+                set)
            (->> (d/q '[:find (pull ?b [* {:block/file [:file/path]}])
                        :where [?b :block/name] [?b :block/file]]
                      db)
                 (map (comp #(get-in % [:block/file :file/path]) first))
                 set))
-        "Journal and pages files on disk should equal ones in db")
+        "Files on disk should equal ones in db")
 
-    (is (= (count (filter #(re-find #"journals/" (:file/path %))
-                          files))
+    (is (= (count (filter #(re-find #"journals/" %) files))
            (->> (d/q '[:find (count ?b)
                        :where
                        [?b :block/journal? true]
@@ -118,12 +92,8 @@
                 (into {})))
         "Task marker counts")
 
-    (is (= {:markdown 3140 :org 460}
-           (->> (d/q '[:find (pull ?b [*]) :where [?b :block/format]] db)
-                (map first)
-                (group-by :block/format)
-                (map (fn [[k v]] [k (count v)]))
-                (into {})))
+    (is (= {:markdown 3143 :org 460}
+           (get-block-format-counts db))
         "Block format counts")
 
     (is (= {:title 98 :id 98
@@ -145,7 +115,7 @@
             :block/priority 4
             :block/deadline 1
             :block/collapsed? 22
-            :block/heading-level 57
+            :block/heading-level 60
             :block/repeated? 1}
            (->> [:block/scheduled :block/priority :block/deadline :block/collapsed?
                  :block/heading-level :block/repeated?]
@@ -161,3 +131,30 @@
                 (map (comp :block/original-name first))
                 set))
         "Has correct namespaces")))
+
+(defn docs-graph-assertions
+  "These are common assertions that should pass in both graph-parser and main
+  logseq app. It is important to run these in both contexts to ensure that the
+  functionality in frontend.handler.repo and logseq.graph-parser remain the
+  same"
+  [db files]
+  ;; Counts assertions help check for no major regressions. These counts should
+  ;; only increase over time as the docs graph rarely has deletions
+  (testing "Counts"
+    (is (= 211 (count files)) "Correct file count")
+    (is (= 40943 (count (d/datoms db :eavt))) "Correct datoms count")
+
+    (is (= 3600
+           (ffirst
+            (d/q '[:find (count ?b)
+                   :where [?b :block/path-refs ?bp] [?bp :block/name]] db)))
+        "Correct referenced blocks count")
+    (is (= 21
+           (ffirst
+            (d/q '[:find (count ?b)
+                   :where [?b :block/content ?content]
+                   [(clojure.string/includes? ?content "+BEGIN_QUERY")]]
+                 db)))
+        "Advanced query count"))
+
+  (query-assertions db files))

deps/graph-parser/src/logseq/graph_parser/text.cljs

@@ -1,4 +1,4 @@
-(ns ^:nbb-compatible logseq.graph-parser.text
+(ns logseq.graph-parser.text
   (:require ["path" :as path]
             [goog.string :as gstring]
             [clojure.string :as string]

deps/graph-parser/src/logseq/graph_parser/utf8.cljs

@@ -1,4 +1,4 @@
-(ns ^:nbb-compatible logseq.graph-parser.utf8)
+(ns logseq.graph-parser.utf8)
 
 (defonce encoder
   (js/TextEncoder. "utf-8"))

deps/graph-parser/src/logseq/graph_parser/util.cljs

@@ -1,4 +1,4 @@
-(ns ^:nbb-compatible logseq.graph-parser.util
+(ns logseq.graph-parser.util
   "Util fns shared between graph-parser and rest of app. Util fns only rely on
   clojure standard libraries."
   (:require [clojure.walk :as walk]

deps/graph-parser/test/logseq/graph_parser_test.cljs → deps/graph-parser/test/logseq/graph_parser/cli_test.cljs

@@ -1,17 +1,13 @@
-(ns logseq.graph-parser-test
+(ns logseq.graph-parser.cli-test
   (:require [cljs.test :refer [deftest]]
-            [logseq.graph-parser :as graph-parser]
-            [logseq.graph-parser.db :as gp-db]
+            [logseq.graph-parser.cli :as gp-cli]
             [logseq.graph-parser.test.docs-graph-helper :as docs-graph-helper]))
 
-
 ;; Integration test that test parsing a large graph like docs
-(deftest ^:integration parse-and-load-files-to-db
+(deftest ^:integration parse-graph
   (let [graph-dir "test/docs"
         _ (docs-graph-helper/clone-docs-repo-if-not-exists graph-dir)
-        files (docs-graph-helper/build-graph-files graph-dir)
-        conn (gp-db/start-conn)
-        _ (graph-parser/parse conn files)
+        {:keys [conn files]} (gp-cli/parse-graph graph-dir)
         db @conn]
 
     (docs-graph-helper/docs-graph-assertions db files)))

deps/graph-parser/test/logseq/graph_parser/mldoc_test.cljs

@@ -2,6 +2,7 @@
   (:require [logseq.graph-parser.mldoc :as gp-mldoc]
             [clojure.string :as string]
             [logseq.graph-parser.test.docs-graph-helper :as docs-graph-helper]
+            [logseq.graph-parser.cli :as gp-cli]
             [cljs.test :refer [testing deftest are is]]))
 
 (deftest test-link
@@ -100,7 +101,7 @@
 (deftest ^:integration test->edn
   (let [graph-dir "test/docs"
         _ (docs-graph-helper/clone-docs-repo-if-not-exists graph-dir)
-        files (docs-graph-helper/build-graph-files graph-dir)
+        files (gp-cli/build-graph-files graph-dir)
         asts-by-file (->> files
                           (map (fn [{:file/keys [path content]}]
                                  (let [format (if (string/ends-with? path ".org")
@@ -116,10 +117,10 @@
             "Drawer" 1,
             "Example" 20,
             "Footnote_Definition" 2,
-            "Heading" 3493,
+            "Heading" 3496,
             "Hiccup" 15,
-            "List" 36,
-            "Paragraph" 411,
+            "List" 37,
+            "Paragraph" 417,
             "Properties" 104,
             "Property_Drawer" 188,
             "Quote" 9,

deps/graph-parser/test/logseq/graph_parser/nbb_test_runner.cljs

@@ -6,7 +6,7 @@
             [logseq.graph-parser.block-test]
             [logseq.graph-parser.property-test]
             [logseq.graph-parser.extract-test]
-            [logseq.graph-parser-test]))
+            [logseq.graph-parser.cli-test]))
 
 (defmethod cljs.test/report [:cljs.test/default :end-run-tests] [m]
   (when-not (cljs.test/successful? m)
@@ -19,4 +19,4 @@
                'logseq.graph-parser.property-test
                'logseq.graph-parser.block-test
                'logseq.graph-parser.extract-test
-               'logseq.graph-parser-test))
+               'logseq.graph-parser.cli-test))

docs/dev-practices.md

@@ -62,7 +62,7 @@ scripts/lint_rules.clj
 
 Namespaces have the metadata flag `^:nbb-compatible` indicate they are compatible with https://github.com/logseq/nbb-logseq. This compatibility is necessary in order for namespaces to be reused by the frontend and CLIs. To confirm these compatibilities, run:
 ```
-bb test:load-nbb-compatible-namespaces
+bb test:load-namespaces-with-nbb
 ```
 
 ## Testing

scripts/large_vars.clj

@@ -5,12 +5,13 @@
   the team to maintain and understand them."
   (:require [babashka.pods :as pods]
             [clojure.pprint :as pprint]
+            [clojure.edn :as edn]
             [clojure.set :as set]))
 
 (pods/load-pod 'clj-kondo/clj-kondo "2021.12.19")
 (require '[pod.borkdude.clj-kondo :as clj-kondo])
 
-(def config
+(def default-config
   ;; TODO: Discuss with team and agree on lower number
   {:max-lines-count 100
    ;; Vars with these metadata flags are allowed. Name should indicate the reason
@@ -23,7 +24,9 @@
 
 (defn -main
   [args]
-  (let [paths (or args ["src"])
+  (let [paths [(or (first args) "src")]
+        config (or (some->> (second args) edn/read-string (merge default-config))
+                   default-config)
         {{:keys [var-definitions]} :analysis}
         (clj-kondo/run!
          {:lint paths
@@ -37,6 +40,8 @@
                               {:var (:name m)
                                :lines-count lines-count
                                :filename (:filename m)}))))
+                  ;; cljc ones repeat
+                  distinct
                   (sort-by :lines-count (fn [x y] (compare y x))))]
     (if (seq vars)
       (do

scripts/src/logseq/tasks/nbb.clj

@@ -1,15 +1,16 @@
 (ns logseq.tasks.nbb
   (:require [pod.borkdude.clj-kondo :as clj-kondo]
+            [clojure.string :as str]
             [babashka.tasks :refer [shell]]))
 
 (defn- fetch-meta-namespaces
   "Return namespaces with metadata"
   [paths]
-  (let [paths (or (seq paths) ["src"])
-        {{:keys [namespace-definitions]} :analysis}
+  (let [{{:keys [namespace-definitions]} :analysis}
         (clj-kondo/run!
          {:lint paths
-          :config {:output {:analysis {:namespace-definitions {:meta true}}}}})
+          :config {:output {:analysis {:namespace-definitions {:meta true
+                                                               :lang :cljs}}}}})
         matches (keep (fn [m]
                         (when (:meta m)
                           {:ns   (:name m)
@@ -17,14 +18,29 @@
                       namespace-definitions)]
     matches))
 
+(defn- validate-namespaces
+  [namespaces classpath dir]
+  (assert (seq namespaces) "There must be some namespaces to check")
+  (doseq [n namespaces]
+    (println "Requiring" n "...")
+    (shell {:dir dir} "yarn nbb-logseq -cp" classpath "-e" (format "(require '[%s])" n)))
+  (println "Success!"))
+
 (defn load-compatible-namespaces
   "Check nbb-compatible namespaces can be required by nbb-logseq"
   []
   (let [namespaces (map :ns
                         (filter #(get-in % [:meta :nbb-compatible])
                                 (fetch-meta-namespaces ["src/main"])))]
-    (assert (seq namespaces) "There must be some nbb namespaces to check")
-    (doseq [n namespaces]
-      (println "Requiring" n "...")
-      (shell "yarn nbb-logseq -cp src/main -e" (format "(require '[%s])" n)))
-    (println "Success!")))
+    (validate-namespaces namespaces "src/main" ".")))
+
+(defn load-all-namespaces
+  "Check all namespaces in source path(s) can be required by nbb-logseq"
+  [dir & paths]
+  (let [{{:keys [namespace-definitions]} :analysis}
+        (clj-kondo/run!
+         {:lint (map #(str dir "/" %) paths)
+          :config {:output {:analysis {:namespace-definitions {:lang :cljs}}}}})]
+    (validate-namespaces (map :name namespace-definitions)
+                         (str/join ":" paths)
+                         dir)))

src/main/frontend/handler/file.cljs

@@ -94,8 +94,6 @@
                         (db/delete-file-blocks! repo-url file)
                         (when first-page (db/delete-page-blocks repo-url (:block/name first-page))))
                        (distinct))]
-    ;; TODO: Remove
-    (when (seq delete-blocks) (prn :DELETE-BLOCKS (count delete-blocks)))
     (when-let [current-file (page-exists-in-another-file repo-url first-page file)]
       (when (not= file current-file)
         (let [error (str "Page already exists with another file: " current-file ", current file: " file)]

src/main/frontend/handler/repo.cljs

@@ -22,9 +22,8 @@
             [shadow.resource :as rc]
             [frontend.db.persist :as db-persist]
             [logseq.graph-parser.util :as gp-util]
-            [logseq.graph-parser.config :as gp-config]
+            [logseq.graph-parser :as graph-parser]
             [electron.ipc :as ipc]
-            [clojure.set :as set]
             [clojure.core.async :as async]
             [frontend.encrypt :as encrypt]))
 
@@ -215,30 +214,19 @@
 
 (defn- parse-files-and-create-default-files-inner!
   [repo-url files delete-files delete-blocks file-paths db-encrypted? re-render? re-render-opts opts]
-  (let [support-files (filter
-                       (fn [file]
-                         (let [format (gp-util/get-format (:file/path file))]
-                           (contains? (set/union #{:edn :css} gp-config/mldoc-support-formats) format)))
-                       files)
-        support-files (sort-by :file/path support-files)
-        {journals true non-journals false} (group-by (fn [file] (string/includes? (:file/path file) "journals/")) support-files)
-        {built-in true others false} (group-by (fn [file]
-                                                 (or (string/includes? (:file/path file) "contents.")
-                                                     (string/includes? (:file/path file) ".edn")
-                                                     (string/includes? (:file/path file) "custom.css"))) non-journals)
-        support-files' (concat (reverse journals) built-in others)
+  (let [supported-files (graph-parser/filter-files files)
         new-graph? (:new-graph? opts)
         delete-data (->> (concat delete-files delete-blocks)
                          (remove nil?))
-        chan (async/to-chan! support-files')
+        chan (async/to-chan! supported-files)
         graph-added-chan (async/promise-chan)]
     (when (seq delete-data) (db/transact! repo-url delete-data))
     (state/set-current-repo! repo-url)
-    (state/set-parsing-state! {:total (count support-files')})
+    (state/set-parsing-state! {:total (count supported-files)})
     ;; Synchronous for tests for not breaking anything
     (if util/node-test?
       (do
-        (doseq [file support-files']
+        (doseq [file supported-files]
           (state/set-parsing-state! (fn [m]
                                       (assoc m :current-parsing-file (:file/path file))))
           (parse-and-load-file! repo-url file new-graph?))

src/test/frontend/handler/repo_test.cljs

@@ -1,9 +1,9 @@
 (ns frontend.handler.repo-test
-  (:require [cljs.test :refer [deftest use-fixtures is testing]]
+  (:require [cljs.test :refer [deftest use-fixtures]]
             [frontend.handler.repo :as repo-handler]
             [frontend.test.helper :as test-helper]
+            [logseq.graph-parser.cli :as gp-cli]
             [logseq.graph-parser.test.docs-graph-helper :as docs-graph-helper]
-            [datascript.core :as d]
             [frontend.db.conn :as conn]))
 
 (use-fixtures :each {:before test-helper/start-test-db!
@@ -13,18 +13,8 @@
 (deftest ^:integration parse-and-load-files-to-db
   (let [graph-dir "src/test/docs"
         _ (docs-graph-helper/clone-docs-repo-if-not-exists graph-dir)
-        files (docs-graph-helper/build-graph-files graph-dir)
+        files (gp-cli/build-graph-files graph-dir)
         _ (repo-handler/parse-files-and-load-to-db! test-helper/test-db files {:re-render? false})
         db (conn/get-db test-helper/test-db)]
 
-    #_:clj-kondo/ignore ;; buggy unresolved var
-    (docs-graph-helper/docs-graph-assertions db files)
-
-    (testing "Delete previous file data when re-parsing a file"
-      (repo-handler/parse-files-and-load-to-db! test-helper/test-db
-                                                (filter #(re-find #"pages/tutorial.md" (:file/path %))
-                                                        files)
-                                                {:re-render? false})
-      (is (= 206 (count files)) "Correct file count")
-      (is (= 40888 (count (d/datoms db :eavt))) "Correct datoms count")
-      )))
+    (docs-graph-helper/docs-graph-assertions db (map :file/path files))))