(ns frontend.handler.repo-conversion-test "Repo tests of directory conversion" (:require [cljs.test :refer [deftest use-fixtures is testing]] [clojure.string :as string] [logseq.graph-parser.cli :as gp-cli] [logseq.graph-parser.util :as gp-util] [logseq.graph-parser.test.docs-graph-helper :as docs-graph-helper] [logseq.graph-parser.config :as gp-config] [frontend.test.helper :as test-helper] [frontend.handler.page :as page-handler] [frontend.handler.conversion :as conversion-handler] [frontend.handler.repo :as repo-handler] [frontend.db.conn :as conn] [datascript.core :as d])) (use-fixtures :each {:before test-helper/start-test-db! :after test-helper/destroy-test-db!}) (defn- query-assertions-v067 [db graph-dir files] (testing "Query based stats" (is (= (->> files ;; logseq files aren't saved under :block/file (remove #(string/includes? % (str graph-dir "/" gp-config/app-name "/"))) set) (->> (d/q '[:find (pull ?b [* {:block/file [:file/path]}]) :where [?b :block/name] [?b :block/file]] db) (map (comp #(get-in % [:block/file :file/path]) first)) set)) "Files on disk should equal ones in db") (is (= (count (filter #(re-find #"journals/" %) files)) (->> (d/q '[:find (count ?b) :where [?b :block/journal? true] [?b :block/name] [?b :block/file]] db) ffirst)) "Journal page count on disk equals count in db") (is (= {"CANCELED" 2 "DONE" 6 "LATER" 4 "NOW" 5} (->> (d/q '[:find (pull ?b [*]) :where [?b :block/marker]] db) (map first) (group-by :block/marker) (map (fn [[k v]] [k (count v)])) (into {}))) "Task marker counts") (is (= {:markdown 3141 :org 460} (docs-graph-helper/get-block-format-counts db)) "Block format counts") (is (= {:title 98 :id 98 :updated-at 47 :created-at 47 :card-last-score 6 :card-repeats 6 :card-next-schedule 6 :card-last-interval 6 :card-ease-factor 6 :card-last-reviewed 6 :alias 6 :logseq.macro-arguments 94 :logseq.macro-name 94 :heading 64} (docs-graph-helper/get-top-block-properties db)) "Counts for top block properties") (is (= {:title 98 :alias 6 :tags 3 :permalink 2 :name 1 :type 1 :related 1 :sample 1 :click 1 :id 1 :example 1} (docs-graph-helper/get-all-page-properties db)) "Counts for all page properties") (is (= {:block/scheduled 2 :block/priority 4 :block/deadline 1 :block/collapsed? 22 :block/repeated? 1} (->> [:block/scheduled :block/priority :block/deadline :block/collapsed? :block/repeated?] (map (fn [attr] [attr (ffirst (d/q [:find (list 'count '?b) :where ['?b attr]] db))])) (into {}))) "Counts for blocks with common block attributes") (is (= #{"term" "setting" "book" "templates" "Query" "Query/table" "page"} (->> (d/q '[:find (pull ?n [*]) :where [?b :block/namespace ?n]] db) (map (comp :block/original-name first)) set)) "Has correct namespaces"))) (defn docs-graph-assertions-v067 "These are common assertions that should pass in both graph-parser and main logseq app. It is important to run these in both contexts to ensure that the functionality in frontend.handler.repo and logseq.graph-parser remain the same" [db graph-dir files] ;; Counts assertions help check for no major regressions. These counts should ;; only increase over time as the docs graph rarely has deletions (testing "Counts" (is (= 211 (count files)) "Correct file count") (is (= 42200 (count (d/datoms db :eavt))) "Correct datoms count") (is (= 3600 (ffirst (d/q '[:find (count ?b) :where [?b :block/path-refs ?bp] [?bp :block/name]] db))) "Correct referenced blocks count") (is (= 21 (ffirst (d/q '[:find (count ?b) :where [?b :block/content ?content] [(clojure.string/includes? ?content "+BEGIN_QUERY")]] db))) "Advanced query count")) (query-assertions-v067 db graph-dir files)) (defn- convert-to-triple-lowbar [path] (let [original-body (gp-util/path->file-body path) ;; only test file name parsing, don't consider title prop overriding rename-target (:target (#'conversion-handler/calc-rename-target-impl :legacy :triple-lowbar original-body nil))] (if rename-target #_:clj-kondo/ignore (do #_(prn "conversion triple-lowbar: " original-body " -> " rename-target) (#'page-handler/compute-new-file-path path rename-target)) path))) (defn- convert-graph-files-path "Given a list of files, converts them according to the given conversion function" [files conversion-fn] (map (fn [file] (assoc file :file/path (conversion-fn (:file/path file)))) files)) ;; Integration test that test parsing a large graph like docs ;; Check if file name conversion from old version of docs is working (deftest ^:integration convert-v067-filenames-parse-and-load-files-to-db (let [graph-dir "src/test/docs" _ (docs-graph-helper/clone-docs-repo-if-not-exists graph-dir "v0.6.7") files (#'gp-cli/build-graph-files graph-dir {}) ;; Converting the v0.6.7 ver docs graph under the old namespace naming rule to the new one (:repo/dir-version 0->3) files (convert-graph-files-path files convert-to-triple-lowbar) _ (repo-handler/parse-files-and-load-to-db! test-helper/test-db files {:re-render? false :verbose false}) db (conn/get-db test-helper/test-db)] ;; Result under new naming rule after conversion should be the same as the old one (docs-graph-assertions-v067 db graph-dir (map :file/path files))))