(ns frontend.db.model "Core db functions." ;; TODO: Remove this config once how repos are passed to this ns are standardized {:clj-kondo/config {:linters {:unused-binding {:level :off}}}} (:require [clojure.set :as set] [clojure.string :as string] [clojure.walk :as walk] [datascript.core :as d] [frontend.config :as config] [frontend.date :as date] [logseq.db.schema :as db-schema] [frontend.db.conn :as conn] [frontend.db.react :as react] [frontend.db.utils :as db-utils] [frontend.state :as state] [frontend.util :as util :refer [react]] [logseq.graph-parser.util :as gp-util] [logseq.graph-parser.text :as text] [logseq.db.rules :refer [rules]] [logseq.db.default :as default-db] [frontend.util.drawer :as drawer])) ;; lazy loading (def initial-blocks-length 50) (def step-loading-blocks 25) ;; TODO: extract to specific models and move data transform logic to the ;; corresponding handlers. ;; Use it as an input argument for datalog queries (def block-attrs '[:db/id :block/uuid :block/parent :block/left :block/collapsed? :block/format :block/refs :block/_refs :block/path-refs :block/tags :block/content :block/marker :block/priority :block/properties :block/pre-block? :block/scheduled :block/deadline :block/repeated? :block/created-at :block/updated-at :block/file :block/heading-level {:block/page [:db/id :block/name :block/original-name :block/journal-day]} {:block/_parent ...}]) (defn pull-block [id] (when-let [repo (state/get-current-repo)] (-> (react/q repo [:frontend.db.react/block id] {:query-fn (fn [_] (db-utils/pull (butlast block-attrs) id))} nil) react))) (defn get-original-name [page-entity] (or (:block/original-name page-entity) (:block/name page-entity))) (defn get-tag-pages [repo tag-name] (when tag-name (d/q '[:find ?original-name ?name :in $ ?tag :where [?e :block/name ?tag] [?page :block/tags ?e] [?page :block/original-name ?original-name] [?page :block/name ?name]] (conn/get-db repo) (util/page-name-sanity-lc tag-name)))) (defn get-all-tagged-pages [repo] (d/q '[:find ?page-name ?tag :where [?page :block/tags ?e] [?e :block/name ?tag] [?page :block/name ?page-name]] (conn/get-db repo))) (defn get-all-namespace-relation [repo] (d/q '[:find ?page-name ?parent :where [?page :block/name ?page-name] [?page :block/namespace ?e] [?e :block/name ?parent]] (conn/get-db repo))) (defn get-pages [repo] (->> (d/q '[:find ?page-original-name :where [?page :block/name ?page-name] [(get-else $ ?page :block/original-name ?page-name) ?page-original-name]] (conn/get-db repo)) (map first))) (defn get-all-pages [repo] (d/q '[:find [(pull ?page [*]) ...] :where [?page :block/name]] (conn/get-db repo))) (defn get-page-alias [repo page-name] (when-let [db (and repo (conn/get-db repo))] (some->> (d/q '[:find ?alias :in $ ?page-name :where [?page :block/name ?page-name] [?page :block/alias ?alias]] db (util/page-name-sanity-lc page-name)) db-utils/seq-flatten distinct))) (defn get-alias-source-page [repo alias] (when-let [db (and repo (conn/get-db repo))] (let [alias (util/page-name-sanity-lc alias) pages (->> (d/q '[:find (pull ?p [*]) :in $ ?alias :where [?a :block/name ?alias] [?p :block/alias ?a]] db alias) (db-utils/seq-flatten))] (when (seq pages) (some (fn [page] (let [aliases (->> (get-in page [:block/properties :alias]) (map util/page-name-sanity-lc) set)] (when (contains? aliases alias) page))) pages))))) (defn get-files [repo] (when-let [db (conn/get-db repo)] (->> (d/q '[:find ?path ;; ?modified-at :where [?file :file/path ?path] ;; [?file :file/last-modified-at ?modified-at] ] db) (seq) ;; (sort-by last) (reverse)))) (defn get-files-v2 [repo] (when-let [db (conn/get-db repo)] (->> (d/q '[:find ?file ?path ;; ?modified-at :where [?file :file/path ?path] ;; [?file :file/last-modified-at ?modified-at] ] db) (seq) ;; (sort-by last) (reverse)))) (defn get-files-blocks [repo-url paths] (let [paths (set paths) pred (fn [_db e] (contains? paths e))] (-> (d/q '[:find ?block :in $ ?pred :where [?file :file/path ?path] [(?pred $ ?path)] [?p :block/file ?file] [?block :block/page ?p]] (conn/get-db repo-url) pred) db-utils/seq-flatten))) (defn get-file-blocks [repo-url path] (-> (d/q '[:find ?block :in $ ?path :where [?file :file/path ?path] [?p :block/file ?file] [?block :block/page ?p]] (conn/get-db repo-url) path) db-utils/seq-flatten)) (defn set-file-last-modified-at! [repo path last-modified-at] (when (and repo path last-modified-at) (when-let [conn (conn/get-db repo false)] (d/transact! conn [{:file/path path :file/last-modified-at last-modified-at}] {:skip-refresh? true})))) (defn get-file-last-modified-at [repo path] (when (and repo path) (when-let [db (conn/get-db repo)] (-> (d/entity db [:file/path path]) :file/last-modified-at)))) (defn file-exists? [repo path] (when (and repo path) (when-let [db (conn/get-db repo)] (d/entity db [:file/path path])))) (defn get-files-full [repo] (when-let [db (conn/get-db repo)] (->> (d/q '[:find (pull ?file [*]) :where [?file :file/path]] db) (flatten)))) (defn get-file ([path] (get-file (state/get-current-repo) path)) ([repo path] (when (and repo path) (when-let [db (conn/get-db repo)] (:file/content (d/entity db [:file/path path])))))) (defn get-custom-css [] (when-let [repo (state/get-current-repo)] (get-file (config/get-file-path repo "logseq/custom.css")))) (defn get-block-by-uuid [id] (db-utils/entity [:block/uuid (if (uuid? id) id (uuid id))])) (defn query-block-by-uuid [id] (db-utils/pull [:block/uuid (if (uuid? id) id (uuid id))])) (defn get-page-format [page-name] (or (let [page (db-utils/entity [:block/name (util/safe-page-name-sanity-lc page-name)])] (or (:block/format page) (when-let [file (:block/file page)] (when-let [path (:file/path (db-utils/entity (:db/id file)))] (gp-util/get-format path))))) (state/get-preferred-format) :markdown)) (defn page-alias-set [repo-url page] (when-let [page-id (:db/id (db-utils/entity repo-url [:block/name (util/safe-page-name-sanity-lc page)]))] (->> (d/q '[:find ?e :in $ ?page-name % :where [?page :block/name ?page-name] (alias ?page ?e)] (conn/get-db repo-url) (util/safe-page-name-sanity-lc page) '[[(alias ?e2 ?e1) [?e2 :block/alias ?e1]] [(alias ?e2 ?e1) [?e1 :block/alias ?e2]] [(alias ?e1 ?e3) [?e1 :block/alias ?e2] [?e2 :block/alias ?e3]] [(alias ?e3 ?e1) [?e1 :block/alias ?e2] [?e2 :block/alias ?e3]]]) db-utils/seq-flatten (set) (set/union #{page-id})))) (defn get-entities-by-ids ([ids] (get-entities-by-ids (state/get-current-repo) ids)) ([repo ids] (when repo (db-utils/pull-many repo '[*] ids)))) (defn get-page-names-by-ids ([ids] (get-page-names-by-ids (state/get-current-repo) ids)) ([repo ids] (when repo (->> (db-utils/pull-many repo '[:block/name] ids) (map :block/name))))) (defn get-page-alias-names [repo page-name] (let [alias-ids (page-alias-set repo page-name)] (when (seq alias-ids) (let [names (->> (get-page-names-by-ids repo alias-ids) distinct (remove #(= (util/page-name-sanity-lc %) (util/page-name-sanity-lc page-name)))) lookup-refs (map (fn [name] [:block/name (util/page-name-sanity-lc name)]) names)] (->> (db-utils/pull-many repo '[:block/name :block/original-name] lookup-refs) (map (fn [m] (or (:block/original-name m) (:block/name m))))))))) (defn with-pages [blocks] (let [pages-ids (->> (map (comp :db/id :block/page) blocks) (remove nil?)) pages (when (seq pages-ids) (db-utils/pull-many '[:db/id :block/name :block/original-name :block/journal-day] pages-ids)) pages-map (reduce (fn [acc p] (assoc acc (:db/id p) p)) {} pages) blocks (map (fn [block] (assoc block :block/page (get pages-map (:db/id (:block/page block))))) blocks)] blocks)) (defn get-page-properties [page] (when-let [page (db-utils/entity [:block/name (util/safe-page-name-sanity-lc page)])] (:block/properties page))) ;; FIXME: alert (defn sort-by-left ([blocks parent] (sort-by-left blocks parent {:check? true})) ([blocks parent {:keys [check?]}] (let [blocks (util/distinct-by :db/id blocks)] (when check? (when (not= (count blocks) (count (set (map :block/left blocks)))) (let [duplicates (->> (map (comp :db/id :block/left) blocks) frequencies (filter (fn [[_k v]] (> v 1))) (map (fn [[k _v]] (let [left (db-utils/pull k)] {:left left :duplicates (->> (filter (fn [block] (= k (:db/id (:block/left block)))) blocks) (map #(select-keys % [:db/id :block/level :block/content :block/file])))}))))] (util/pprint duplicates))) (assert (= (count blocks) (count (set (map :block/left blocks)))) "Each block should have a different left node")) (let [left->blocks (reduce (fn [acc b] (assoc acc (:db/id (:block/left b)) b)) {} blocks)] (loop [block parent result []] (if-let [next (get left->blocks (:db/id block))] (recur next (conj result next)) (vec result))))))) (defn try-sort-by-left [blocks parent] (let [result' (sort-by-left blocks parent {:check? false})] (if (= (count result') (count blocks)) result' blocks))) (defn sort-by-left-recursive [form] (walk/postwalk (fn [f] (if (and (map? f) (:block/_parent f)) (let [children (:block/_parent f)] (-> f (dissoc :block/_parent) (assoc :block/children (sort-by-left children f)))) f)) form)) (defn get-sorted-page-block-ids [page-id] (let [root (db-utils/entity page-id)] (loop [result [] children (sort-by-left (:block/_parent root) root)] (if (seq children) (let [child (first children)] (recur (conj result (:db/id child)) (concat (sort-by-left (:block/_parent child) child) (rest children)))) result)))) (defn sort-page-random-blocks "Blocks could be non consecutive." [blocks] (assert (every? #(= (:block/page %) (:block/page (first blocks))) blocks) "Blocks must to be in a same page.") (let [page-id (:db/id (:block/page (first blocks))) ;; TODO: there's no need to sort all the blocks sorted-ids (get-sorted-page-block-ids page-id) blocks-map (zipmap (map :db/id blocks) blocks)] (keep blocks-map sorted-ids))) (defn has-children? ([block-id] (has-children? (conn/get-db) block-id)) ([db block-id] (some? (:block/_parent (d/entity db [:block/uuid block-id]))))) (defn- collapsed-and-has-children? [db block] (and (:block/collapsed? block) (has-children? db (:block/uuid block)))) (defn get-by-parent-&-left [db parent-id left-id] (when (and parent-id left-id) (let [lefts (:block/_left (d/entity db left-id))] (some (fn [node] (when (and (= parent-id (:db/id (:block/parent node))) (not= parent-id (:db/id node))) node)) lefts)))) (defn- get-next-outdented-block "Get the next outdented block of the block that has the `id`. e.g. - a - b - c - d The next outdented block of `c` is `d`." [db id] (when-let [block (d/entity db id)] (let [parent (:block/parent block)] (if-let [parent-sibling (get-by-parent-&-left db (:db/id (:block/parent parent)) (:db/id parent))] parent-sibling (get-next-outdented-block db (:db/id parent)))))) (defn top-block? [block] (= (:db/id (:block/parent block)) (:db/id (:block/page block)))) (defn get-block-parent ([block-id] (get-block-parent (state/get-current-repo) block-id)) ([repo block-id] (when-let [db (conn/get-db repo)] (when-let [block (d/entity db [:block/uuid block-id])] (:block/parent block))))) ;; non recursive query (defn get-block-parents ([repo block-id] (get-block-parents repo block-id 100)) ([repo block-id depth] (loop [block-id block-id parents (list) d 1] (if (> d depth) parents (if-let [parent (get-block-parent repo block-id)] (recur (:block/uuid parent) (conj parents parent) (inc d)) parents))))) ;; Use built-in recursive (defn get-block-parents-v2 [repo block-id] (d/pull (conn/get-db repo) '[:db/id :block/collapsed? :block/properties {:block/parent ...}] [:block/uuid block-id])) (defn get-next-open-block ([db block] (get-next-open-block db block nil)) ([db block scoped-block-id] (let [block-id (:db/id block) block-parent-id (:db/id (:block/parent block)) next-block (or (if (and (collapsed-and-has-children? db block) (not= block-id scoped-block-id)) ; skips children ;; Sibling (get-by-parent-&-left db block-parent-id block-id) (or ;; Child (get-by-parent-&-left db block-id block-id) ;; Sibling (get-by-parent-&-left db block-parent-id block-id))) ;; Next outdented block (get-next-outdented-block db block-id))] (if (and scoped-block-id next-block) (let [parents (->> (get-block-parents (state/get-current-repo) (:block/uuid next-block)) (map :db/id) (set))] (when (contains? parents scoped-block-id) next-block)) next-block)))) (defn get-paginated-blocks-no-cache "Result should be sorted." [db start-id {:keys [limit include-start? scoped-block-id end-id]}] (when-let [start (d/entity db start-id)] (let [scoped-block-parents (when scoped-block-id (let [block (d/entity db scoped-block-id)] (->> (get-block-parents (state/get-current-repo) (:block/uuid block)) (map :db/id) (set)))) result (loop [block start result []] (if (and limit (>= (count result) limit)) result (let [next-block (get-next-open-block db block scoped-block-id)] (if next-block (cond (and (seq scoped-block-parents) (contains? scoped-block-parents (:db/id (:block/parent next-block)))) result (and end-id (= end-id (:db/id next-block))) (conj result next-block) :else (recur next-block (conj result next-block))) result))))] (if include-start? (cons start result) result)))) (defn get-block-last-direct-child "Notice: if `not-collapsed?` is true, will skip searching for any collapsed block." ([db db-id] (get-block-last-direct-child db db-id true)) ([db db-id not-collapsed?] (when-let [block (d/entity db db-id)] (when (if not-collapsed? (not (collapsed-and-has-children? db block)) true) (let [children (:block/_parent block) all-left (set (concat (map (comp :db/id :block/left) children) [db-id])) all-ids (set (map :db/id children))] (first (set/difference all-ids all-left))))))) (defn get-block-last-child [db db-id] (let [last-child (get-block-last-direct-child db db-id)] (loop [prev last-child last-child last-child] (if last-child (recur last-child (get-block-last-direct-child db last-child)) prev)))) (defn get-prev-open-block [db id] (let [block (d/entity db id) left (:block/left block) left-id (:db/id left)] (if (= (:db/id left) (:db/id (:block/parent block))) left-id (if (util/collapsed? left) left-id (or (get-block-last-child db (:db/id left)) left-id))))) (defn recursive-child? [repo child-id parent-id] (let [*last-node (atom nil)] (loop [node (db-utils/entity repo child-id)] (when-not (= @*last-node node) (reset! *last-node node) (if node (let [parent (:block/parent node)] (if (= (:db/id parent) parent-id) true (recur parent))) false))))) (defn get-prev-sibling [db id] (when-let [e (d/entity db id)] (let [left (:block/left e)] (when (not= (:db/id left) (:db/id (:block/parent e))) left)))) (defn get-right-sibling [db db-id] (when-let [block (d/entity db db-id)] (get-by-parent-&-left db (:db/id (:block/parent block)) db-id))) (defn last-child-block? "The child block could be collapsed." [db parent-id child-id] (when-let [child (d/entity db child-id)] (cond (= parent-id child-id) true (get-right-sibling db child-id) false :else (last-child-block? db parent-id (:db/id (:block/parent child)))))) (defn- consecutive-block? [block-1 block-2] (let [db (conn/get-db) aux-fn (fn [block-1 block-2] (and (= (:block/page block-1) (:block/page block-2)) (or ;; sibling or child (= (:db/id (:block/left block-2)) (:db/id block-1)) (when-let [prev-sibling (get-prev-sibling db (:db/id block-2))] (last-child-block? db (:db/id prev-sibling) (:db/id block-1))))))] (or (aux-fn block-1 block-2) (aux-fn block-2 block-1)))) (defn get-non-consecutive-blocks [blocks] (vec (keep-indexed (fn [i _block] (when (< (inc i) (count blocks)) (when-not (consecutive-block? (nth blocks i) (nth blocks (inc i))) (nth blocks i)))) blocks))) (defn- get-start-id-for-pagination-query [repo-url current-db {:keys [db-before tx-meta] :as tx-report} result outliner-op page-id block-id tx-block-ids] (let [db-before (or db-before current-db) cached-ids (map :db/id @result) cached-ids-set (set (conj cached-ids page-id)) first-changed-id (cond (= (:real-outliner-op tx-meta) :indent-outdent) (if (state/logical-outdenting?) (first (:move-blocks tx-meta)) (last (:move-blocks tx-meta))) (= outliner-op :move-blocks) (let [{:keys [move-blocks target from-page to-page]} tx-meta] (cond (= page-id target) ; move to the first block nil (and from-page to-page (not= from-page to-page)) (if (= page-id from-page) (first move-blocks) target) :else ;; same page, get the most top block before dragging (let [match-ids (set (conj move-blocks target))] (loop [[id & others] cached-ids] (if id (if (contains? match-ids id) id (when (seq others) (recur others))) nil))))) :else (let [insert? (= :insert-blocks outliner-op)] (some #(when (and (or (and insert? (not (contains? cached-ids-set %))) true) (recursive-child? repo-url % block-id)) %) tx-block-ids)))] (when first-changed-id (or (get-prev-open-block db-before first-changed-id) (get-prev-open-block current-db first-changed-id))))) (defn- build-paginated-blocks-from-cache "Notice: tx-report could be nil." [repo-url tx-report result outliner-op page-id block-id tx-block-ids scoped-block-id] (let [{:keys [tx-meta]} tx-report current-db (conn/get-db repo-url)] (cond (and (or (:undo? tx-meta) (:redo? tx-meta)) @result) (let [blocks-range (:pagination-blocks-range tx-meta) [start-block-id end-block-id] (:new blocks-range)] (get-paginated-blocks-no-cache current-db start-block-id {:end-id end-block-id :include-start? true :scoped-block-id scoped-block-id})) (contains? #{:save-block :delete-blocks} outliner-op) @result (contains? #{:insert-blocks :collapse-expand-blocks :move-blocks} outliner-op) (when-let [start-id (get-start-id-for-pagination-query repo-url current-db tx-report result outliner-op page-id block-id tx-block-ids)] (let [start-page? (:block/name (db-utils/entity start-id))] (when-not start-page? (let [previous-blocks (take-while (fn [b] (not= start-id (:db/id b))) @result) limit (-> (max (- initial-blocks-length (count previous-blocks)) (count tx-block-ids)) (+ 25)) more (get-paginated-blocks-no-cache current-db start-id {:limit limit :include-start? true :scoped-block-id scoped-block-id})] (concat previous-blocks more))))) :else nil))) (defn get-paginated-blocks "Get paginated blocks for a page or a specific block. `scoped-block-id`: if specified, returns its children only." ([repo-url block-id] (get-paginated-blocks repo-url block-id {})) ([repo-url block-id {:keys [pull-keys start-block limit use-cache? scoped-block-id] :or {pull-keys '[* :block/_refs] limit initial-blocks-length use-cache? true scoped-block-id nil}}] (when block-id (assert (integer? block-id) (str "wrong block-id: " block-id)) (let [entity (db-utils/entity repo-url block-id) page? (some? (:block/name entity)) page-entity (if page? entity (:block/page entity)) page-id (:db/id page-entity) bare-page-map {:db/id page-id :block/name (:block/name page-entity) :block/original-name (:block/original-name page-entity) :block/journal-day (:block/journal-day page-entity)} query-key (if page? :frontend.db.react/page-blocks :frontend.db.react/block-and-children)] (some-> (react/q repo-url [query-key block-id] {:use-cache? use-cache? :query-fn (fn [db tx-report result] (let [tx-data (:tx-data tx-report) refs (some->> (filter #(= :block/refs (:a %)) tx-data) (map :v)) tx-block-ids (distinct (-> (map :e tx-data) (concat refs))) [tx-id->block cached-id->block] (when (and tx-report result) (let [blocks (->> (db-utils/pull-many repo-url pull-keys tx-block-ids) (remove nil?))] [(zipmap (mapv :db/id blocks) blocks) (zipmap (mapv :db/id @result) @result)])) limit (if (and result @result) (max (+ (count @result) 5) limit) limit) outliner-op (get-in tx-report [:tx-meta :outliner-op]) blocks (build-paginated-blocks-from-cache repo-url tx-report result outliner-op page-id block-id tx-block-ids scoped-block-id) blocks (or blocks (get-paginated-blocks-no-cache (conn/get-db repo-url) block-id {:limit limit :include-start? (not page?) :scoped-block-id scoped-block-id})) block-eids (map :db/id blocks) blocks (if (and (seq tx-id->block) (not (contains? #{:move-blocks} outliner-op))) (map (fn [id] (or (get tx-id->block id) (get cached-id->block id) (db-utils/pull repo-url pull-keys id))) block-eids) (db-utils/pull-many repo-url pull-keys block-eids)) blocks (remove (fn [b] (nil? (:block/content b))) blocks)] (map (fn [b] (assoc b :block/page bare-page-map)) blocks)))} nil) react))))) (defn get-page-blocks-no-cache ([page] (get-page-blocks-no-cache (state/get-current-repo) page nil)) ([repo-url page] (get-page-blocks-no-cache repo-url page nil)) ([repo-url page {:keys [pull-keys] :or {pull-keys '[*]}}] (when page (let [page (util/page-name-sanity-lc page) page-id (:db/id (db-utils/entity repo-url [:block/name page])) db (conn/get-db repo-url)] (when page-id (let [datoms (d/datoms db :avet :block/page page-id) block-eids (mapv :e datoms)] (db-utils/pull-many repo-url pull-keys block-eids))))))) (defn get-page-blocks-count [repo page-id] (when-let [db (conn/get-db repo)] (count (d/datoms db :avet :block/page page-id)))) (defn page-exists? "Whether a page exists." [page-name] (when page-name (db-utils/entity [:block/name (util/page-name-sanity-lc page-name)]))) (defn page-empty? "Whether a page is empty. Does it has a non-page block? `page-id` could be either a string or a db/id." [repo page-id] (when-let [db (conn/get-db repo)] (let [page-id (if (string? page-id) [:block/name (util/safe-page-name-sanity-lc page-id)] page-id) page (d/entity db page-id)] (nil? (:block/_left page))))) (defn page-empty-or-dummy? [repo page-id] (or (page-empty? repo page-id) (when-let [db (conn/get-db repo)] (let [datoms (d/datoms db :avet :block/page page-id)] (and (= (count datoms) 1) (= "" (:block/content (db-utils/pull (:e (first datoms)))))))))) (defn parents-collapsed? [repo block-id] (when-let [block (:block/parent (get-block-parents-v2 repo block-id))] (->> (tree-seq map? (fn [x] [(:block/parent x)]) block) (some util/collapsed?)))) (defn get-block-page [repo block-id] (when-let [block (db-utils/entity repo [:block/uuid block-id])] (db-utils/entity repo (:db/id (:block/page block))))) (defn get-pages-by-name-partition [repo partition] (when-let [db (conn/get-db repo)] (when-not (string/blank? partition) (let [partition (util/page-name-sanity-lc (string/trim partition)) ids (->> (d/datoms db :aevt :block/name) (filter (fn [datom] (let [page (:v datom)] (string/includes? page partition)))) (map :e))] (when (seq ids) (db-utils/pull-many repo '[:db/id :block/name :block/original-name] ids)))))) (defn get-block-children-ids [repo block-uuid] (when-let [db (conn/get-db repo)] (when-let [eid (:db/id (db-utils/entity repo [:block/uuid block-uuid]))] (let [get-children-ids (fn get-children-ids [eid] (mapcat (fn [datom] (let [id (first datom)] (cons (:block/uuid (d/entity db id)) (get-children-ids id)))) (d/datoms db :avet :block/parent eid)))] (get-children-ids eid))))) (defn get-block-immediate-children "Doesn't include nested children." [repo block-uuid] (when-let [db (conn/get-db repo)] (-> (d/q '[:find [(pull ?b [*]) ...] :in $ ?parent-id :where [?parent :block/uuid ?parent-id] [?b :block/parent ?parent]] db block-uuid) (sort-by-left (db-utils/entity [:block/uuid block-uuid]))))) (defn get-block-children "Including nested children." [repo block-uuid] (let [ids (get-block-children-ids repo block-uuid)] (when (seq ids) (let [ids' (map (fn [id] [:block/uuid id]) ids)] (db-utils/pull-many repo '[*] ids'))))) ;; TODO: use the tree directly (defn- flatten-tree [blocks-tree] (if-let [children (:block/_parent blocks-tree)] (cons (dissoc blocks-tree :block/_parent) (mapcat flatten-tree children)) [blocks-tree])) (defn get-block-and-children [repo block-uuid] (some-> (d/q '[:find [(pull ?block ?block-attrs) ...] :in $ ?id ?block-attrs :where [?block :block/uuid ?id]] (conn/get-db repo) block-uuid block-attrs) first flatten-tree)) (defn get-file-page ([file-path] (get-file-page file-path true)) ([file-path original-name?] (when-let [repo (state/get-current-repo)] (when-let [db (conn/get-db repo)] (some-> (d/q (if original-name? '[:find ?page-name :in $ ?path :where [?file :file/path ?path] [?page :block/file ?file] [?page :block/original-name ?page-name]] '[:find ?page-name :in $ ?path :where [?file :file/path ?path] [?page :block/file ?file] [?page :block/name ?page-name]]) db file-path) db-utils/seq-flatten first))))) (defn get-page-file ([page-name] (get-page-file (state/get-current-repo) page-name)) ([repo page-name] (some-> (or (db-utils/entity repo [:block/name page-name]) (db-utils/entity repo [:block/original-name page-name])) :block/file))) (defn get-block-file-path [block] (when-let [page-id (:db/id (:block/page block))] (:file/path (:block/file (db-utils/entity page-id))))) (defn get-file-page-id [file-path] (when-let [repo (state/get-current-repo)] (when-let [db (conn/get-db repo)] (some-> (d/q '[:find ?page :in $ ?path :where [?file :file/path ?path] [?page :block/name] [?page :block/file ?file]] db file-path) db-utils/seq-flatten first)))) (defn get-page [page-name] (if-let [id (parse-uuid page-name)] (db-utils/entity [:block/uuid id]) (db-utils/entity [:block/name (util/page-name-sanity-lc page-name)]))) (defn get-redirect-page-name "Given any readable page-name, return the exact page-name in db. If page doesn't exists yet, will return the passed `page-name`. Accepts both sanitized or unsanitized names. alias?: if true, alias is allowed to be returned; otherwise, it would be deref." ([page-name] (get-redirect-page-name page-name false)) ([page-name alias?] (when page-name (let [page-name' (util/page-name-sanity-lc page-name) page-entity (db-utils/entity [:block/name page-name'])] (cond alias? page-name' (nil? page-entity) page-name (page-empty-or-dummy? (state/get-current-repo) (:db/id page-entity)) (let [source-page (get-alias-source-page (state/get-current-repo) page-name')] (or (when source-page (:block/name source-page)) page-name')) :else page-name'))))) (defn get-page-original-name [page-name] (when (string? page-name) (let [page (db-utils/pull [:block/name (util/page-name-sanity-lc page-name)])] (or (:block/original-name page) (:block/name page))))) (defn get-journals-length [] (let [today (db-utils/date->int (js/Date.))] (d/q '[:find (count ?page) . :in $ ?today :where [?page :block/journal? true] [?page :block/journal-day ?journal-day] [(<= ?journal-day ?today)]] (conn/get-db (state/get-current-repo)) today))) (defn get-latest-journals ([n] (get-latest-journals (state/get-current-repo) n)) ([repo-url n] (when (conn/get-db repo-url) (let [date (js/Date.) _ (.setDate date (- (.getDate date) (dec n))) today (db-utils/date->int (js/Date.))] (->> (react/q repo-url [:frontend.db.react/journals] {:use-cache? false} '[:find [(pull ?page [*]) ...] :in $ ?today :where [?page :block/name ?page-name] [?page :block/journal? true] [?page :block/journal-day ?journal-day] [(<= ?journal-day ?today)]] today) (react) (sort-by :block/journal-day) (reverse) (take n)))))) ;; get pages that this page referenced (defn get-page-referenced-pages [repo page] (when-let [db (conn/get-db repo)] (let [page-name (util/safe-page-name-sanity-lc page) pages (page-alias-set repo page) page-id (:db/id (db-utils/entity [:block/name page-name])) ref-pages (d/q '[:find [?ref-page-name ...] :in $ ?pages :where [(untuple ?pages) [?page ...]] [?block :block/page ?page] [?block :block/refs ?ref-page] [?ref-page :block/name ?ref-page-name]] db pages)] (mapv (fn [page] [page (get-page-alias repo page)]) ref-pages)))) (def ns-char "/") (def ns-re #"/") (defn- get-parents-namespace-list "Return list of parents namespace" [page-namespace & nested-found] (if (text/namespace-page? page-namespace) (let [pre-nested-vec (drop-last (string/split page-namespace ns-re)) my-nested-found (if (nil? nested-found) [] nested-found)] (if (= (count pre-nested-vec) 1) (conj my-nested-found (nth pre-nested-vec 0)) (let [pre-nested-str (string/join ns-char pre-nested-vec)] (recur pre-nested-str (conj my-nested-found pre-nested-str))))) [])) (defn- get-unnecessary-namespaces-name "Return unnecessary namespace from a list of page's name" [pages-list] (->> pages-list (remove nil?) (mapcat get-parents-namespace-list) distinct)) (defn- remove-nested-namespaces-link "Remove relations between pages and their nested namespace" [pages-relations] (let [pages-relations-to-return (distinct (mapcat identity (for [item (for [a-link-from (mapv (fn [a-rel] (first a-rel)) pages-relations)] [a-link-from (mapv (fn [a-rel] (second a-rel)) (filterv (fn [link-target] (= a-link-from (first link-target))) pages-relations))]) :let [list-to (get item 1) page (get item 0) namespaces-to-remove (get-unnecessary-namespaces-name list-to) list-to-without-nested-ns (filterv (fn [elem] (not (some #{elem} namespaces-to-remove))) list-to) node-links (for [item-ok list-to-without-nested-ns] [page item-ok])]] (seq node-links))))] pages-relations-to-return)) ;; Ignore files with empty blocks for now (defn get-pages-relation [repo with-journal?] (when-let [db (conn/get-db repo)] (let [q (if with-journal? '[:find ?page ?ref-page-name :where [?p :block/name ?page] [?block :block/page ?p] [?block :block/refs ?ref-page] [?ref-page :block/name ?ref-page-name]] '[:find ?page ?ref-page-name :where [?p :block/journal? false] [?p :block/name ?page] [?block :block/page ?p] [?block :block/refs ?ref-page] [?ref-page :block/name ?ref-page-name]])] (->> (d/q q db) (map (fn [[page ref-page-name]] [page ref-page-name])) (remove-nested-namespaces-link))))) ;; get pages who mentioned this page ;; TODO: use :block/_refs (defn get-pages-that-mentioned-page [repo page include-journals] (when (conn/get-db repo) (let [page-id (:db/id (db-utils/entity [:block/name (util/safe-page-name-sanity-lc page)])) pages (page-alias-set repo page) query-base '[:find ?mentioned-page-name :in $ ?pages ?page-name :where [?block :block/refs ?p] [(contains? ?pages ?p)] [?block :block/page ?mentioned-page] [?mentioned-page :block/name ?mentioned-page-name]] query (if include-journals query-base (conj query-base '[?mentioned-page :block/journal? false])) mentioned-pages (->> (react/q repo [:frontend.db.react/page<-pages page-id] {:use-cache? false} query pages page) react db-utils/seq-flatten)] (mapv (fn [page] [page (get-page-alias repo page)]) mentioned-pages)))) (defn get-page-referenced-blocks-full ([page] (get-page-referenced-blocks-full (state/get-current-repo) page nil)) ([page options] (get-page-referenced-blocks-full (state/get-current-repo) page options)) ([repo page options] (when repo (when-let [db (conn/get-db repo)] (let [page-id (:db/id (db-utils/entity [:block/name (util/safe-page-name-sanity-lc page)])) pages (page-alias-set repo page) aliases (set/difference pages #{page-id})] (->> (d/q '[:find [(pull ?block ?block-attrs) ...] :in $ [?ref-page ...] ?block-attrs :where [?block :block/path-refs ?ref-page]] db pages (butlast block-attrs)) (remove (fn [block] (= page-id (:db/id (:block/page block))))) db-utils/group-by-page (map (fn [[k blocks]] (let [k (if (contains? aliases (:db/id k)) (assoc k :block/alias? true) k)] [k blocks]))))))))) (defn get-page-referenced-blocks ([page] (get-page-referenced-blocks (state/get-current-repo) page nil)) ([page options] (get-page-referenced-blocks (state/get-current-repo) page options)) ([repo page options] (when repo (when (conn/get-db repo) (let [page-id (:db/id (db-utils/entity [:block/name (util/safe-page-name-sanity-lc page)])) pages (page-alias-set repo page) aliases (set/difference pages #{page-id})] (->> (react/q repo [:frontend.db.react/refs page-id] {:use-cache? false :query-fn (fn [] (let [entities (mapcat (fn [id] (:block/_path-refs (db-utils/entity id))) pages) blocks (map (fn [e] {:block/parent (:block/parent e) :block/left (:block/left e) :block/page (:block/page e)}) entities)] {:entities entities :blocks blocks}))} nil) react :entities (remove (fn [block] (= page-id (:db/id (:block/page block))))))))))) (defn get-date-scheduled-or-deadlines [journal-title] (when-let [date (date/journal-title->int journal-title)] (let [future-days (state/get-scheduled-future-days)] (when-let [repo (state/get-current-repo)] (->> (react/q repo [:custom :scheduled-deadline journal-title] {} '[:find [(pull ?block ?block-attrs) ...] :in $ ?day ?future ?block-attrs :where (or [?block :block/scheduled ?d] [?block :block/deadline ?d]) [(get-else $ ?block :block/repeated? false) ?repeated] [(get-else $ ?block :block/marker "NIL") ?marker] [(not= ?marker "DONE")] [(not= ?marker "CANCELED")] [(not= ?marker "CANCELLED")] [(<= ?d ?future)] (or-join [?repeated ?d ?day] [(true? ?repeated)] [(>= ?d ?day)])] date (+ date future-days) block-attrs) react (sort-by-left-recursive) db-utils/group-by-page))))) (defn- pattern [name] (re-pattern (str "(?i)(^|[^\\[#0-9a-zA-Z]|((^|[^\\[])\\[))" (util/regex-escape name) "($|[^0-9a-zA-Z])"))) (defn get-page-unlinked-references [page] (when-let [repo (state/get-current-repo)] (when (conn/get-db repo) (let [page (util/safe-page-name-sanity-lc page) page-id (:db/id (db-utils/entity [:block/name page])) alias-names (get-page-alias-names repo page) patterns (->> (conj alias-names page) (map pattern)) filter-fn (fn [datom] (some (fn [p] (re-find p (->> (:v datom) (drawer/remove-logbook)))) patterns))] (->> (react/q repo [:frontend.db.react/page-unlinked-refs page-id] {:query-fn (fn [db _tx-report _result] (let [ids (->> (d/datoms db :aevt :block/content) (filter filter-fn) (map :e)) result (d/pull-many db block-attrs ids)] (remove (fn [block] (= page-id (:db/id (:block/page block)))) result)))} nil) react (sort-by-left-recursive) db-utils/group-by-page))))) (defn get-block-referenced-blocks ([block-uuid] (get-block-referenced-blocks block-uuid {})) ([block-uuid options] (when-let [repo (state/get-current-repo)] (when (conn/get-db repo) (let [block (db-utils/entity [:block/uuid block-uuid]) query-result (->> (react/q repo [:frontend.db.react/refs (:db/id block)] {} '[:find [(pull ?ref-block ?block-attrs) ...] :in $ ?block-uuid ?block-attrs :where [?block :block/uuid ?block-uuid] [?ref-block :block/refs ?block]] block-uuid block-attrs) react (sort-by-left-recursive))] (db-utils/group-by-page query-result)))))) (defn journal-page? "sanitized page-name only" [page-name] (:block/journal? (db-utils/entity [:block/name page-name]))) (defn get-public-pages [db] (-> (d/q '[:find ?p :where [?p :block/name] [?p :block/properties ?properties] [(get ?properties :public) ?pub] [(= true ?pub)]] db) (db-utils/seq-flatten))) (defn get-public-false-pages [db] (-> (d/q '[:find ?p :where [?p :block/name] [?p :block/properties ?properties] [(get ?properties :public) ?pub] [(= false ?pub)]] db) (db-utils/seq-flatten))) (defn get-public-false-block-ids [db] (-> (d/q '[:find ?b :where [?p :block/name] [?p :block/properties ?properties] [(get ?properties :public) ?pub] [(= false ?pub)] [?b :block/page ?p]] db) (db-utils/seq-flatten))) (defn get-all-templates [] (let [pred (fn [_db properties] (some? (:template properties)))] (->> (d/q '[:find ?b ?p :in $ ?pred :where [?b :block/properties ?p] [(?pred $ ?p)]] (conn/get-db) pred) (map (fn [[e m]] [(get m :template) e])) (into {})))) (defn get-all-properties [] (let [properties (d/q '[:find [?p ...] :where [_ :block/properties ?p]] (conn/get-db)) properties (remove (fn [m] (empty? m)) properties)] (->> (map keys properties) (apply concat) distinct sort))) (defn get-property-values [property] (let [pred (fn [_db properties] (get properties property))] (->> (d/q '[:find [?property-val ...] :in $ ?pred :where [_ :block/properties ?p] [(?pred $ ?p) ?property-val]] (conn/get-db) pred) (map (fn [x] (if (coll? x) x [x]))) (apply concat) (map str) (remove string/blank?) (distinct) (sort)))) (defn get-template-by-name [name] (when (string? name) (->> (d/q '[:find [(pull ?b [*]) ...] :in $ ?name :where [?b :block/properties ?p] [(get ?p :template) ?t] [(= ?t ?name)]] (conn/get-db) name) (sort-by :block/name) (first)))) (defonce blocks-count-cache (atom nil)) (defn blocks-count ([] (blocks-count true)) ([cache?] (if (and cache? @blocks-count-cache) @blocks-count-cache (when-let [db (conn/get-db)] (let [n (count (d/datoms db :avet :block/uuid))] (reset! blocks-count-cache n) n))))) (defn get-all-referenced-blocks-uuid "Get all uuids of blocks with any back link exists." [] (when-let [db (conn/get-db)] (d/q '[:find [?refed-uuid ...] :where ;; ?referee-b is block with ref towards ?refed-b [?refed-b :block/uuid ?refed-uuid] [?referee-b :block/refs ?refed-b]] db))) ;; block/uuid and block/content (defn get-all-block-contents [] (when-let [db (conn/get-db)] (->> (d/datoms db :avet :block/uuid) (map :v) (map (fn [id] (let [e (db-utils/entity [:block/uuid id])] (when (and (not (:block/name e)) (not (string/blank? (:block/content e)))) {:db/id (:db/id e) :block/uuid id :block/page (:db/id (:block/page e)) :block/content (:block/content e) :block/format (:block/format e)})))) (remove nil?)))) (defn get-assets [datoms] (keep (fn [datom] (when (= :block/content (:a datom)) (let [matched (re-seq #"\([./]*/assets/([^)]+)\)" (:v datom)) matched (get (into [] matched) 0) path (get matched 1)] (when (and (string? path) (not (string/ends-with? path ".js"))) path)))) datoms)) (defn clean-export! [db] (let [remove? #(contains? #{"me" "recent" "file"} %) non-public-pages (get-public-false-pages db) non-public-datoms (get-public-false-block-ids db) non-public-datom-ids (set (concat non-public-pages non-public-datoms)) filtered-db (d/filter db (fn [_db datom] (let [ns (namespace (:a datom))] (and (not (remove? ns)) (not (contains? #{:block/file} (:a datom))) (not (contains? non-public-datom-ids (:e datom))))))) datoms (d/datoms filtered-db :eavt) assets (get-assets datoms)] [@(d/conn-from-datoms datoms db-schema/schema) assets])) (defn filter-only-public-pages-and-blocks [db] (let [public-pages (get-public-pages db)] (when (seq public-pages) (let [public-pages (set public-pages) exported-namespace? #(contains? #{"block" "me" "recent"} %) filtered-db (d/filter db (fn [db datom] (let [ns (namespace (:a datom))] (and (not (contains? #{:block/file} (:a datom))) (not= ns "file") (or (not (exported-namespace? ns)) (and (= ns "block") (or (contains? public-pages (:e datom)) (contains? public-pages (:db/id (:block/page (d/entity db (:e datom)))))))))))) datoms (d/datoms filtered-db :eavt) assets (get-assets datoms)] [@(d/conn-from-datoms datoms db-schema/schema) assets])))) (defn delete-blocks [repo-url files _delete-page?] (when (seq files) (let [blocks (get-files-blocks repo-url files)] (mapv (fn [eid] [:db.fn/retractEntity eid]) blocks)))) (defn delete-files [files] (mapv (fn [path] [:db.fn/retractEntity [:file/path path]]) files)) (defn delete-file-blocks! [repo-url path] (let [blocks (get-file-blocks repo-url path)] (mapv (fn [eid] [:db.fn/retractEntity eid]) blocks))) (defn delete-page-blocks [repo-url page] (when page (when-let [db (conn/get-db repo-url)] (let [page (db-utils/pull [:block/name (util/page-name-sanity-lc page)])] (when page (let [datoms (d/datoms db :avet :block/page (:db/id page)) block-eids (mapv :e datoms)] (mapv (fn [eid] [:db.fn/retractEntity eid]) block-eids))))))) (defn delete-pages-by-files [files] (let [pages (->> (mapv get-file-page files) (remove nil?))] (when (seq pages) (mapv (fn [page] [:db.fn/retractEntity [:block/name page]]) (map util/page-name-sanity-lc pages))))) (defn set-file-content! ([repo path content] (set-file-content! repo path content {})) ([repo path content opts] (when (and repo path) (let [tx-data {:file/path path :file/content content}] (db-utils/transact! repo [tx-data] (merge opts {:skip-refresh? true})))))) (defn get-pre-block [repo page-id] (-> (d/q '[:find (pull ?b [*]) :in $ ?page :where [?b :block/page ?page] [?b :block/pre-block? true]] (conn/get-db repo) page-id) ffirst)) (defn get-namespace-pages "Accepts both sanitized and unsanitized namespaces" [repo namespace] (assert (string? namespace)) (let [namespace (util/page-name-sanity-lc namespace)] (d/q '[:find [(pull ?c [:db/id :block/name :block/original-name :block/namespace {:block/file [:db/id :file/path]}]) ...] :in $ % ?namespace :where [?p :block/name ?namespace] (namespace ?p ?c)] (conn/get-db repo) rules namespace))) (defn- tree [flat-col root] (let [sort-fn #(sort-by :block/name %) children (group-by :block/namespace flat-col) namespace-children (fn namespace-children [parent-id] (map (fn [m] (assoc m :namespace/children (sort-fn (namespace-children {:db/id (:db/id m)})))) (sort-fn (get children parent-id))))] (namespace-children root))) (defn get-namespace-hierarchy "Unsanitized namespaces" [repo namespace] (let [children (get-namespace-pages repo namespace) namespace-id (:db/id (db-utils/entity [:block/name (util/page-name-sanity-lc namespace)])) root {:db/id namespace-id} col (conj children root)] (tree col root))) (defn get-page-namespace [repo page] (:block/namespace (db-utils/entity repo [:block/name (util/page-name-sanity-lc page)]))) (defn get-page-namespace-routes [repo page] (assert (string? page)) (when-let [db (conn/get-db repo)] (when-not (string/blank? page) (let [page (util/page-name-sanity-lc (string/trim page)) page-exist? (db-utils/entity repo [:block/name page]) ids (if page-exist? '() (->> (d/datoms db :aevt :block/name) (filter (fn [datom] (string/ends-with? (:v datom) (str "/" page)))) (map :e)))] (when (seq ids) (db-utils/pull-many repo '[:db/id :block/name :block/original-name {:block/file [:db/id :file/path]}] ids)))))) (defn get-orphaned-pages [{:keys [repo pages empty-ref-f] :or {repo (state/get-current-repo) empty-ref-f (fn [page] (zero? (count (:block/_refs page))))}}] (let [pages (->> (or pages (get-pages repo)) (remove nil?)) built-in-pages (set (map string/lower-case default-db/built-in-pages-names)) orphaned-pages (->> (map (fn [page] (let [name (util/page-name-sanity-lc page)] (when-let [page (db-utils/entity [:block/name name])] (and (empty-ref-f page) (or (page-empty? repo (:db/id page)) (let [first-child (first (:block/_left page)) children (:block/_page page)] (and first-child (= 1 (count children)) (contains? #{"" "-" "*"} (string/trim (:block/content first-child)))))) (not (contains? built-in-pages name)) (not (:block/_namespace page)) ;; a/b/c might be deleted but a/b/c/d still exists (for backward compatibility) (not (and (string/includes? name "/") (not (:block/journal? page)))) page)))) pages) (remove false?) (remove nil?))] orphaned-pages)) (defn get-macro-blocks [repo macro-name] (d/q '[:find [(pull ?b [*]) ...] :in $ ?macro-name :where [?b :block/type "macro"] [?b :block/properties ?properties] [(get ?properties :logseq.macro-name) ?name] [(= ?name ?macro-name)]] (conn/get-db repo) macro-name))