Merge remote-tracking branch 'origin/feat/db' into feat/hnswlib+transformer-js

bb.edn

@@ -75,7 +75,9 @@
 
   dev:db-transact
   {:doc "Transact against a DB graph's datascript db"
-   :task (apply shell {:dir "deps/outliner"} "yarn -s nbb-logseq script/transact.cljs" *command-line-args*)}
+   :requires ([babashka.fs :as fs])
+   :task (apply shell {:dir "deps/outliner" :extra-env {"ORIGINAL_PWD" (fs/cwd)}}
+                "yarn -s nbb-logseq script/transact.cljs" *command-line-args*)}
 
   dev:db-create
   {:doc "Create a DB graph given a sqlite.build EDN file"

deps/common/src/logseq/common/uuid.cljs

@@ -12,7 +12,7 @@ the remaining chars for data of this type"
   (let [journal-day-str  (str journal-day)
         part1 (subs journal-day-str 0 4)
         part2 (subs journal-day-str 4 8)]
-    (uuid (str "00000001-" part1 "-" part2 "-0000-000000000000"))))
+    (uuid (str "00000001" "-" part1 "-" part2 "-0000-000000000000"))))
 
 (defn- fill-with-0
   [s length]
@@ -49,3 +49,16 @@ the remaining chars for data of this type"
      :db-ident-block-uuid (gen-db-ident-block-uuid v)
      :migrate-new-block-uuid (gen-block-uuid v "00000003")
      :builtin-block-uuid (gen-block-uuid v "00000004"))))
+
+(defn gen-journal-template-block
+  "Persistent uuid for journal template block"
+  [journal-uuid template-block-uuid]
+  (assert (uuid? journal-uuid) (str journal-uuid))
+  (assert (uuid? template-block-uuid) (str template-block-uuid))
+  (uuid
+   (str "00000005"
+        "-"
+       ;; journal day
+        (subs (str journal-uuid) 9 23)
+       ;; template block uuid
+        (subs (str template-block-uuid) 23))))

deps/db/script/create_graph.cljs

@@ -20,6 +20,17 @@
     path
     (node-path/join (or js/process.env.ORIGINAL_PWD ".") path)))
 
+(defn- get-dir-and-db-name
+  "Gets dir and db name for use with open-db! Works for relative and absolute paths and
+   defaults to ~/logseq/graphs/ when no '/' present in name"
+  [graph-dir]
+  (if (string/includes? graph-dir "/")
+    (let [resolve-path' #(if (node-path/isAbsolute %) %
+                             ;; $ORIGINAL_PWD used by bb tasks to correct current dir
+                             (node-path/join (or js/process.env.ORIGINAL_PWD ".") %))]
+      ((juxt node-path/dirname node-path/basename) (resolve-path' graph-dir)))
+    [(node-path/join (os/homedir) "logseq" "graphs") graph-dir]))
+
 (def spec
   "Options spec"
   {:help {:alias :h
@@ -34,9 +45,7 @@
             (println (str "Usage: $0 GRAPH-NAME EDN-PATH [OPTIONS]\nOptions:\n"
                           (cli/format-opts {:spec spec})))
             (js/process.exit 1))
-        [dir db-name] (if (string/includes? graph-dir "/")
-                        ((juxt node-path/dirname node-path/basename) graph-dir)
-                        [(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
+        [dir db-name] (get-dir-and-db-name graph-dir)
         sqlite-build-edn (merge {:auto-create-ontology? true}
                                 (-> (resolve-path edn-path) fs/readFileSync str edn/read-string))
         conn (outliner-cli/init-conn dir db-name {:classpath (cp/get-classpath) :import-type :cli/create-graph})

deps/db/script/query.cljs

@@ -23,12 +23,14 @@
                            (clj->js (merge {:stdio "inherit"} opts))))
 
 (defn- get-dir-and-db-name
-  "Gets dir and db name for use with open-db!"
+  "Gets dir and db name for use with open-db! Works for relative and absolute paths and
+   defaults to ~/logseq/graphs/ when no '/' present in name"
   [graph-dir]
   (if (string/includes? graph-dir "/")
-    (let [graph-dir'
-          (node-path/join (or js/process.env.ORIGINAL_PWD ".") graph-dir)]
-      ((juxt node-path/dirname node-path/basename) graph-dir'))
+    (let [resolve-path' #(if (node-path/isAbsolute %) %
+                             ;; $ORIGINAL_PWD used by bb tasks to correct current dir
+                             (node-path/join (or js/process.env.ORIGINAL_PWD ".") %))]
+      ((juxt node-path/dirname node-path/basename) (resolve-path' graph-dir)))
     [(node-path/join (os/homedir) "logseq" "graphs") graph-dir]))
 
 (def spec

deps/db/script/validate_db.cljs

@@ -50,6 +50,17 @@
         (js/process.exit 1))
       (println "Valid!"))))
 
+(defn- get-dir-and-db-name
+  "Gets dir and db name for use with open-db! Works for relative and absolute paths and
+   defaults to ~/logseq/graphs/ when no '/' present in name"
+  [graph-dir]
+  (if (string/includes? graph-dir "/")
+    (let [resolve-path' #(if (node-path/isAbsolute %) %
+                             ;; $ORIGINAL_PWD used by bb tasks to correct current dir
+                             (node-path/join (or js/process.env.ORIGINAL_PWD ".") %))]
+      ((juxt node-path/dirname node-path/basename) (resolve-path' graph-dir)))
+    [(node-path/join (os/homedir) "logseq" "graphs") graph-dir]))
+
 (def spec
   "Options spec"
   {:help {:alias :h
@@ -75,11 +86,7 @@
     (validate-db* db ent-maps options)))
 
 (defn- validate-graph [graph-dir options]
-  (let [[dir db-name] (if (string/includes? graph-dir "/")
-                        (let [graph-dir'
-                              (node-path/join (or js/process.env.ORIGINAL_PWD ".") graph-dir)]
-                          ((juxt node-path/dirname node-path/basename) graph-dir'))
-                        [(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
+  (let [[dir db-name] (get-dir-and-db-name graph-dir)
         conn (try (sqlite-cli/open-db! dir db-name)
                   (catch :default e
                     (println "Error: For graph" (str (pr-str graph-dir) ":") (str e))

deps/graph-parser/script/db_import.cljs

@@ -125,6 +125,17 @@
       (p/let [_ (gp-exporter/export-doc-files conn files' <read-file doc-options)]
         {:import-state (:import-state doc-options)}))))
 
+(defn- get-dir-and-db-name
+  "Gets dir and db name for use with open-db! Works for relative and absolute paths and
+   defaults to ~/logseq/graphs/ when no '/' present in name"
+  [graph-dir]
+  (if (string/includes? graph-dir "/")
+    (let [resolve-path' #(if (node-path/isAbsolute %) %
+                             ;; $ORIGINAL_PWD used by bb tasks to correct current dir
+                             (node-path/join (or js/process.env.ORIGINAL_PWD ".") %))]
+      ((juxt node-path/dirname node-path/basename) (resolve-path' graph-dir)))
+    [(node-path/join (os/homedir) "logseq" "graphs") graph-dir]))
+
 (def spec
   "Options spec"
   {:help {:alias :h
@@ -160,10 +171,7 @@
             (println (str "Usage: $0 FILE-GRAPH DB-GRAPH [OPTIONS]\nOptions:\n"
                           (cli/format-opts {:spec spec})))
             (js/process.exit 1))
-        [dir db-name] (if (string/includes? db-graph-dir "/")
-                        (let [graph-dir' (resolve-path db-graph-dir)]
-                          ((juxt node-path/dirname node-path/basename) graph-dir'))
-                        [(node-path/join (os/homedir) "logseq" "graphs") db-graph-dir])
+        [dir db-name] (get-dir-and-db-name db-graph-dir)
         file-graph' (resolve-path file-graph)
         conn (outliner-cli/init-conn dir db-name {:classpath (cp/get-classpath)})
         directory? (.isDirectory (fs/statSync file-graph'))

deps/outliner/script/transact.cljs

@@ -10,15 +10,24 @@
             [logseq.outliner.db-pipeline :as db-pipeline]
             [nbb.core :as nbb]))
 
+(defn- get-dir-and-db-name
+  "Gets dir and db name for use with open-db! Works for relative and absolute paths and
+   defaults to ~/logseq/graphs/ when no '/' present in name"
+  [graph-dir]
+  (if (string/includes? graph-dir "/")
+    (let [resolve-path' #(if (node-path/isAbsolute %) %
+                             ;; $ORIGINAL_PWD used by bb tasks to correct current dir
+                             (node-path/join (or js/process.env.ORIGINAL_PWD ".") %))]
+      ((juxt node-path/dirname node-path/basename) (resolve-path' graph-dir)))
+    [(node-path/join (os/homedir) "logseq" "graphs") graph-dir]))
+
 (defn -main [args]
   (when (< (count args) 3)
     (println "Usage: $0 GRAPH-DIR QUERY TRANSACT-FN")
     (js/process.exit 1))
   (let [[graph-dir query* transact-fn*] args
         dry-run? (contains? (set args) "-n")
-        [dir db-name] (if (string/includes? graph-dir "/")
-                        ((juxt node-path/dirname node-path/basename) graph-dir)
-                        [(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
+        [dir db-name] (get-dir-and-db-name graph-dir)
         conn (sqlite-cli/open-db! dir db-name)
         ;; find blocks to update
         query (into (edn/read-string query*) [:in '$ '%]) ;; assumes no :in are in queries

e2e-tests/logseq-api.spec.ts

@@ -215,7 +215,7 @@ test('(DB graph): block related apis',
     prop1 = await callAPI('get_property', 'map1')
     const b1p = await callAPI('get_block_property', b1.uuid, 'map1')
 
-    expect(prop1.schema.type).toBe('map')
+    expect(prop1.type).toBe('map')
     expect(b1p).toEqual({a: 1})
 
     // await page.pause()

libs/src/LSPlugin.caller.ts

@@ -50,6 +50,7 @@ class LSPluginCaller extends EventEmitter {
     }
   }
 
+  // run in host
   async connectToChild() {
     if (this._connected) return
 
@@ -303,7 +304,7 @@ class LSPluginCaller extends EventEmitter {
 
           this._call = async (...args: any) => {
             // parent all will get message before handshake
-            await refChild.call(LSPMSGFn(pl.id), {
+            refChild.call(LSPMSGFn(pl.id), {
               type: args[0],
               payload: Object.assign(args[1] || {}, {
                 $$pid: pl.id,

libs/src/LSPlugin.core.ts

@@ -695,7 +695,9 @@ class PluginLocal extends EventEmitter<
         options.url = this._resolveResourceFullUrl(options.url, this._localRoot)
 
         // file:// for native
-        if (!this.isWebPlugin && !options.url.startsWith('file:')) {
+        if (!this.isWebPlugin &&
+          !options.url.startsWith('file:') &&
+          !options.url.startsWith('lsp:')) {
           options.url = 'assets://' + options.url
         }
       }

libs/src/LSPlugin.ts

@@ -315,7 +315,7 @@ export type ExternalCommandType =
   | 'logseq.ui/toggle-theme'
   | 'logseq.ui/toggle-wide-mode'
 
-export type UserProxyTags = 'app' | 'editor' | 'db' | 'git' | 'ui' | 'assets' | 'utils'
+export type UserProxyNSTags = 'app' | 'editor' | 'db' | 'git' | 'ui' | 'assets' | 'utils'
 
 export type SearchIndiceInitStatus = boolean
 export type SearchBlockItem = {

libs/src/LSPlugin.user.ts

@@ -30,7 +30,7 @@ import {
   IUserOffHook,
   IGitProxy,
   IUIProxy,
-  UserProxyTags,
+  UserProxyNSTags,
   BlockUUID,
   BlockEntity,
   IDatom,
@@ -734,7 +734,7 @@ export class LSPluginUser
   /**
    * @internal
    */
-  _makeUserProxy(target: any, tag?: UserProxyTags) {
+  _makeUserProxy(target: any, nstag?: UserProxyNSTags) {
     const that = this
     const caller = this.caller
 
@@ -744,13 +744,13 @@ export class LSPluginUser
 
         return function (this: any, ...args: any) {
           if (origMethod) {
-            if (args?.length !== 0) args.concat(tag)
+            if (args?.length !== 0) args.concat(nstag)
             const ret = origMethod.apply(that, args)
             if (ret !== PROXY_CONTINUE) return ret
           }
 
           // Handle hook
-          if (tag) {
+          if (nstag) {
             const hookMatcher = propKey.toString().match(/^(once|off|on)/i)
 
             if (hookMatcher != null) {
@@ -771,7 +771,7 @@ export class LSPluginUser
                 opts = args[2]
               }
 
-              type = `hook:${tag}:${safeSnakeCase(type)}`
+              type = `hook:${nstag}:${safeSnakeCase(type)}`
 
               caller[f](type, handler)
 
@@ -796,13 +796,13 @@ export class LSPluginUser
           let method = propKey as string
 
           // TODO: refactor api call with the explicit tag
-          if ((['git', 'ui', 'assets', 'utils'] as UserProxyTags[]).includes(tag)) {
-            method = tag + '_' + method
+          if ((['git', 'ui', 'assets', 'utils'] as UserProxyNSTags[]).includes(nstag)) {
+            method = nstag + '_' + method
           }
 
           // Call host
           return caller.callAsync(`api:call`, {
-            tag,
+            tag: nstag,
             method,
             args: args,
           })

src/electron/electron/core.cljs

@@ -72,7 +72,7 @@
                  (re-find #"(?i)^/[a-zA-Z]:" path))
              (callback #js {:path path})
 
-             ;; assume winwdows unc path
+             ;; assume windows unc path
              utils/win32?
              (do (logger/debug :resolve-assets-url url)
                  (callback #js {:path (str "//" path)}))

src/main/frontend/common/file/core.cljs

@@ -31,11 +31,16 @@
 
 (defn- transform-content
   [repo db {:block/keys [collapsed? format pre-block? title page properties] :as b} level {:keys [heading-to-list?]} context]
-  (let [block-ref-not-saved? (and (seq (:block/_refs (d/entity db (:db/id b))))
+  (let [db-based? (sqlite-util/db-based-graph? repo)
+        block-ref-not-saved? (and (seq (:block/_refs (d/entity db (:db/id b))))
                                   (not (string/includes? title (str (:block/uuid b))))
-                                  (not (sqlite-util/db-based-graph? repo)))
+                                  (not db-based?))
         heading (:heading properties)
         markdown? (= :markdown format)
+        title (if db-based?
+                ;; replace [[uuid]] with block's content
+                (:block/title (assoc (d/entity db (:db/id b)) :block.temp/search? true))
+                title)
         content (or title "")
         page-first-child? (= (:db/id b) (ldb/get-first-child db (:db/id page)))
         pre-block? (or pre-block?
@@ -82,11 +87,10 @@
                                     (string/blank? new-content))
                               ""
                               " ")]
-                    (str prefix sep new-content)))
-        content (if block-ref-not-saved?
-                  (gp-property/insert-property repo format content :id (str (:block/uuid b)))
-                  content)]
-    content))
+                    (str prefix sep new-content)))]
+    (if block-ref-not-saved?
+      (gp-property/insert-property repo format content :id (str (:block/uuid b)))
+      content)))
 
 (defn- tree->file-content-aux
   [repo db tree {:keys [init-level] :as opts} context]

src/main/frontend/handler/dnd.cljs

@@ -1,6 +1,7 @@
 (ns frontend.handler.dnd
   "Provides fns for drag and drop"
-  (:require [frontend.db :as db]
+  (:require [frontend.config :as config]
+            [frontend.db :as db]
             [frontend.handler.block :as block-handler]
             [frontend.handler.editor :as editor-handler]
             [frontend.handler.property :as property-handler]
@@ -8,6 +9,7 @@
             [frontend.modules.outliner.ui :as ui-outliner-tx]
             [frontend.state :as state]
             [logseq.common.util.block-ref :as block-ref]
+            [logseq.common.util.page-ref :as page-ref]
             [logseq.db :as ldb]))
 
 (defn move-blocks
@@ -25,10 +27,10 @@
     (cond
       ;; alt pressed, make a block-ref
       (and alt-key? (= (count blocks) 1))
-      (do
+      (let [->ref (if (config/db-based-graph?) page-ref/->page-ref block-ref/->block-ref)]
         (property-handler/file-persist-block-id! (state/get-current-repo) (:block/uuid first-block))
         (editor-handler/api-insert-new-block!
-         (block-ref/->block-ref (:block/uuid first-block))
+         (->ref (:block/uuid first-block))
          {:block-uuid (:block/uuid target-block)
           :sibling? (not nested?)
           :before? top?}))

src/main/frontend/handler/editor.cljs

@@ -1941,7 +1941,8 @@
                                             size)
       (let [new-meta (merge metadata size)
             image-part (first (string/split full_text #"\{"))
-            new-full-text (str image-part (pr-str new-meta))
+            md-link? (string/starts-with? image-part "![")
+            new-full-text (str (if md-link? image-part (str "![image](" image-part ")")) (pr-str new-meta))
             block (db/entity [:block/uuid block-id])
             value (:block/title block)
             new-value (string/replace value full_text new-full-text)]

src/main/frontend/worker/pipeline.cljs

@@ -7,6 +7,7 @@
             [frontend.worker.react :as worker-react]
             [frontend.worker.state :as worker-state]
             [logseq.common.defkeywords :refer [defkeywords]]
+            [logseq.common.uuid :as common-uuid]
             [logseq.db :as ldb]
             [logseq.db.frontend.validate :as db-validate]
             [logseq.db.sqlite.export :as sqlite-export]
@@ -49,6 +50,8 @@
 (defn- insert-tag-templates
   [repo conn tx-report]
   (let [db (:db-after tx-report)
+        journal-id (:db/id (d/entity db :logseq.class/Journal))
+        journal-template? (some (fn [d] (and (:added d) (= (:a d) :block/tags) (= (:v d) journal-id))) (:tx-data tx-report))
         tx-data (some->> (:tx-data tx-report)
                          (filter (fn [d] (and (= (:a d) :block/tags) (:added d))))
                          (group-by :e)
@@ -56,9 +59,12 @@
                                    (let [object (d/entity db e)
                                          template-blocks (->> (mapcat (fn [id]
                                                                         (let [tag (d/entity db id)
+                                                                              journal? (= journal-id id)
                                                                               parents (ldb/get-page-parents tag {:node-class? true})
                                                                               templates (mapcat :logseq.property/_template-applied-to (conj parents tag))]
-                                                                          templates))
+                                                                          (cond->> templates
+                                                                            journal?
+                                                                            (map (fn [t] (assoc t :journal tag))))))
                                                                       (set (map :v datoms)))
                                                               distinct
                                                               (sort-by :block/created-at)
@@ -68,10 +74,17 @@
                                                                               blocks (->>
                                                                                       (cons (assoc (first template-blocks) :logseq.property/used-template (:db/id template))
                                                                                             (rest template-blocks))
-                                                                                      (map (fn [e] (assoc (into {} e) :db/id (:db/id e)))))]
+                                                                                      (map (fn [e]
+                                                                                             (cond->
+                                                                                              (assoc (into {} e) :db/id (:db/id e))
+                                                                                               (:journal template)
+                                                                                               (assoc :block/uuid
+                                                                                                      (common-uuid/gen-journal-template-block (:block/uuid (:journal template))
+                                                                                                                                              (:block/uuid e)))))))]
                                                                           blocks))))]
                                      (when (seq template-blocks)
-                                       (let [result (outliner-core/insert-blocks repo conn template-blocks object {:sibling? false})]
+                                       (let [result (outliner-core/insert-blocks repo conn template-blocks object {:sibling? false
+                                                                                                                   :keep-uuid? journal-template?})]
                                          (:tx-data result)))))))]
     tx-data))
 

src/main/frontend/worker/search.cljs

@@ -334,7 +334,7 @@ DROP TRIGGER IF EXISTS blocks_au;
   (drop-tables-and-triggers! db)
   (create-tables-and-triggers! db))
 
-(defn get-all-block-contents
+(defn get-all-blocks
   [db]
   (when db
     (->> (d/datoms db :avet :block/uuid)
@@ -347,7 +347,7 @@ DROP TRIGGER IF EXISTS blocks_au;
 (defn build-blocks-indice
   [repo db]
   (build-fuzzy-search-indice repo db)
-  (->> (get-all-block-contents db)
+  (->> (get-all-blocks db)
        (keep block->index)
        (bean/->js)))
 

src/main/logseq/api.cljs

@@ -903,7 +903,10 @@
   [k]
   (this-as this
            (p/let [prop (-get-property this k)]
-             (bean/->js (sdk-utils/normalize-keyword-for-json prop)))))
+             (-> prop
+               (assoc :type (:logseq.property/type prop))
+               (sdk-utils/normalize-keyword-for-json)
+               (bean/->js)))))
 
 (defn ^:export upsert_property
   "schema:
@@ -929,7 +932,8 @@
                                 (string? (:cardinality schema))
                                 (update :cardinality keyword)
                                 (string? (:type schema))
-                                (update :type keyword))
+                                (-> (assoc :logseq.property/type (keyword (:type schema)))
+                                  (dissoc :type)))
                        p (db-property-handler/upsert-property! k schema
                                                                (cond-> opts
                                                                  name