Enhance/api storages for graph assets (#6488)

* improve(api): assets storage for plugin

Co-authored-by: charlie <[email protected]>

libs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@logseq/libs",
-  "version": "0.0.8",
+  "version": "0.0.10",
   "description": "Logseq SDK libraries",
   "main": "dist/lsplugin.user.js",
   "typings": "index.d.ts",

libs/src/LSPlugin.ts

@@ -3,7 +3,7 @@ import * as CSS from 'csstype'
 import EventEmitter from 'eventemitter3'
 import { LSPluginCaller } from './LSPlugin.caller'
 import { LSPluginExperiments } from './modules/LSPlugin.Experiments'
-import { LSPluginFileStorage } from './modules/LSPlugin.Storage'
+import { IAsyncStorage, LSPluginFileStorage } from './modules/LSPlugin.Storage'
 import { LSPluginRequest } from './modules/LSPlugin.Request'
 
 export type WithOptional<T, K extends keyof T> = Omit<T, K> & Partial<Pick<T, K>>;
@@ -169,6 +169,7 @@ export interface BlockEntity {
   unordered: boolean
   content: string
   page: IEntityID
+  properties?: Record<string, any>
 
   // optional fields in dummy page
   anchor?: string
@@ -196,9 +197,12 @@ export interface PageEntity {
   file?: IEntityID
   namespace?: IEntityID
   children?: Array<PageEntity>
+  properties?: Record<string, any>
   format?: 'markdown' | 'org'
   journalDay?: number
   updatedAt?: number
+
+  [key: string]: any
 }
 
 export type BlockIdentity = BlockUUID | Pick<BlockEntity, 'uuid'>
@@ -412,9 +416,9 @@ export interface IAppProxy {
 
   // hook events
   onCurrentGraphChanged: IUserHook
-  onGraphAfterIndexed: IUserHook<{repo: string}>
+  onGraphAfterIndexed: IUserHook<{ repo: string }>
   onThemeModeChanged: IUserHook<{ mode: 'dark' | 'light' }>
-  onThemeChanged: IUserHook<Partial<{name: string, mode: string, pid: string, url: string}>>
+  onThemeChanged: IUserHook<Partial<{ name: string, mode: string, pid: string, url: string }>>
   onBlockRendererSlotted: IUserSlotHook<{ uuid: BlockUUID }>
 
   /**
@@ -786,7 +790,7 @@ export interface IAssetsProxy {
    * @added 0.0.2
    * @param exts
    */
-  listFilesOfCurrentGraph(exts: string | string[]): Promise<{
+  listFilesOfCurrentGraph(exts?: string | string[]): Promise<{
     path: string
     size: number
     accessTime: number
@@ -794,6 +798,12 @@ export interface IAssetsProxy {
     changeTime: number
     birthTime: number
   }>
+
+  /**
+   * @example https://github.com/logseq/logseq/pull/6488
+   * @added 0.0.10
+   */
+  makeSandboxStorage(): IAsyncStorage
 }
 
 export interface ILSPluginThemeManager {
@@ -963,6 +973,7 @@ export interface ILSPluginUser extends EventEmitter<LSPluginUserEvents> {
   DB: IDBProxy
   Git: IGitProxy
   UI: IUIProxy
+  Assets: IAssetsProxy
 
   Request: LSPluginRequest
   FileStorage: LSPluginFileStorage

libs/src/LSPlugin.user.ts

@@ -38,7 +38,7 @@ import {
 import Debug from 'debug'
 import * as CSS from 'csstype'
 import EventEmitter from 'eventemitter3'
-import { LSPluginFileStorage } from './modules/LSPlugin.Storage'
+import { IAsyncStorage, LSPluginFileStorage } from './modules/LSPlugin.Storage'
 import { LSPluginExperiments } from './modules/LSPlugin.Experiments'
 import { LSPluginRequest } from './modules/LSPlugin.Request'
 
@@ -311,8 +311,18 @@ const db: Partial<IDBProxy> = {
 }
 
 const git: Partial<IGitProxy> = {}
+
 const ui: Partial<IUIProxy> = {}
-const assets: Partial<IAssetsProxy> = {}
+
+const assets: Partial<IAssetsProxy> = {
+  makeSandboxStorage(
+    this: LSPluginUser
+  ): IAsyncStorage {
+    return new LSPluginFileStorage(
+      this, { assets: true }
+    )
+  }
+}
 
 type uiState = {
   key?: number

libs/src/modules/LSPlugin.Storage.ts

@@ -9,6 +9,8 @@ export interface IAsyncStorage {
 
   hasItem(key: string): Promise<boolean>
 
+  allKeys(): Promise<Array<string>>
+
   clear(): Promise<void>
 }
 
@@ -18,8 +20,14 @@ export interface IAsyncStorage {
 class LSPluginFileStorage implements IAsyncStorage {
   /**
    * @param ctx
+   * @param opts
    */
-  constructor(private ctx: LSPluginUser) {}
+  constructor(
+    private ctx: LSPluginUser,
+    private opts?: {
+      assets: boolean
+    }
+  ) {}
 
   /**
    * plugin id
@@ -32,20 +40,20 @@ class LSPluginFileStorage implements IAsyncStorage {
    * @param key A string as file name that support nested directory
    * @param value Storage value
    */
-  setItem(key: string, value: string): Promise<void> {
+  setItem(key: string, value: string | any): Promise<void> {
     return this.ctx.caller.callAsync(`api:call`, {
       method: 'write-plugin-storage-file',
-      args: [this.ctxId, key, value],
+      args: [this.ctxId, key, value, this.opts?.assets],
     })
   }
 
   /**
    * @param key
    */
-  getItem(key: string): Promise<string | undefined> {
+  getItem(key: string): Promise<string | any> {
     return this.ctx.caller.callAsync(`api:call`, {
       method: 'read-plugin-storage-file',
-      args: [this.ctxId, key],
+      args: [this.ctxId, key, this.opts?.assets],
     })
   }
 
@@ -55,7 +63,17 @@ class LSPluginFileStorage implements IAsyncStorage {
   removeItem(key: string): Promise<void> {
     return this.ctx.caller.call(`api:call`, {
       method: 'unlink-plugin-storage-file',
-      args: [this.ctxId, key],
+      args: [this.ctxId, key, this.opts?.assets],
+    })
+  }
+
+  /**
+   * Get all path file keys
+   */
+  allKeys(): Promise<Array<string>> {
+    return this.ctx.caller.callAsync(`api:call`, {
+      method: 'list-plugin-storage-files',
+      args: [this.ctxId, this.opts?.assets]
     })
   }
 
@@ -65,7 +83,7 @@ class LSPluginFileStorage implements IAsyncStorage {
   clear(): Promise<void> {
     return this.ctx.caller.call(`api:call`, {
       method: 'clear-plugin-storage-files',
-      args: [this.ctxId],
+      args: [this.ctxId, this.opts?.assets],
     })
   }
 
@@ -75,7 +93,7 @@ class LSPluginFileStorage implements IAsyncStorage {
   hasItem(key: string): Promise<boolean> {
     return this.ctx.caller.callAsync(`api:call`, {
       method: 'exist-plugin-storage-file',
-      args: [this.ctxId, key],
+      args: [this.ctxId, key, this.opts?.assets],
     })
   }
 }

src/electron/electron/handler.cljs

@@ -55,8 +55,13 @@
 (defmethod handle :readdir [_window [_ dir]]
   (readdir dir))
 
+(defmethod handle :listdir [_window [_ dir flat?]]
+  (when (and dir (fs-extra/pathExistsSync dir))
+    (js-utils/deepReadDir dir (if (boolean? flat?) flat? true))))
+
 (defmethod handle :unlink [_window [_ repo-dir path]]
-  (if (plugin/dotdir-file? path)
+  (if (or (plugin/dotdir-file? path)
+          (plugin/assetsdir-file? path))
     (fs/unlinkSync path)
     (try
       (logger/info ::unlink {:path path})
@@ -374,8 +379,10 @@
 
 (defmethod handle :getAssetsFiles [^js win [_ {:keys [exts]}]]
   (when-let [graph-path (state/get-window-graph-path win)]
-    (p/let [^js files (js-utils/getAllFiles (.join path graph-path "assets") (clj->js exts))]
-      files)))
+    (when-let [assets-path (.join path graph-path "assets")]
+      (when (fs-extra/pathExistsSync assets-path)
+        (p/let [^js files (js-utils/getAllFiles assets-path (clj->js exts))]
+          files)))))
 
 (defn close-watcher-when-orphaned!
   "When it's the last window for the directory, close the watcher."

src/electron/electron/plugin.cljs

@@ -23,6 +23,10 @@
   [file]
   (and file (string/starts-with? (path/normalize file) cfgs/dot-root)))
 
+(defn assetsdir-file?
+  [file]
+  (and (string? file) (string/includes? file "assets/storages")))
+
 ;; Get a release by tag name: /repos/{owner}/{repo}/releases/tags/{tag}
 ;; Get the latest release: /repos/{owner}/{repo}/releases/latest
 ;; Zipball https://api.github.com/repos/{owner}/{repo}/zipball

src/electron/electron/utils.cljs

@@ -35,7 +35,7 @@
 
 (defn get-ls-dotdir-root
   []
-  (let [lg-dir (str (.getPath app "home") "/.logseq")]
+  (let [lg-dir (path/join (.getPath app "home") ".logseq")]
     (if-not (fs/existsSync lg-dir)
       (do (fs/mkdirSync lg-dir) lg-dir)
       lg-dir)))

src/electron/electron/utils.js

@@ -1,5 +1,5 @@
-import path from "path";
-import fs from 'fs';
+import path from 'path'
+import fse from 'fs-extra'
 
 // workaround from https://github.com/electron/electron/issues/426#issuecomment-658901422
 // We set an intercept on incoming requests to disable x-frame-options
@@ -20,13 +20,13 @@ export const disableXFrameOptions = (win) => {
 }
 
 export async function getAllFiles (dir, exts) {
-  const dirents = await readdir(dir, { withFileTypes: true })
+  const dirents = await fse.readdir(dir, { withFileTypes: true })
 
-  if (exts) {
+  if (exts != null) {
     !Array.isArray(exts) && (exts = [exts])
 
     exts = exts.map(it => {
-      if (it && !it.startsWith('.')) {
+      if (typeof it === 'string' && it !== '' && !it.startsWith('.')) {
         it = '.' + it
       }
 
@@ -35,12 +35,18 @@ export async function getAllFiles (dir, exts) {
   }
 
   const files = await Promise.all(dirents.map(async (dirent) => {
-    if (exts && !exts.includes(path.extname(dirent.name))) {
+    const filePath = path.resolve(dir, dirent.name)
+
+    if (dirent.isDirectory()) {
+      return getAllFiles(filePath, exts)
+    }
+
+    if (exts && !exts.includes(path.extname(dirent.name)?.toLowerCase())) {
       return null
     }
 
-    const filePath = path.resolve(dir, dirent.name)
-    const fileStats = await lstat(filePath)
+    const fileStats = await fse.lstat(filePath)
+
     const stats = {
       size: fileStats.size,
       accessTime: fileStats.atimeMs,
@@ -48,9 +54,23 @@ export async function getAllFiles (dir, exts) {
       changeTime: fileStats.ctimeMs,
       birthTime: fileStats.birthtimeMs
     }
-    return dirent.isDirectory() ? getAllFiles(filePath) : {
-      path: filePath, ...stats
-    }
+
+    return { path: filePath, ...stats }
   }))
   return files.flat().filter(it => it != null)
 }
+
+export async function deepReadDir (dirPath, flat = true) {
+  const ret = await Promise.all(
+    (await fse.readdir(dirPath)).map(async (entity) => {
+      const root = path.join(dirPath, entity)
+      return (await fse.lstat(root)).isDirectory() ? await deepReadDir(root) : root
+    })
+  )
+
+  if (flat) {
+    return ret?.flat()
+  }
+
+  return ret
+}

src/main/frontend/config.cljs

@@ -420,6 +420,12 @@
     (string/replace
      source "../assets" (util/format "%s://%s/assets" protocol (get-repo-dir (state/get-current-repo))))))
 
+(defn get-current-repo-assets-root
+  []
+  (when-let [repo-root (and (local-db? (state/get-current-repo))
+                            (get-repo-dir (state/get-current-repo)))]
+    (util/node-path.join repo-root "assets")))
+
 (defn get-custom-js-path
   ([]
    (get-custom-js-path (state/get-current-repo)))

src/main/logseq/api.cljs

@@ -147,86 +147,136 @@
           path (util/node-path.join path "package.json")]
       (fs/write-file! repo "" path (js/JSON.stringify data nil 2) {:skip-compare? true}))))
 
+(defn ^:private write_rootdir_file
+  [file content sub-root root-dir]
+  (p/let [repo           ""
+          path           (util/node-path.join root-dir sub-root)
+          exist?         (fs/file-exists? path "")
+          _              (when-not exist? (fs/mkdir-recur! path))
+          user-path      (util/node-path.join path file)
+          sub-dir?       (string/starts-with? user-path path)
+          _              (when-not sub-dir?
+                           (log/info :debug user-path)
+                           (throw "write file denied"))
+          user-path-root (util/node-path.dirname user-path)
+          exist?         (fs/file-exists? user-path-root "")
+          _              (when-not exist? (fs/mkdir-recur! user-path-root))
+          _              (fs/write-file! repo "" user-path content {:skip-compare? true})]
+    user-path))
+
 (defn ^:private write_dotdir_file
   [file content sub-root]
-  (p/let [repo ""
-          path (plugin-handler/get-ls-dotdir-root)
-          path (util/node-path.join path sub-root)
-          exist? (fs/file-exists? path "")
-          _ (when-not exist? (fs/mkdir-recur! path))
+  (some-> (plugin-handler/get-ls-dotdir-root)
+          (p/then #(write_rootdir_file file content sub-root %))))
+
+(defn ^:private write_assetsdir_file
+  [file content sub-root]
+  (if-let [assets-dir (config/get-current-repo-assets-root)]
+    (write_rootdir_file file content sub-root assets-dir)
+    false))
+
+(defn ^:private read_rootdir_file
+  [file sub-root root-dir]
+  (p/let [path      (util/node-path.join root-dir sub-root)
           user-path (util/node-path.join path file)
-          sub-dir? (string/starts-with? user-path path)
-          _ (when-not sub-dir?
-              (log/info :debug user-path)
-              (throw "write file denied"))
-          user-path-root (util/node-path.dirname user-path)
-          exist? (fs/file-exists? user-path-root "")
-          _ (when-not exist? (fs/mkdir-recur! user-path-root))
-          _ (fs/write-file! repo "" user-path content {:skip-compare? true})]
-    user-path))
+          sub-dir?  (string/starts-with? user-path path)
+          _         (when-not sub-dir? (log/info :debug user-path) (throw "read file denied"))
+          exist?    (fs/file-exists? "" user-path)
+          _         (when-not exist? (log/info :debug user-path) (throw "file not existed"))
+          content   (fs/read-file "" user-path)]
+    content))
 
 (defn ^:private read_dotdir_file
   [file sub-root]
-  (p/let [path (plugin-handler/get-ls-dotdir-root)
-          path (util/node-path.join path sub-root)
+  (some-> (plugin-handler/get-ls-dotdir-root)
+          (p/then #(read_rootdir_file file sub-root %))))
+
+(defn ^:private read_assetsdir_file
+  [file sub-root]
+  (when-let [root-dir (config/get-current-repo-assets-root)]
+    (read_rootdir_file file sub-root root-dir)))
+
+(defn ^:private unlink_rootdir_file!
+  [file sub-root root-dir]
+  (p/let [repo      ""
+          path      (util/node-path.join root-dir sub-root)
           user-path (util/node-path.join path file)
-          sub-dir? (string/starts-with? user-path path)
-          _ (when-not sub-dir? (log/info :debug user-path) (throw "read file denied"))
-          exist? (fs/file-exists? "" user-path)
-          _ (when-not exist? (log/info :debug user-path) (throw "file not existed"))
-          content (fs/read-file "" user-path)]
-    content))
+          sub-dir?  (string/starts-with? user-path path)
+          _         (when-not sub-dir? (log/info :debug user-path) (throw "access file denied"))
+          exist?    (fs/file-exists? "" user-path)
+          _         (when-not exist? (log/info :debug user-path) (throw "file not existed"))
+          _         (fs/unlink! repo user-path {})]))
 
 (defn ^:private unlink_dotdir_file!
   [file sub-root]
-  (p/let [repo ""
-          path (plugin-handler/get-ls-dotdir-root)
-          path (util/node-path.join path sub-root)
-          user-path (util/node-path.join path file)
-          sub-dir? (string/starts-with? user-path path)
-          _ (when-not sub-dir? (log/info :debug user-path) (throw "access file denied"))
-          exist? (fs/file-exists? "" user-path)
-          _ (when-not exist? (log/info :debug user-path) (throw "file not existed"))
-          _ (fs/unlink! repo user-path {})]))
+  (some-> (plugin-handler/get-ls-dotdir-root)
+          (p/then #(unlink_rootdir_file! file sub-root %))))
+
+(defn ^:private unlink_assetsdir_file!
+  [file sub-root]
+  (when-let [root-dir (config/get-current-repo-assets-root)]
+    (unlink_rootdir_file! file sub-root root-dir)))
 
 (def ^:export write_user_tmp_file
   (fn [file content]
     (write_dotdir_file file content "tmp")))
 
 (def ^:export write_plugin_storage_file
-  (fn [plugin-id file content]
-    (write_dotdir_file
-      file content
-      (let [plugin-id (util/node-path.basename plugin-id)]
-        (util/node-path.join "storages" plugin-id)))))
+  (fn [plugin-id file content assets?]
+    (let [plugin-id (util/node-path.basename plugin-id)
+          sub-root  (util/node-path.join "storages" plugin-id)]
+      (if (true? assets?)
+        (write_assetsdir_file file content sub-root)
+        (write_dotdir_file file content sub-root)))))
 
 (def ^:export read_plugin_storage_file
-  (fn [plugin-id file]
-    (let [plugin-id (util/node-path.basename plugin-id)]
-      (read_dotdir_file
-        file (util/node-path.join "storages" plugin-id)))))
+  (fn [plugin-id file assets?]
+    (let [plugin-id (util/node-path.basename plugin-id)
+          sub-root (util/node-path.join "storages" plugin-id)]
+      (if (true? assets?)
+        (read_assetsdir_file file sub-root)
+        (read_dotdir_file file sub-root)))))
 
 (def ^:export unlink_plugin_storage_file
-  (fn [plugin-id file]
-    (let [plugin-id (util/node-path.basename plugin-id)]
-      (unlink_dotdir_file!
-        file (util/node-path.join "storages" plugin-id)))))
+  (fn [plugin-id file assets?]
+    (let [plugin-id (util/node-path.basename plugin-id)
+          sub-root (util/node-path.join "storages" plugin-id)]
+      (if (true? assets?)
+        (unlink_assetsdir_file! file sub-root)
+        (unlink_dotdir_file! file sub-root)))))
 
 (def ^:export exist_plugin_storage_file
-  (fn [plugin-id file]
-    (p/let [root (plugin-handler/get-ls-dotdir-root)
+  (fn [plugin-id file assets?]
+    (p/let [root      (if (true? assets?)
+                        (config/get-current-repo-assets-root)
+                        (plugin-handler/get-ls-dotdir-root))
             plugin-id (util/node-path.basename plugin-id)
-            exist? (fs/file-exists?
-                     (util/node-path.join root "storages" plugin-id)
-                     file)]
+            exist?    (fs/file-exists?
+                       (util/node-path.join root "storages" plugin-id)
+                       file)]
       exist?)))
 
 (def ^:export clear_plugin_storage_files
-  (fn [plugin-id]
-    (p/let [root (plugin-handler/get-ls-dotdir-root)
+  (fn [plugin-id assets?]
+    (p/let [root      (if (true? assets?)
+                        (config/get-current-repo-assets-root)
+                        (plugin-handler/get-ls-dotdir-root))
             plugin-id (util/node-path.basename plugin-id)]
       (fs/rmdir! (util/node-path.join root "storages" plugin-id)))))
 
+(def ^:export list_plugin_storage_files
+  (fn [plugin-id assets?]
+    (p/let [root       (if (true? assets?)
+                         (config/get-current-repo-assets-root)
+                         (plugin-handler/get-ls-dotdir-root))
+            plugin-id  (util/node-path.basename plugin-id)
+            files-path (util/node-path.join root "storages" plugin-id)
+            ^js files  (ipc/ipc :listdir files-path)]
+      (when (js-iterable? files)
+        (bean/->js
+         (map #(some-> (string/replace-first % files-path "")
+                       (string/replace #"^/+" "")) files))))))
+
 (def ^:export load_user_preferences
   (fn []
     (p/let [repo ""