fix: track changes in values editor

+ move storage to bg and add Storage command so that all changes are going through the cache which is staggeringly fast due to direct same-process calls

src/background/index.js

@@ -6,7 +6,7 @@ import * as tld from '@/common/tld';
 import * as sync from './sync';
 import { commands } from './utils';
 import { getData, getSizes, checkRemove } from './utils/db';
-import { initialize } from './utils/init';
+import { extensionOrigin, initialize } from './utils/init';
 import { getOption, hookOptions } from './utils/options';
 import './utils/clipboard';
 import './utils/hotkeys';
@@ -67,10 +67,25 @@ const commandsToSyncIfTruthy = [
   'CheckRemove',
   'CheckUpdate',
 ];
+const commandsForSelf = [
+  // TODO: maybe just add a prefix for all content-exposed commands?
+  ...commandsToSync,
+  ...commandsToSyncIfTruthy,
+  'ExportZip',
+  'GetAllOptions',
+  'GetData',
+  'GetSizes',
+  'GetOptions',
+  'SetOptions',
+  'SetValueStores',
+  'Storage',
+];
 
-async function handleCommandMessage(req, src) {
-  const { cmd } = req;
-  const res = await commands[cmd]?.(req.data, src);
+async function handleCommandMessage({ cmd, data } = {}, src) {
+  if (src && src.origin !== extensionOrigin && commandsForSelf.includes(cmd)) {
+    throw `Command is only allowed in extension context: ${cmd}`;
+  }
+  const res = await commands[cmd]?.(data, src);
   if (commandsToSync.includes(cmd)
   || res && commandsToSyncIfTruthy.includes(cmd)) {
     sync.sync();

src/background/utils/db.js

@@ -4,7 +4,6 @@ import {
 } from '@/common';
 import { ICON_PREFIX, INJECT_PAGE, INJECT_AUTO, TIMEOUT_WEEK } from '@/common/consts';
 import { deepSize, forEachEntry, forEachKey, forEachValue } from '@/common/object';
-import storage from '@/common/storage';
 import pluginEvents from '../plugin/events';
 import { getNameURI, parseMeta, newScript, getDefaultCustom } from './script';
 import { testScript, testBlacklist } from './tester';
@@ -12,6 +11,7 @@ import { preInitialize } from './init';
 import { commands } from './message';
 import patchDB from './patch-db';
 import { setOption } from './options';
+import storage from './storage';
 
 export const store = {
   /** @type VMScript[] */
@@ -41,7 +41,7 @@ Object.assign(commands, {
   },
   /** @return {Promise<string>} */
   GetScriptCode(id) {
-    return storage.code.getOne(id);
+    return storage.code[Array.isArray(id) ? 'getMulti' : 'getOne'](id);
   },
   GetScriptVer(opts) {
     const script = getScript(opts);

src/background/utils/init.js

@@ -1,4 +1,5 @@
 export const extensionRoot = browser.runtime.getURL('/');
+export const extensionOrigin = extensionRoot.slice(0, -1);
 
 export const preInitialize = [];
 export const postInitialize = [];

src/background/utils/options.js

@@ -1,9 +1,9 @@
 import { debounce, ensureArray, initHooks, normalizeKeys } from '@/common';
 import { deepCopy, deepEqual, mapEntry, objectGet, objectSet } from '@/common/object';
 import defaults from '@/common/options-defaults';
-import storage from '@/common/storage';
 import { preInitialize } from './init';
 import { commands } from './message';
+import storage from './storage';
 
 Object.assign(commands, {
   /** @return {Object} */

src/background/utils/patch-db.js

@@ -1,5 +1,5 @@
-import storage from '@/common/storage';
 import { parseMeta } from './script';
+import storage from './storage';
 
 export default () => new Promise((resolve, reject) => {
   console.info('Upgrade database...');

src/background/utils/popup-tracker.js

@@ -16,6 +16,7 @@ postInitialize.push(() => {
 
 function onPopupOpened(port) {
   const tabId = +port.name;
+  if (!tabId) return;
   popupTabs[tabId] = 1;
   sendTabCmd(tabId, 'PopupShown', true);
   port.onDisconnect.addListener(onPopupClosed);

src/background/utils/preinject.js

@@ -5,7 +5,6 @@ import {
 } from '@/common/consts';
 import initCache from '@/common/cache';
 import { forEachEntry, objectPick, objectSet } from '@/common/object';
-import storage from '@/common/storage';
 import ua from '@/common/ua';
 import { getScriptsByURL, ENV_CACHE_KEYS, ENV_REQ_KEYS, ENV_SCRIPTS, ENV_VALUE_IDS } from './db';
 import { extensionRoot, postInitialize } from './init';
@@ -13,6 +12,7 @@ import { commands } from './message';
 import { getOption, hookOptions } from './options';
 import { popupTabs } from './popup-tracker';
 import { clearRequestsByTabId } from './requests';
+import storage from './storage';
 import { clearStorageCache, onStorageChanged } from './storage-cache';
 import { addValueOpener, clearValueOpener } from './values';
 

src/background/utils/requests-core.js

@@ -1,11 +1,10 @@
 import { buffer2string, getUniqId, isEmpty, noop } from '@/common';
 import { forEachEntry } from '@/common/object';
 import ua from '@/common/ua';
-import { extensionRoot } from './init';
+import { extensionOrigin } from './init';
 
 let encoder;
 
-const VM_ORIGIN = extensionRoot.slice(0, -1);
 export const VM_VERIFY = getUniqId('VM-Verify');
 /** @typedef {{
   anonymous: boolean,
@@ -60,7 +59,7 @@ const headersToInject = {};
 const isVmVerify = header => header.name === VM_VERIFY;
 const isNotCookie = header => !/^cookie2?$/i.test(header.name);
 const isSendable = header => !isVmVerify(header)
-  && !(/^origin$/i.test(header.name) && header.value === VM_ORIGIN);
+  && !(/^origin$/i.test(header.name) && header.value === extensionOrigin);
 const isSendableAnon = header => isSendable(header) && isNotCookie(header);
 const SET_COOKIE_RE = /^set-cookie2?$/i;
 const SET_COOKIE_VALUE_RE = /^\s*(?:__(Secure|Host)-)?([^=\s]+)\s*=\s*(")?([!#-+\--:<-[\]-~]*)\3(.*)/;

src/background/utils/storage-cache.js

@@ -1,13 +1,17 @@
-import { debounce, initHooks, isEmpty } from '@/common';
+import { debounce, ensureArray, initHooks, isEmpty } from '@/common';
 import initCache from '@/common/cache';
+import { WATCH_STORAGE } from '@/common/consts';
 import { deepCopy, deepCopyDiff, forEachEntry } from '@/common/object';
-import storage from '@/common/storage';
 import { store } from './db';
+import storage from './storage';
 
 /** Throttling browser API for `storage.value`, processing requests sequentially,
  so that we can supersede an earlier chained request if it's obsolete now,
  e.g. in a chain like [GET:foo, SET:foo=bar] `bar` will be used in GET. */
 let valuesToFlush = {};
+/** @type {Object<string,function[]>} */
+let valuesToWatch = {};
+const watchers = {};
 /** Reading the entire db in init/vacuum/sizing shouldn't be cached for long. */
 const TTL_SKIM = 5e3;
 /** Keeping data for long time since chrome.storage.local is insanely slow in Chrome,
@@ -114,6 +118,45 @@ storage.api = {
   },
 };
 
+window[WATCH_STORAGE] = fn => {
+  const id = performance.now();
+  watchers[id] = fn;
+  return id;
+};
+browser.runtime.onConnect.addListener(port => {
+  if (!port.name.startsWith(WATCH_STORAGE)) return;
+  const { id, cfg } = JSON.parse(port.name.slice(WATCH_STORAGE.length));
+  const fn = id ? watchers[id] : port.postMessage.bind(port);
+  watchStorage(fn, cfg);
+  port.onDisconnect.addListener(() => {
+    watchStorage(fn, cfg, false);
+    delete watchers[id];
+  });
+});
+
+function watchStorage(fn, cfg, state = true) {
+  if (state && !valuesToWatch) {
+    valuesToWatch = {};
+  }
+  cfg::forEachEntry(([area, ids]) => {
+    const { prefix } = storage[area];
+    for (const id of ensureArray(ids)) {
+      const key = prefix + id;
+      const list = valuesToWatch[key] || state && (valuesToWatch[key] = []);
+      const i = list ? list.indexOf(fn) : -1;
+      if (i >= 0 && !state) {
+        list.splice(i, 1);
+        if (!list.length) delete valuesToWatch[key];
+      } else if (i < 0 && state) {
+        list.push(fn);
+      }
+    }
+  });
+  if (isEmpty(valuesToWatch)) {
+    valuesToWatch = null;
+  }
+}
+
 function batch(state) {
   cache.batch(state);
   dbKeys.batch(state);
@@ -127,11 +170,28 @@ function updateScriptMap(key, val) {
   }
 }
 
-function flush() {
+async function flush() {
   const keys = Object.keys(valuesToFlush);
   const toRemove = keys.filter(key => !valuesToFlush[key] && delete valuesToFlush[key]);
-  if (!isEmpty(valuesToFlush)) api.set(valuesToFlush);
-  if (toRemove.length) api.remove(toRemove);
+  const toFlush = valuesToFlush;
   valuesToFlush = {};
+  if (!isEmpty(toFlush)) await api.set(toFlush);
+  if (toRemove.length) await api.remove(toRemove);
+  if (valuesToWatch) setTimeout(notifyWatchers, 0, toFlush, toRemove);
   fire({ keys });
 }
+
+function notifyWatchers(toFlush, toRemove) {
+  const byFn = new Map();
+  let newValue;
+  let changes;
+  for (const key in valuesToWatch) {
+    if ((newValue = toFlush[key]) || toRemove.includes(key)) {
+      for (const fn of valuesToWatch[key]) {
+        if (!(changes = byFn.get(fn))) byFn.set(fn, changes = {});
+        changes[key] = { newValue };
+      }
+    }
+  }
+  byFn.forEach((val, fn) => fn(val));
+}

src/background/utils/storage-fetch.js

@@ -1,5 +1,5 @@
 import { isDataUri, makeRaw, request } from '@/common';
-import storage from '@/common/storage';
+import storage from './storage';
 
 /** @type { function(url, options, check): Promise<void> } or throws on error */
 storage.cache.fetch = cacheOrFetch({

src/common/storage.js → src/background/utils/storage.js

@@ -1,5 +1,6 @@
 import { mapEntry } from '@/common/object';
-import { ensureArray } from './util';
+import { ensureArray } from '@/common/util';
+import { commands } from './message';
 
 let api = browser.storage.local;
 
@@ -84,3 +85,9 @@ storage::mapEntry((val, name) => {
   }
 });
 export default storage;
+
+Object.assign(commands, {
+  Storage([area, method, ...args]) {
+    return storage[area][method](...args);
+  },
+});

src/background/utils/values.js

@@ -1,8 +1,8 @@
 import { isEmpty, sendTabCmd } from '@/common';
 import { forEachEntry, objectGet, objectSet } from '@/common/object';
-import storage from '@/common/storage';
 import { getScript } from './db';
 import { commands } from './message';
+import storage from './storage';
 
 const nest = (obj, key) => obj[key] || (obj[key] = {}); // eslint-disable-line no-return-assign
 /** { scriptId: { tabId: { frameId: {key: raw}, ... }, ... } } */

src/common/consts.js

@@ -21,6 +21,7 @@ export const INJECT_MAPPING = {
 export const METABLOCK_RE = /(?:^|\n)\s*\/\/\x20==UserScript==([\s\S]*?\n)\s*\/\/\x20==\/UserScript==|$/;
 export const ICON_PREFIX = '/public/images/icon';
 export const INJECTABLE_TAB_URL_RE = /^(https?|file|ftps?):/;
+export const WATCH_STORAGE = 'watchStorage';
 
 // `browser` is a local variable since we remove the global `chrome` and `browser` in injected*
 // to prevent exposing them to userscripts with `@inject-into content`

src/common/ui/code.vue

@@ -73,11 +73,10 @@ import 'codemirror/addon/hint/anyword-hint';
 import CodeMirror from 'codemirror';
 import Tooltip from 'vueleton/lib/tooltip/bundle';
 import ToggleButton from '@/common/ui/toggle-button';
-import { debounce, getUniqId, i18n } from '@/common';
+import { debounce, getUniqId, i18n, sendCmdDirectly } from '@/common';
 import { deepEqual, forEachEntry, objectPick } from '@/common/object';
 import hookSetting from '@/common/hook-setting';
 import options from '@/common/options';
-import storage from '@/common/storage';
 import './code-autocomplete';
 import { killTrailingSpaces } from './code-trailing-spaces';
 
@@ -557,10 +556,11 @@ export default {
       });
       userOpts = newUserOpts;
     });
-    storage.base.getOne('editorSearch').then(prev => {
+    sendCmdDirectly('Storage', ['base', 'getOne', 'editorSearch']).then(prev => {
       const { search } = this;
       const saveSearchLater = debounce(() => {
-        storage.base.setOne('editorSearch', objectPick(search, ['query', 'replace', 'options']));
+        sendCmdDirectly('Storage', ['base', 'setOne', 'editorSearch',
+          objectPick(search, ['query', 'replace', 'options'])]);
       }, 500);
       const searchAgain = () => {
         saveSearchLater();

src/common/ui/externals.vue

@@ -34,9 +34,8 @@
 </template>
 
 <script>
-import { dataUri2text, formatByteLength, makeDataUri } from '@/common';
+import { dataUri2text, formatByteLength, makeDataUri, sendCmdDirectly } from '@/common';
 import VmCode from '@/common/ui/code';
-import storage from '@/common/storage';
 
 export default {
   props: ['value', 'cmOptions', 'commands', 'install', 'errors'],
@@ -83,7 +82,7 @@ export default {
           raw = install.deps[depsUrl];
         } else {
           const key = this.value.custom.pathMap?.[url] || url;
-          raw = await storage[isReq ? 'require' : 'cache'].getOne(key);
+          raw = await sendCmdDirectly('Storage', [isReq ? 'require' : 'cache', 'getOne', key]);
           if (!isReq) raw = makeDataUri(raw, key);
         }
         if (isReq || !raw) {

src/options/views/edit/values.vue

@@ -91,16 +91,15 @@
 <script>
 import { dumpScriptValue, formatByteLength, isEmpty, sendCmdDirectly } from '@/common';
 import { handleTabNavigation, keyboardService } from '@/common/keyboard';
-import { deepEqual, mapEntry } from '@/common/object';
+import { deepCopy, deepEqual, mapEntry } from '@/common/object';
+import { WATCH_STORAGE } from '@/common/consts';
 import Icon from '@/common/ui/icon';
-import storage from '@/common/storage';
 import { showMessage } from '@/common/ui';
 import { store } from '../../utils';
 
 const PAGE_SIZE = 25;
 const MAX_LENGTH = 1024;
 const MAX_JSON_DURATION = 10; // ms
-let scriptStorageKey;
 let focusedElement;
 
 const reparseJson = (str) => {
@@ -151,12 +150,11 @@ export default {
     active(val) {
       if (val) {
         (this.current ? this.$refs.value : focusedElement)?.focus();
-        storage.value.getOne(this.script.props.id).then(data => {
+        sendCmdDirectly('Storage', ['value', 'getOne', this.script.props.id]).then(data => {
           if (!this.values && this.setData(data) && this.keys.length) {
             this.autofocus(true);
           }
         });
-        scriptStorageKey = storage.value.prefix + this.script?.props.id;
         this.disposeList = [
           keyboardService.register('pageup', () => flipPage(this, -1), conditionNotEdit),
           keyboardService.register('pagedown', () => flipPage(this, 1), conditionNotEdit),
@@ -164,7 +162,21 @@ export default {
       } else {
         this.disposeList?.forEach(dispose => dispose());
       }
-      browser.storage.onChanged[`${val ? 'add' : 'remove'}Listener`](this.onStorageChanged);
+      // toggle storage watcher
+      if (val) {
+        const fn = this.onStorageChanged;
+        const bg = browser.extension.getBackgroundPage();
+        this[WATCH_STORAGE] = browser.runtime.connect({
+          name: WATCH_STORAGE + JSON.stringify({
+            cfg: { value: this.script.props.id },
+            id: bg?.[WATCH_STORAGE](fn),
+          }),
+        });
+        if (!bg) this[WATCH_STORAGE].onMessage.addListener(fn);
+      } else {
+        this[WATCH_STORAGE]?.disconnect();
+        this[WATCH_STORAGE] = null;
+      }
     },
     current(val, oldVal) {
       if (val) {
@@ -222,7 +234,8 @@ export default {
       }
     },
     calcSize() {
-      store.storageSize = this.keys.reduce((sum, key) => sum + this.values[key].length - 1, 0);
+      store.storageSize = this.keys.reduce((sum, key) => sum
+        + key.length + 4 + this.values[key].length + 2, 2);
     },
     updateValue({
       key,
@@ -319,13 +332,13 @@ export default {
       current.jsonPaused = performance.now() - t0 > MAX_JSON_DURATION;
     },
     onStorageChanged(changes) {
-      const data = changes[scriptStorageKey]?.newValue;
+      const data = Object.values(changes)[0].newValue;
       if (data) {
         const { current } = this;
         const currentKey = current?.key;
         const valueGetter = current && (currentKey ? this.getValue : this.getValueAll);
         const oldText = valueGetter && valueGetter(currentKey);
-        this.setData(data);
+        this.setData(data instanceof Object ? data : deepCopy(data));
         if (current) {
           const newText = valueGetter(currentKey);
           const curText = current.value;
@@ -341,7 +354,7 @@ export default {
           }
         }
       } else {
-        store.storageSize = 0;
+        this.setData(data);
       }
     },
     onUpDown(evt) {

src/options/views/tab-installed.vue

@@ -149,7 +149,6 @@ import Icon from '@/common/ui/icon';
 import LocaleGroup from '@/common/ui/locale-group';
 import { forEachKey } from '@/common/object';
 import { setRoute, lastRoute } from '@/common/router';
-import storage from '@/common/storage';
 import { keyboardService, handleTabNavigation } from '@/common/keyboard';
 import { loadData } from '@/options';
 import ScriptItem from './script-item';
@@ -481,7 +480,7 @@ export default {
       this.debouncedUpdate();
     },
     async getCodeFromStorage(ids) {
-      const data = await storage.code.getMulti(ids);
+      const data = await sendCmdDirectly('GetScriptCode', ids);
       this.store.scripts.forEach(({ $cache, props: { id } }) => {
         if (id in data) $cache.code = data[id];
       });