feat: separately inject start/body scripts, cache for 1hr (#1354)

* treat invalid @inject-into as defaultInjectInto
* throw on non-runtime messaging errors correctly
* reuse some key names as constants for reliability

src/background/index.js

@@ -1,55 +1,35 @@
 import '#/common/browser';
 import { getActiveTab, makePause, sendCmd } from '#/common';
 import { TIMEOUT_24HOURS, TIMEOUT_MAX } from '#/common/consts';
-import { deepCopy, forEachEntry, objectSet } from '#/common/object';
+import { deepCopy } from '#/common/object';
 import * as tld from '#/common/tld';
 import ua from '#/common/ua';
 import * as sync from './sync';
 import { commands } from './utils';
-import cache from './utils/cache';
 import { getData, checkRemove } from './utils/db';
-import { setBadge } from './utils/icon';
 import { initialize } from './utils/init';
 import { getOption, hookOptions } from './utils/options';
+import { popupTabs } from './utils/popup-tracker';
 import { getInjectedScripts } from './utils/preinject';
 import { SCRIPT_TEMPLATE, resetScriptTemplate } from './utils/template-hook';
 import { resetValueOpener, addValueOpener } from './utils/values';
 import { clearRequestsByTabId } from './utils/requests';
 import './utils/clipboard';
 import './utils/hotkeys';
+import './utils/icon';
 import './utils/notifications';
 import './utils/script';
 import './utils/tabs';
 import './utils/tester';
 import './utils/update';
 
-let isApplied;
-const expose = {};
-
-const optionHandlers = {
-  autoUpdate,
-  expose(val) {
-    val::forEachEntry(([site, isExposed]) => {
-      expose[decodeURIComponent(site)] = isExposed;
-    });
-  },
-  isApplied(val) {
-    isApplied = val;
-  },
-  [SCRIPT_TEMPLATE](val, changes) {
-    resetScriptTemplate(changes);
-  },
-};
-
 hookOptions((changes) => {
-  changes::forEachEntry(function processChange([key, value]) {
-    const handler = optionHandlers[key];
-    if (handler) {
-      handler(value, changes);
-    } else if (key.includes('.')) {
-      objectSet({}, key, value)::forEachEntry(processChange);
-    }
-  });
+  if ('autoUpdate' in changes) {
+    autoUpdate();
+  }
+  if (SCRIPT_TEMPLATE in changes) {
+    resetScriptTemplate(changes);
+  }
   sendCmd('UpdateOptions', changes);
 });
 
@@ -62,29 +42,20 @@ Object.assign(commands, {
   },
   /** @return {Promise<Object>} */
   async GetInjected(_, src) {
-    const { frameId, tab, url } = src;
+    const { frameId, url, tab: { id: tabId } } = src;
     if (!frameId) {
-      resetValueOpener(tab.id);
-      clearRequestsByTabId(tab.id);
+      resetValueOpener(tabId);
+      clearRequestsByTabId(tabId);
     }
-    const res = {
-      expose: !frameId && url.startsWith('https://') && expose[url.split('/', 3)[2]],
-    };
-    if (isApplied) {
-      const data = await getInjectedScripts(url, tab.id, frameId);
-      addValueOpener(tab.id, frameId, data.withValueIds);
-      const badgeData = [data.enabledIds, src];
-      setBadge(...badgeData);
-      // FF bug: the badge is reset because sometimes tabs get their real/internal url later
-      if (ua.isFirefox) cache.put(`badge:${tab.id}${url}`, badgeData);
-      Object.assign(res, data.inject);
-      // Injecting known content mode scripts without waiting for InjectionFeedback
-      const inContent = res.scripts.map(s => !s.code && [s.dataKey, true]).filter(Boolean);
-      if (inContent.length) {
-        // executeScript is slow (in FF at least) so this will run after the response is sent
-        Promise.resolve().then(() => commands.InjectionFeedback(inContent, src));
-      }
+    const res = await getInjectedScripts(url, tabId, frameId);
+    const { feedback, valOpIds } = res._tmp;
+    res.isPopupShown = popupTabs[tabId];
+    // Injecting known content scripts without waiting for InjectionFeedback message.
+    // Running in a separate task because it may take a long time to serialize data.
+    if (feedback.length) {
+      setTimeout(commands.InjectionFeedback, 0, { feedback }, src);
     }
+    addValueOpener(tabId, frameId, valOpIds);
     return res;
   },
   /** @return {Promise<Object>} */
@@ -154,7 +125,6 @@ initialize(() => {
         handleCommandMessage(...args).catch(e => { throw e instanceof Error ? e : new Error(e); }))
       : handleCommandMessage,
   );
-  ['expose', 'isApplied'].forEach(key => optionHandlers[key](getOption(key)));
   setTimeout(autoUpdate, 2e4);
   sync.initialize();
   checkRemove();

src/background/utils/cache.js

@@ -2,7 +2,10 @@ import initCache from '#/common/cache';
 import { commands } from './message';
 
 const cache = initCache({
-  lifetime: 5 * 60 * 1000,
+  /* Keeping the data for one hour since chrome.storage.local is insanely slow in Chrome,
+     it can takes seconds to read it when injecting tabs with a big script/value, which delays
+     all other scripts in this tab and they will never be able to run at document-start. */
+  lifetime: 60 * 60 * 1000,
 });
 
 Object.assign(commands, {

src/background/utils/db.js

@@ -15,9 +15,10 @@ import { commands } from './message';
 import patchDB from './patch-db';
 import { setOption } from './options';
 import './storage-fetch';
+import dataCache from './cache';
 
 const store = {};
-
+storage.base.setDataCache(dataCache);
 storage.script.onDump = (item) => {
   store.scriptMap[item.props.id] = item;
 };
@@ -101,6 +102,7 @@ preInitialize.push(async () => {
   /** @this VMScriptCustom.pathMap */
   const rememberUrl = function _(url) { resUrls.push(this[url] || url); };
   data::forEachEntry(([key, script]) => {
+    dataCache.put(key, script);
     if (key.startsWith(storage.script.prefix)) {
       // {
       //   meta,
@@ -250,16 +252,13 @@ export async function dumpValueStores(valueDict) {
   return valueDict;
 }
 
-const gmValues = [
-  'GM_getValue', 'GM.getValue',
-  'GM_setValue', 'GM.setValue',
-  'GM_listValues', 'GM.listValues',
-  'GM_deleteValue', 'GM.deleteValue',
-];
-
+export const ENV_CACHE_KEYS = 'cacheKeys';
+export const ENV_REQ_KEYS = 'reqKeys';
+export const ENV_VALUE_IDS = 'valueIds';
+const GMVALUES_RE = /^GM[_.](listValues|([gs]et|delete)Value)$/;
+const RUN_AT_RE = /^document-(start|body|end|idle)$/;
 /**
  * @desc Get scripts to be injected to page with specific URL.
- * @return {Promise<Object>}
  */
 export async function getScriptsByURL(url, isTop) {
   const allScripts = testBlacklist(url)
@@ -269,50 +268,82 @@ export async function getScriptsByURL(url, isTop) {
       && (isTop || !(script.custom.noframes ?? script.meta.noframes))
       && testScript(url, script)
     ));
-  const reqKeys = [];
-  const cacheKeys = [];
-  const scripts = allScripts.filter(script => script.config.enabled);
-  scripts.forEach((script) => {
+  const disabledIds = [];
+  /** @namespace VMScriptByUrlData */
+  const [envStart, envDelayed] = [0, 1].map(() => ({
+    ids: [],
+    /** @type {VMInjectedScript[]} */
+    scripts: [],
+    [ENV_CACHE_KEYS]: [],
+    [ENV_REQ_KEYS]: [],
+    [ENV_VALUE_IDS]: [],
+  }));
+  allScripts.forEach((script) => {
+    const { id } = script.props;
+    if (!script.config.enabled) {
+      disabledIds.push(id);
+      return;
+    }
     const { meta, custom } = script;
     const { pathMap = buildPathMap(script) } = custom;
-    meta.require.forEach((key) => {
-      pushUnique(reqKeys, pathMap[key] || key);
-    });
-    meta.resources::forEachValue((key) => {
-      pushUnique(cacheKeys, pathMap[key] || key);
-    });
+    const runAt = `${custom.runAt || meta.runAt || ''}`.match(RUN_AT_RE)?.[1] || 'end';
+    const env = runAt === 'start' || runAt === 'body' ? envStart : envDelayed;
+    env.ids.push(id);
+    if (meta.grant?.some(GMVALUES_RE.test, GMVALUES_RE)) {
+      env[ENV_VALUE_IDS].push(id);
+    }
+    for (const [list, name] of [
+      [meta.require, ENV_REQ_KEYS],
+      [Object.values(meta.resources), ENV_CACHE_KEYS],
+    ]) {
+      list.forEach(key => {
+        key = pathMap[key] || key;
+        if (!envStart[name].includes(key)) {
+          env[name].push(key);
+        }
+      });
+    }
+    /** @namespace VMInjectedScript */
+    env.scripts.push({ ...script, runAt });
   });
-  const ids = allScripts.map(getPropsId);
-  const enabledIds = scripts.map(getPropsId);
-  const withValueIds = scripts
-  .filter(script => script.meta.grant?.some(gm => gmValues.includes(gm)))
-  .map(getPropsId);
-  const [require, cache, values, code] = await Promise.all([
-    storage.require.getMulti(reqKeys),
-    storage.cache.getMulti(cacheKeys),
-    storage.value.getMulti(withValueIds, {}),
-    storage.code.getMulti(enabledIds),
-  ]);
+  if (envDelayed.ids.length) {
+    envDelayed.promise = readEnvironmentData(envDelayed);
+  }
+  /** @namespace VMScriptByUrlData */
   return {
-    // these will be sent to injectScripts()
-    inject: {
-      cache,
-      ids,
-      scripts,
-    },
-    // these will be used only by bg/* and to augment the data above
-    cacheKeys,
-    code,
-    enabledIds,
-    reqKeys,
-    require,
-    values,
-    withValueIds,
+    ...envStart,
+    ...await readEnvironmentData(envStart),
+    disabledIds,
+    envDelayed,
   };
 }
 
-function pushUnique(arr, elem) {
-  if (!arr.includes(elem)) arr.push(elem);
+/**
+ * Object keys == areas in `storage` module.
+ * @namespace VMScriptByUrlData
+ */
+const STORAGE_ROUTES = Object.entries({
+  cache: ENV_CACHE_KEYS,
+  code: 'ids',
+  require: ENV_REQ_KEYS,
+  value: ENV_VALUE_IDS,
+});
+
+async function readEnvironmentData(env) {
+  const keys = [];
+  STORAGE_ROUTES.forEach(([area, srcIds]) => {
+    env[srcIds].forEach(id => {
+      keys.push(storage[area].getKey(id));
+    });
+  });
+  const data = await storage.base.getMulti(keys);
+  STORAGE_ROUTES.forEach(([area, srcIds]) => {
+    env[area] = {};
+    env[srcIds].forEach(id => {
+      env[area][id] = data[storage[area].getKey(id)];
+    });
+  });
+  return env;
 }
 
 /**

src/background/utils/icon.js

@@ -17,6 +17,7 @@ Object.assign(commands, {
     return cache.get(key)
       || cache.put(key, loadImageData(url, { base64: true }).catch(noop), CACHE_DURATION);
   },
+  SetBadge: setBadge,
 });
 
 // Firefox Android does not support such APIs, use noop
@@ -104,10 +105,6 @@ browser.tabs.onRemoved.addListener((id) => {
 
 browser.tabs.onUpdated.addListener((tabId, info, tab) => {
   const { url } = info;
-  if (url && ua.isFirefox && isApplied) {
-    const badgeData = cache.pop(`badge:${tabId}${url}`);
-    if (badgeData) setBadge(...badgeData);
-  }
   if (info.status === 'loading'
       // at least about:newtab in Firefox may open without 'loading' status
       || info.favIconUrl && tab.url.startsWith('about:')) {
@@ -115,7 +112,7 @@ browser.tabs.onUpdated.addListener((tabId, info, tab) => {
   }
 });
 
-export function setBadge(ids, { tab, frameId }) {
+function setBadge(ids, { tab, frameId }) {
   const tabId = tab.id;
   const data = badges[tabId] || {};
   if (!data.idMap || frameId === 0) {

src/background/utils/preinject.js

@@ -1,13 +1,15 @@
-import { getScriptName, getUniqId } from '#/common';
-import { INJECT_CONTENT, INJECTABLE_TAB_URL_RE, METABLOCK_RE } from '#/common/consts';
+import { getScriptName, getUniqId, hasOwnProperty } from '#/common';
+import {
+  INJECT_AUTO, INJECT_CONTENT, INJECT_MAPPING, INJECTABLE_TAB_URL_RE, METABLOCK_RE,
+} from '#/common/consts';
 import initCache from '#/common/cache';
+import { forEachEntry, objectSet } from '#/common/object';
 import storage from '#/common/storage';
 import ua from '#/common/ua';
-import { getScriptsByURL } from './db';
+import { getScriptsByURL, ENV_CACHE_KEYS, ENV_REQ_KEYS, ENV_VALUE_IDS } from './db';
 import { extensionRoot, postInitialize } from './init';
 import { commands } from './message';
 import { getOption, hookOptions } from './options';
-import { popupTabs } from './popup-tracker';
 
 const API_CONFIG = {
   urls: ['*://*/*'], // `*` scheme matches only http and https
@@ -19,44 +21,58 @@ const TIME_KEEP_DATA = 60e3; // 100ms should be enough but the tab may hang or g
 const cacheCode = initCache({ lifetime: TIME_KEEP_DATA });
 const cache = initCache({
   lifetime: TIME_KEEP_DATA,
-  async onDispose(promise) {
-    const data = await promise;
-    data.unregister?.();
-  },
+  onDispose: async promise => (await promise).rcsPromise?.unregister(),
 });
+const KEY_EXPOSE = 'expose';
+const KEY_INJECT_INTO = 'defaultInjectInto';
+const KEY_IS_APPLIED = 'isApplied';
+const expose = {};
+let isApplied;
 let injectInto;
-hookOptions(changes => {
-  injectInto = changes.defaultInjectInto ?? injectInto;
-  if ('isApplied' in changes) togglePreinject(changes.isApplied);
-});
+hookOptions(onOptionChanged);
 postInitialize.push(() => {
-  injectInto = getOption('defaultInjectInto');
-  togglePreinject(getOption('isApplied'));
+  for (const key of [KEY_EXPOSE, KEY_INJECT_INTO, KEY_IS_APPLIED]) {
+    onOptionChanged({ [key]: getOption(key) });
+  }
 });
 
 Object.assign(commands, {
-  InjectionFeedback(feedback, { tab, frameId }) {
-    feedback.forEach(([key, needsInjection]) => {
-      const code = cacheCode.pop(key);
-      // see TIME_KEEP_DATA comment
-      if (needsInjection && code) {
-        browser.tabs.executeScript(tab.id, {
-          code,
-          frameId,
-          runAt: 'document_start',
-        });
+  async InjectionFeedback({ feedId, feedback, pageInjectable }, src) {
+    feedback.forEach(processFeedback, src);
+    if (feedId) {
+      const env = await cache.pop(feedId);
+      if (env) {
+        const { scripts } = env;
+        env.forceContent = !pageInjectable;
+        scripts.map(prepareScript, env).filter(Boolean).forEach(processFeedback, src);
+        return {
+          info: { cache: env.cache },
+          scripts,
+        };
       }
-    });
+    }
   },
 });
 
-// Keys of the object returned by getScriptsByURL()
+/** @this {chrome.runtime.MessageSender} */
+function processFeedback([key, needsInjection]) {
+  const code = cacheCode.pop(key);
+  // see TIME_KEEP_DATA comment
+  if (needsInjection && code) {
+    browser.tabs.executeScript(this.tab.id, {
+      code,
+      frameId: this.frameId,
+      runAt: 'document_start',
+    });
+  }
+}
+
 const propsToClear = {
-  [storage.cache.prefix]: 'cacheKeys',
+  [storage.cache.prefix]: ENV_CACHE_KEYS,
   [storage.code.prefix]: true,
-  [storage.require.prefix]: 'reqKeys',
+  [storage.require.prefix]: ENV_REQ_KEYS,
   [storage.script.prefix]: true,
-  [storage.value.prefix]: 'withValueIds',
+  [storage.value.prefix]: ENV_VALUE_IDS,
 };
 
 browser.storage.onChanged.addListener(async changes => {
@@ -71,13 +87,37 @@ browser.storage.onChanged.addListener(async changes => {
         || data[prop]?.includes(prefix === storage.value.prefix ? +key : key);
     }));
   if (dirty) {
-    clearCache();
+    cache.destroy();
   }
 });
 
-function clearCache() {
-  cacheCode.destroy();
-  cache.destroy();
+function normalizeInjectInto(value) {
+  return INJECT_MAPPING::hasOwnProperty(value)
+    ? value
+    : injectInto || INJECT_AUTO;
+}
+
+function onOptionChanged(changes) {
+  changes::forEachEntry(([key, value]) => {
+    switch (key) {
+    case KEY_INJECT_INTO:
+      injectInto = normalizeInjectInto(value);
+      cache.destroy();
+      break;
+    case KEY_IS_APPLIED:
+      togglePreinject(value);
+      break;
+    case KEY_EXPOSE:
+      value::forEachEntry(([site, isExposed]) => {
+        expose[decodeURIComponent(site)] = isExposed;
+      });
+      break;
+    default:
+      if (key.includes('.')) { // used by `expose.url`
+        onOptionChanged(objectSet({}, key, value));
+      }
+    }
+  });
 }
 
 /** @return {Promise<Object>} */
@@ -90,16 +130,17 @@ function getKey(url, isTop) {
 }
 
 function togglePreinject(enable) {
+  isApplied = enable;
   // Using onSendHeaders because onHeadersReceived in Firefox fires *after* content scripts.
   // And even in Chrome a site may be so fast that preinject on onHeadersReceived won't be useful.
   const onOff = `${enable ? 'add' : 'remove'}Listener`;
   const config = enable ? API_CONFIG : undefined;
-  browser.webRequest.onSendHeaders[onOff](preinject, config);
-  browser.webRequest.onHeadersReceived[onOff](prolong, config);
-  clearCache();
+  browser.webRequest.onSendHeaders[onOff](onSendHeaders, config);
+  browser.webRequest.onHeadersReceived[onOff](onHeadersReceived, config);
+  cache.destroy();
 }
 
-function preinject({ url, tabId, frameId }) {
+function onSendHeaders({ url, tabId, frameId }) {
   if (!INJECTABLE_TAB_URL_RE.test(url)) return;
   const isTop = !frameId;
   const key = getKey(url, isTop);
@@ -111,34 +152,66 @@ function preinject({ url, tabId, frameId }) {
   }
 }
 
-function prolong({ url, frameId }) {
-  cache.hit(getKey(url, !frameId), TIME_AFTER_RECEIVE);
+/** @param {chrome.webRequest.WebResponseHeadersDetails} info */
+function onHeadersReceived(info) {
+  cache.hit(getKey(info.url, !info.frameId), TIME_AFTER_RECEIVE);
+}
+
+function prepare(url, tabId, frameId, isLate) {
+  /** @namespace VMGetInjectedData */
+  const res = {
+    expose: !frameId
+      && url.startsWith('https://')
+      && expose[url.split('/', 3)[2]],
+  };
+  return isApplied
+    ? prepareScripts(url, tabId, frameId, isLate, res)
+    : res;
 }
 
-async function prepare(url, tabId, frameId, isLate) {
+async function prepareScripts(url, tabId, frameId, isLate, res) {
   const data = await getScriptsByURL(url, !frameId);
-  const { inject } = data;
-  inject.scripts.forEach(prepareScript, data);
-  inject.injectInto = injectInto;
-  inject.ua = ua;
-  inject.isFirefox = ua.isFirefox;
-  inject.isPopupShown = popupTabs[tabId];
+  const { envDelayed, scripts } = data;
+  const feedback = scripts.map(prepareScript, data).filter(Boolean);
+  const more = envDelayed.promise;
+  const feedId = getUniqId(`${tabId}:${frameId}:`);
+  /** @namespace VMGetInjectedData */
+  Object.assign(res, {
+    feedId, // InjectionFeedback id for envDelayed
+    injectInto,
+    scripts,
+    hasMore: !!more, // tells content bridge to expect envDelayed
+    ids: data.disabledIds, // content bridge adds the actually running ids and sends via SetPopup
+    info: {
+      cache: data.cache,
+      isFirefox: ua.isFirefox,
+      ua,
+    },
+  });
+  Object.defineProperty(res, '_tmp', {
+    value: {
+      feedback,
+      valOpIds: [...data[ENV_VALUE_IDS], ...envDelayed[ENV_VALUE_IDS]],
+    },
+  });
   if (!isLate && browser.contentScripts) {
-    registerScriptDataFF(data, url, !!frameId);
+    registerScriptDataFF(data, res, url, !!frameId);
   }
-  return data;
+  if (more) cache.put(feedId, more);
+  return res;
 }
 
-/** @this data */
-function prepareScript(script, index, scripts) {
+/** @this {VMScriptByUrlData} */
+function prepareScript(script) {
   const { custom, meta, props } = script;
   const { id } = props;
-  const { require, values } = this;
+  const { forceContent, require, value } = this;
   const code = this.code[id];
   const dataKey = getUniqId('VMin');
   const displayName = getScriptName(script);
-  const name = encodeURIComponent(displayName.replace(/[#&',/:;?@=]/g, replaceWithFullWidthForm));
-  const isContent = (custom.injectInto || meta.injectInto || injectInto) === INJECT_CONTENT;
+  const name = encodeURIComponent(displayName.replace(/[#&',/:;?@=+]/g, replaceWithFullWidthForm));
+  const realm = normalizeInjectInto(custom.injectInto || meta.injectInto);
+  const isContent = realm === INJECT_CONTENT || forceContent && realm === INJECT_AUTO;
   const pathMap = custom.pathMap || {};
   const reqs = meta.require?.map(key => require[pathMap[key] || key]).filter(Boolean);
   // trying to avoid progressive string concatenation of potentially huge code slices
@@ -151,7 +224,6 @@ function prepareScript(script, index, scripts) {
     ...reqsSlices,
     // adding a nested IIFE to support 'use strict' in the code when there are @requires
     hasReqs ? '(()=>{' : '',
-    // TODO: move code above @require
     code,
     // adding a new line in case the code ends with a line comment
     code.endsWith('\n') ? '' : '\n',
@@ -162,14 +234,17 @@ function prepareScript(script, index, scripts) {
     `\n//# sourceURL=${extensionRoot}${ua.isFirefox ? '%20' : ''}${name}.user.js#${id}`,
   ].join('');
   cacheCode.put(dataKey, injectedCode, TIME_KEEP_DATA);
-  scripts[index] = {
-    ...script,
+  /** @namespace VMInjectedScript */
+  Object.assign(script, {
     dataKey,
     displayName,
-    code: isContent ? '' : injectedCode,
+    // code will be `true` if the desired realm is PAGE which is not injectable
+    code: isContent ? '' : forceContent || injectedCode,
+    injectInto: realm,
     metaStr: code.match(METABLOCK_RE)[1] || '',
-    values: values[id],
-  };
+    values: value[id],
+  });
+  return isContent && [dataKey, true];
 }
 
 function replaceWithFullWidthForm(s) {
@@ -178,27 +253,25 @@ function replaceWithFullWidthForm(s) {
 }
 
 const resolveDataCodeStr = `(${(data) => {
-  const { vmResolve } = window;
+  // not using `window` because this code can't reach its replacement set by guardGlobals
+  const { vmResolve } = this;
   if (vmResolve) {
     vmResolve(data);
   } else {
     // running earlier than the main content script for whatever reason
-    window.vmData = data;
+    this.vmData = data;
   }
 }})`;
 
-function registerScriptDataFF(data, url, allFrames) {
-  const promise = browser.contentScripts.register({
+// TODO: rework the whole thing to register scripts individually with real `matches`
+function registerScriptDataFF(data, inject, url, allFrames) {
+  data::forEachEntry(([key]) => delete data[key]); // releasing the contents for garbage collection
+  data.rcsPromise = browser.contentScripts.register({
     allFrames,
     js: [{
-      code: `${resolveDataCodeStr}(${JSON.stringify(data.inject)})`,
+      code: `${resolveDataCodeStr}(${JSON.stringify(inject)})`,
     }],
     matches: url.split('#', 1),
     runAt: 'document_start',
   });
-  data.unregister = async () => {
-    data.unregister = null;
-    const r = await promise;
-    r.unregister();
-  };
 }

src/common/browser.js

@@ -51,15 +51,21 @@ if (!global.browser?.runtime?.sendMessage) {
       // Using (...results) for API callbacks that return several results (we don't use them though)
       thisArg::func(...args, (...results) => {
         let err = chrome.runtime.lastError;
+        let isRuntime;
         if (err) {
           err = err.message;
+          isRuntime = true;
         } else if (preprocessorFunc) {
           err = preprocessorFunc(resolve, ...results);
         } else {
           resolve(results[0]);
         }
         // Prefer `reject` over `throw` which stops debugger in 'pause on exceptions' mode
-        if (err) reject(new Error(`${err}\n${stackInfo.stack}`));
+        if (err) {
+          err = new Error(`${err}\n${stackInfo.stack}`);
+          err.isRuntime = isRuntime;
+          reject(err);
+        }
       });
       if (process.env.DEBUG) promise.catch(err => console.warn(args, err?.message || err));
       return promise;

src/common/cache.js

@@ -15,6 +15,7 @@ export default function initCache({
   let batchStartTime;
   // eslint-disable-next-line no-return-assign
   const getNow = () => batchStarted && batchStartTime || (batchStartTime = performance.now());
+  /** @namespace VMCache */
   return {
     batch, get, getValues, pop, put, del, has, hit, destroy,
   };

src/common/index.js

@@ -105,7 +105,7 @@ export async function sendMessageRetry(payload, retries = 10) {
       const data = await sendMessage(payload);
       if (data) return data;
     } catch (e) {
-      if (typeof e === 'string') throw e; // not a connection error which is an object
+      if (!e.isRuntime) throw e;
     }
     await makePause(pauseDuration);
     pauseDuration *= 2;

src/common/object.js

@@ -62,10 +62,16 @@ export function objectPurify(obj) {
   return obj;
 }
 
+/**
+ * @param {{}} obj
+ * @param {string[]} keys
+ * @param {function(value,key):?} [transform]
+ * @returns {{}}
+ */
 export function objectPick(obj, keys, transform) {
   return keys::reduce((res, key) => {
     let value = obj?.[key];
-    if (transform) value = transform(value);
+    if (transform) value = transform(value, key);
     if (value != null) res[key] = value;
     return res;
   }, {});

src/common/storage.js

@@ -1,51 +1,85 @@
+import { deepCopy, forEachEntry } from '#/common/object';
 import { blob2base64, ensureArray } from './util';
 
+/** @type VMCache */
+let dataCache;
+const browserStorageLocal = browser.storage.local;
+const onStorageChanged = changes => {
+  changes::forEachEntry(([key, { newValue }]) => {
+    if (newValue == null) {
+      dataCache.del(key);
+    } else {
+      dataCache.put(key, newValue);
+    }
+  });
+};
+
 const base = {
   prefix: '',
+  setDataCache(val) {
+    dataCache = val;
+    browser.storage.onChanged.addListener(onStorageChanged);
+  },
   getKey(id) {
     return `${this.prefix}${id}`;
   },
-  getOne(id) {
-    const key = this.getKey(id);
-    return browser.storage.local.get(key).then(data => data[key]);
+  async getOne(id) {
+    return (await this.getMulti([id]))[id];
   },
   /**
    * @param {string[]} ids
-   * @param {?} def
-   * @param {function(id:string, val:?):?} transform
+   * @param {?} [def]
+   * @param {function(id:string, val:?):?} [transform]
    * @returns {Promise<Object>}
    */
   async getMulti(ids, def, transform) {
-    const data = await browser.storage.local.get(ids.map(this.getKey, this));
-    return ids.reduce((res, id) => {
-      const val = data[this.getKey(id)];
-      res[id] = transform ? transform(id, val) : (val || def);
-      return res;
-    }, {});
+    const res = {};
+    const data = {};
+    const missingKeys = [];
+    ids.forEach(id => {
+      const key = this.getKey(id);
+      const cached = dataCache?.get(key);
+      res[id] = key;
+      if (cached != null) {
+        data[key] = deepCopy(cached);
+      } else {
+        missingKeys.push(key);
+      }
+    });
+    if (missingKeys.length) {
+      Object.assign(data, await browserStorageLocal.get(missingKeys));
+    }
+    res::forEachEntry(([id, key]) => {
+      res[id] = transform
+        ? transform(id, data[key])
+        : data[key] ?? deepCopy(def);
+    });
+    return res;
   },
-  set(id, value) {
-    return id
-      ? browser.storage.local.set({ [this.getKey(id)]: value })
-      : Promise.resolve();
+  // Must be `async` to ensure a Promise is returned when `if` doesn't match
+  async set(id, value) {
+    if (id) return this.dump({ [id]: value });
   },
-  remove(id) {
-    return id
-      ? browser.storage.local.remove(this.getKey(id))
-      : Promise.resolve();
+  // Must be `async` to ensure a Promise is returned when `if` doesn't match
+  async remove(id) {
+    if (id) return this.removeMulti([id]);
   },
-  removeMulti(ids) {
-    return ids.length
-      ? browser.storage.local.remove(ids.map(this.getKey, this))
-      : Promise.resolve();
+  // Must be `async` to ensure a Promise is returned when `if` doesn't match
+  async removeMulti(ids) {
+    if (ids.length) {
+      const keys = ids.map(this.getKey, this);
+      if (dataCache) keys.forEach(dataCache.del);
+      return browserStorageLocal.remove(keys);
+    }
   },
   async dump(data) {
-    const output = !this.prefix
-      ? data
-      : Object.entries(data).reduce((res, [key, value]) => {
-        res[this.getKey(key)] = value;
-        return res;
-      }, {});
-    await browser.storage.local.set(output);
+    const output = {};
+    data::forEachEntry(([id, value]) => {
+      const key = this.getKey(id);
+      output[key] = value;
+      dataCache?.put(key, deepCopy(value));
+    });
+    await browserStorageLocal.set(output);
     return data;
   },
 };

src/injected/content/bridge.js

@@ -7,7 +7,8 @@ import { Error } from '../utils/helpers';
 const handlers = {};
 const bgHandlers = {};
 const bridge = {
-  ids: [],
+  ids: [], // all ids including the disabled ones for SetPopup
+  runningIds: [],
   // userscripts running in the content script context are messaged via invokeGuest
   /** @type Number[] */
   invokableIds: [],

src/injected/content/index.js

@@ -14,6 +14,7 @@ import './tabs';
 
 const IS_FIREFOX = !global.chrome.app;
 const IS_TOP = window.top === window;
+const { invokableIds } = bridge;
 const menus = {};
 let isPopupShown;
 let pendingSetPopup;
@@ -31,29 +32,26 @@ const { split } = '';
   const isXml = document instanceof XMLDocument;
   if (!isXml) injectPageSandbox(contentId, webId);
   // detecting if browser.contentScripts is usable, it was added in FF59 as well as composedPath
-  const scriptData = IS_FIREFOX && Event.prototype.composedPath
+  const data = IS_FIREFOX && Event.prototype.composedPath
     ? await getDataFF(dataPromise)
     : await dataPromise;
   // 1) bridge.post may be overridden in injectScripts
   // 2) cloneInto is provided by Firefox in content scripts to expose data to the page
   bridge.post = bindEvents(contentId, webId, bridge.onHandle, global.cloneInto);
-  bridge.isFirefox = scriptData.isFirefox;
-  bridge.injectInto = scriptData.injectInto;
-  if (scriptData.scripts) injectScripts(contentId, webId, scriptData, isXml);
-  isPopupShown = scriptData.isPopupShown;
+  bridge.ids = data.ids;
+  bridge.isFirefox = data.info.isFirefox;
+  bridge.injectInto = data.injectInto;
+  if (data.scripts) injectScripts(contentId, webId, data, isXml);
+  if (data.expose) bridge.post('Expose');
+  isPopupShown = data.isPopupShown;
   sendSetPopup();
-  // scriptData is the successor of the two ways to request scripts in Firefox,
-  // but it may not contain everything returned by `GetInjected`, for example `expose`.
-  // Use the slower but more complete `injectData` to continue.
-  const injectData = await dataPromise;
-  if (injectData.expose) bridge.post('Expose');
 })().catch(IS_FIREFOX && console.error); // Firefox can't show exceptions in content scripts
 
 bridge.addBackgroundHandlers({
   Command(data) {
     const [cmd] = data;
     const id = +cmd::split(':', 1)[0];
-    const realm = bridge.invokableIds::includes(id) && INJECT_CONTENT;
+    const realm = invokableIds::includes(id) && INJECT_CONTENT;
     bridge.post('Command', data, realm);
   },
   PopupShown(state) {
@@ -63,7 +61,6 @@ bridge.addBackgroundHandlers({
   UpdatedValues(data) {
     const dataPage = {};
     const dataContent = {};
-    const { invokableIds } = bridge;
     objectKeys(data)::forEach((id) => {
       (invokableIds::includes(+id) ? dataContent : dataPage)[id] = data[id];
     });

src/injected/content/inject.js

@@ -1,16 +1,10 @@
-import {
-  INJECT_AUTO,
-  INJECT_CONTENT,
-  INJECT_MAPPING,
-  INJECT_PAGE,
-  browser,
-} from '#/common/consts';
+import { INJECT_CONTENT, INJECT_MAPPING, INJECT_PAGE, browser } from '#/common/consts';
 import { sendCmd } from '#/common';
-import { defineProperty, forEachKey, objectPick } from '#/common/object';
+import { defineProperty, describeProperty, forEachKey } from '#/common/object';
 import {
   append, appendChild, createElementNS, elemByTag, remove, NS_HTML,
   addEventListener, document, removeEventListener,
-  log,
+  forEach, log, Promise, push, then,
 } from '../utils/helpers';
 import bridge from './bridge';
 
@@ -20,17 +14,43 @@ const VMInitInjection = window[process.env.INIT_FUNC_NAME];
 // (the prop is undeletable so a userscript can't fool us on reinjection)
 defineProperty(window, process.env.INIT_FUNC_NAME, { value: 1 });
 
-const stringIncludes = String.prototype.includes;
-
+const stringIncludes = ''.includes;
+const resolvedPromise = Promise.resolve();
+const { runningIds } = bridge;
 let contLists;
 let pgLists;
+/** @type {Object<string,VMInjectionRealm>} */
 let realms;
+/** @type boolean */
+let pageInjectable;
+let badgePromise;
+let numBadgesSent = 0;
 
 bridge.addHandlers({
   // FF bug workaround to enable processing of sourceURL in injected page scripts
   InjectList: injectList,
+  Run(id, realm) {
+    runningIds::push(id);
+    bridge.ids::push(id);
+    if (realm === INJECT_CONTENT) {
+      bridge.invokableIds::push(id);
+    }
+    if (!badgePromise) {
+      badgePromise = resolvedPromise::then(throttledSetBadge);
+    }
+  },
 });
 
+function throttledSetBadge() {
+  const num = runningIds.length;
+  if (numBadgesSent < num) {
+    numBadgesSent = num;
+    return sendCmd('SetBadge', runningIds)::then(() => {
+      badgePromise = throttledSetBadge();
+    });
+  }
+}
+
 export function appendToRoot(node) {
   // DOM spec allows any elements under documentElement
   // https://dom.spec.whatwg.org/#node-trees
@@ -46,71 +66,100 @@ export function injectPageSandbox(contentId, webId) {
   });
 }
 
-export function injectScripts(contentId, webId, data, isXml) {
-  bridge.ids = data.ids;
+/**
+ * @param {string} contentId
+ * @param {string} webId
+ * @param {VMGetInjectedData} data
+ * @param {boolean} isXml
+ */
+export async function injectScripts(contentId, webId, data, isXml) {
   // eslint-disable-next-line prefer-rest-params
   if (!elemByTag('*')) return onElement('*', injectScripts, ...arguments);
-  let injectable = isXml ? false : null;
-  const bornReady = ['interactive', 'complete'].includes(document.readyState);
-  const info = objectPick(data, ['cache', 'isFirefox', 'ua']);
+  const { hasMore, info } = data;
+  pageInjectable = isXml ? false : null;
   realms = {
+    /** @namespace VMInjectionRealm */
     [INJECT_CONTENT]: {
       injectable: () => true,
+      /** @namespace VMRunAtLists */
       lists: contLists = { start: [], body: [], end: [], idle: [] },
-      ids: [],
       info,
     },
     [INJECT_PAGE]: {
-      // eslint-disable-next-line no-return-assign
-      injectable: () => injectable ?? (injectable = checkInjectable()),
+      injectable: () => pageInjectable ?? checkInjectable(),
       lists: pgLists = { start: [], body: [], end: [], idle: [] },
-      ids: [],
       info,
     },
   };
-  sendCmd('InjectionFeedback', data.scripts.map((script) => {
-    const { custom, dataKey, meta, props: { id } } = script;
-    const desiredRealm = custom.injectInto || meta.injectInto || data.injectInto;
-    const internalRealm = INJECT_MAPPING[desiredRealm] || INJECT_MAPPING[INJECT_AUTO];
-    const realm = internalRealm.find(key => realms[key]?.injectable());
-    let needsInjection;
+  const feedback = data.scripts.map((script) => {
+    const { id } = script.props;
+    const realm = INJECT_MAPPING[script.injectInto].find(key => realms[key]?.injectable());
     // If the script wants this specific realm, which is unavailable, we won't inject it at all
     if (realm) {
-      const { ids, lists } = realms[realm];
-      let runAt = bornReady ? 'start'
-        : `${custom.runAt || meta.runAt || ''}`.replace(/^document-/, '');
-      const list = lists[runAt] || lists[runAt = 'end'];
-      needsInjection = realm === INJECT_CONTENT;
-      script.stage = needsInjection && runAt !== 'start' && runAt;
-      ids.push(id);
-      list.push(script);
+      const realmData = realms[realm];
+      realmData.lists[script.runAt].push(script); // 'start' or 'body' per getScriptsByURL()
+      realmData.is = true;
     } else {
       bridge.failedIds.push(id);
     }
-    return [dataKey, needsInjection];
-  }));
-  setupContentInvoker(contentId, webId);
+    return [script.dataKey, realm === INJECT_CONTENT];
+  });
+  const moreData = sendCmd('InjectionFeedback', {
+    feedback,
+    feedId: data.feedId,
+    pageInjectable: pageInjectable ?? (hasMore && checkInjectable()),
+  });
+  // saving while safe
+  const getReadyState = hasMore && describeProperty(Document.prototype, 'readyState').get;
+  if (realms[INJECT_CONTENT].is) {
+    setupContentInvoker(contentId, webId);
+  }
   injectAll('start');
-  if (pgLists.body[0] || contLists.body[0]) {
-    onElement('body', injectAll, 'body');
+  const onBody = (pgLists.body[0] || contLists.body[0])
+    && onElement('body', injectAll, 'body');
+  // document-end, -idle
+  if (hasMore) {
+    data = await moreData;
+    if (data) await injectDelayedScripts(data, getReadyState);
   }
-  if (pgLists.idle[0] || contLists.idle[0] || pgLists.end[0] || contLists.end[0]) {
-    document::addEventListener('DOMContentLoaded', () => {
-      injectAll('end');
-      injectAll('idle');
-    }, { once: true });
+  if (onBody) {
+    await onBody;
   }
+  realms = null;
+  pgLists = null;
+  contLists = null;
+}
+
+async function injectDelayedScripts({ info, scripts }, getReadyState) {
+  realms::forEachKey(r => {
+    realms[r].info = info;
+  });
+  scripts::forEach(script => {
+    const { code, runAt } = script;
+    if (code && !pageInjectable) {
+      bridge.failedIds::push(script.props.id);
+    } else {
+      (code ? pgLists : contLists)[runAt]::push(script);
+    }
+    script.stage = !code && runAt;
+  });
+  if (document::getReadyState() === 'loading') {
+    await new Promise(resolve => {
+      document::addEventListener('DOMContentLoaded', resolve, { once: true });
+    });
+  }
+  injectAll('end');
+  injectAll('idle');
 }
 
 function checkInjectable() {
-  let res = false;
   bridge.addHandlers({
     Pong() {
-      res = true;
+      pageInjectable = true;
     },
   });
   bridge.post('Ping');
-  return res;
+  return pageInjectable;
 }
 
 function inject(item) {
@@ -136,69 +185,67 @@ function inject(item) {
 }
 
 function injectAll(runAt) {
-  const isStart = runAt === 'start';
-  // Not using destructuring of arrays because we don't know if @@iterator is safe.
   realms::forEachKey((realm) => {
     const realmData = realms[realm];
-    const isPage = realm === INJECT_PAGE;
-    realmData.lists::forEachKey((name) => {
-      const items = realmData.lists[name];
-      // All lists for content mode are processed jointly when runAt='start'
-      if ((isPage ? name === runAt : isStart) && items.length) {
-        bridge.post('ScriptData', { items, runAt: name, info: realmData.info }, realm);
-        realmData.info = undefined;
-        if (isPage && !bridge.isFirefox) {
-          injectList(runAt);
-        }
+    const items = realmData.lists[runAt];
+    if (items.length) {
+      bridge.post('ScriptData', { items, runAt, info: realmData.info }, realm);
+      realmData.info = undefined;
+      if (realm === INJECT_PAGE && !bridge.isFirefox) {
+        injectList(runAt);
       }
-    });
+    }
   });
-  if (!isStart && contLists[runAt].length) {
+  if (runAt !== 'start' && contLists[runAt].length) {
     bridge.post('RunAt', runAt, INJECT_CONTENT);
   }
-  if (runAt === 'idle') {
-    realms = null;
-    pgLists = null;
-    contLists = null;
-  }
 }
 
 async function injectList(runAt) {
-  const isIdle = runAt === 'idle';
   const list = pgLists[runAt];
   // Not using for-of because we don't know if @@iterator is safe.
-  for (let i = 0; i < list.length; i += 1) {
-    if (isIdle) await sendCmd('SetTimeout', 0);
-    inject(list[i]);
+  for (let i = 0, item; (item = list[i]); i += 1) {
+    if (item.code) {
+      if (runAt === 'idle') await sendCmd('SetTimeout', 0);
+      if (runAt === 'end') await 0;
+      inject(item);
+      item.code = '';
+    }
   }
 }
 
+/**
+ * @param {string} tag
+ * @param {function} cb - callback runs immediately, unlike a chained then()
+ * @param {?} [args]
+ * @returns {Promise<void>}
+ */
 function onElement(tag, cb, ...args) {
-  if (elemByTag(tag)) {
-    cb(...args);
-  } else {
-    // This function runs before any userscripts, but MutationObserver callback may run
-    // after content-mode userscripts so we'll have to use safe calls there
-    const { disconnect } = MutationObserver.prototype;
-    const observer = new MutationObserver(() => {
-      if (elemByTag(tag)) {
-        observer::disconnect();
-        cb(...args);
-      }
-    });
-    // documentElement may be replaced so we'll observe the entire document
-    observer.observe(document, { childList: true, subtree: true });
-  }
+  return new Promise(resolve => {
+    if (elemByTag(tag)) {
+      cb(...args);
+      resolve();
+    } else {
+      const observer = new MutationObserver(() => {
+        if (elemByTag(tag)) {
+          observer::disconnect(); // eslint-disable-line no-use-before-define
+          cb(...args);
+          resolve();
+        }
+      });
+      // This function starts before any content-mode userscripts, but observer's callback
+      // will run after document-start content-mode userscripts, so we'll use safe calls.
+      const { disconnect } = observer;
+      // documentElement may be replaced so we'll observe the entire document
+      observer.observe(document, { childList: true, subtree: true });
+    }
+  });
 }
 
 function setupContentInvoker(contentId, webId) {
-  const invokableIds = realms[INJECT_CONTENT].ids;
-  if (invokableIds.length) {
-    const invoke = {
-      [INJECT_CONTENT]: VMInitInjection()(webId, contentId, bridge.onHandle),
-    };
-    const postViaBridge = bridge.post;
-    bridge.invokableIds = invokableIds;
-    bridge.post = (cmd, params, realm) => (invoke[realm] || postViaBridge)(cmd, params);
-  }
+  const invokeContent = VMInitInjection()(webId, contentId, bridge.onHandle);
+  const postViaBridge = bridge.post;
+  bridge.post = (cmd, params, realm) => (
+    (realm === INJECT_CONTENT ? invokeContent : postViaBridge)(cmd, params)
+  );
 }

src/injected/web/bridge.js

@@ -5,6 +5,7 @@ import { Promise } from '../utils/helpers';
 const handlers = {};
 const callbacks = {};
 const bridge = {
+  cache: {},
   callbacks,
   addHandlers(obj) {
     assign(handlers, obj);

src/injected/web/gm-api.js

@@ -263,7 +263,7 @@ function getResource(context, name, isBlob) {
   if (key) {
     let res = isBlob && context.urls[key];
     if (!res) {
-      const raw = store.cache[context.pathMap[key] || key];
+      const raw = bridge.cache[context.pathMap[key] || key];
       if (raw) {
         const dataPos = raw::lastIndexOf(',');
         const bin = atob(dataPos < 0 ? raw : raw::slice(dataPos + 1));

src/injected/web/index.js

@@ -1,5 +1,5 @@
 import { INJECT_PAGE, INJECT_CONTENT } from '#/common/consts';
-import { defineProperty, describeProperty } from '#/common/object';
+import { assign, defineProperty, describeProperty } from '#/common/object';
 import { bindEvents } from '../utils';
 import { document, forEach, log, logging, remove, Promise, then } from '../utils/helpers';
 import bridge from './bridge';
@@ -12,6 +12,7 @@ import './tabs';
 
 // Make sure to call safe::methods() in code that may run after userscripts
 
+const { window } = global;
 const { KeyboardEvent, MouseEvent } = global;
 const { get: getCurrentScript } = describeProperty(Document.prototype, 'currentScript');
 
@@ -56,9 +57,8 @@ bridge.addHandlers({
   },
   ScriptData({ info, items, runAt }) {
     if (info) {
-      bridge.isFirefox = info.isFirefox;
-      bridge.ua = info.ua;
-      store.cache = info.cache;
+      assign(info.cache, bridge.cache);
+      assign(bridge, info);
     }
     if (items) {
       const { stage } = items[0];
@@ -88,8 +88,8 @@ bridge.addHandlers({
 });
 
 function createScriptData(item) {
-  const { dataKey, values } = item;
-  store.values[item.props.id] = values;
+  const { dataKey } = item;
+  store.values[item.props.id] = item.values || {};
   if (window[dataKey]) { // executeScript ran before GetInjected response
     onCodeSet(item, window[dataKey]);
   } else {
@@ -107,7 +107,10 @@ async function onCodeSet(item, fn) {
   if (process.env.DEBUG) {
     log('info', [bridge.mode], item.displayName);
   }
-  const run = () => wrapGM(item)::fn(logging.error);
+  const run = () => {
+    wrapGM(item)::fn(logging.error);
+    bridge.post('Run', item.props.id);
+  };
   const el = document::getCurrentScript();
   const wait = waiters[stage];
   if (el) el::remove();

test/injected/gm-resource.test.js

@@ -1,7 +1,7 @@
 import test from 'tape';
 import { buffer2string } from '#/common';
 import { wrapGM } from '#/injected/web/gm-wrapper';
-import store from '#/injected/web/store';
+import bridge from '#/injected/web/bridge';
 
 const stringAsBase64 = str => btoa(buffer2string(new TextEncoder().encode(str).buffer));
 
@@ -31,7 +31,7 @@ const script = {
   },
 };
 const wrapper = wrapGM(script);
-store.cache = {
+bridge.cache = {
   [script.meta.resources.foo]: `text/plain,${stringAsBase64(RESOURCE_TEXT)}`,
 };