fix: send chunks first + refactor

src/background/utils/requests.js

@@ -24,13 +24,13 @@ addPublicCommands({
       id,
       tabId,
       frameId,
-      events,
       xhr: new XMLHttpRequest(),
     };
-    return httpRequest(opts, src, cb)
+    return httpRequest(opts, events, src, cb)
     .catch(events.includes('error') && (err => cb({
       id,
       error: err.message,
+      data: null,
       type: 'error',
     })));
   },
@@ -54,19 +54,27 @@ addPublicCommands({
 /* 1MB takes ~20ms to encode/decode so it doesn't block the process of the extension and web page,
  * which lets us and them be responsive to other events or user input. */
 const CHUNK_SIZE = 1e6;
+const BLOB_LIFE = 60e3;
+const SEND_XHR_PROPS = ['readyState', 'status', 'statusText'];
+const SEND_PROGRESS_PROPS = ['lengthComputable', 'loaded', 'total'];
 
-async function blob2chunk(response, index) {
+function blob2chunk(response, index) {
   return blob2base64(response, index * CHUNK_SIZE, CHUNK_SIZE);
 }
 
 function blob2objectUrl(response) {
   const url = URL.createObjectURL(response);
-  cache.put(`xhrBlob:${url}`, setTimeout(commands.RevokeBlob, 60e3, url), 61e3);
+  cache.put(`xhrBlob:${url}`, setTimeout(URL.revokeObjectURL, BLOB_LIFE, url), BLOB_LIFE);
   return url;
 }
 
-/** @param {GMReq.BG} req */
-function xhrCallbackWrapper(req) {
+/**
+ * @param {GMReq.BG} req
+ * @param {GMReq.EventType[]} events
+ * @param {boolean} blobbed
+ * @param {boolean} chunked
+ */
+function xhrCallbackWrapper(req, events, blobbed, chunked) {
   let lastPromise = Promise.resolve();
   let contentType;
   let dataSize;
@@ -75,7 +83,7 @@ function xhrCallbackWrapper(req) {
   let responseText;
   let responseHeaders;
   let sent = false;
-  const { id, blobbed, chunked, xhr } = req;
+  const { id, xhr } = req;
   // Chrome encodes messages to UTF8 so they can grow up to 4x but 64MB is the message size limit
   const getChunk = blobbed && blob2objectUrl || chunked && blob2chunk;
   const getResponseHeaders = () => {
@@ -87,7 +95,7 @@ function xhrCallbackWrapper(req) {
   };
   return (evt) => {
     if (!contentType) {
-      contentType = xhr.getResponseHeader('Content-Type') || 'application/octet-stream';
+      contentType = xhr.getResponseHeader('Content-Type') || '';
     }
     if (xhr.response !== response) {
       response = xhr.response;
@@ -104,27 +112,35 @@ function xhrCallbackWrapper(req) {
       }
     }
     const { type } = evt;
-    const shouldNotify = req.events.includes(type);
-    // only send response when XHR is complete
+    const shouldNotify = events.includes(type);
+    // Sending only when XHR is complete. TODO: send partial delta since last time in onprogress?
     const shouldSendResponse = xhr.readyState === 4 && shouldNotify && !sent;
     if (!shouldNotify && type !== 'loadend') {
       return;
     }
     lastPromise = lastPromise.then(async () => {
+      if (shouldSendResponse) {
+        for (let i = 1; i < numChunks; i += 1) {
+          await req.cb({
+            id,
+            chunk: i * CHUNK_SIZE,
+            data: await getChunk(response, i),
+            size: dataSize,
+          });
+        }
+      }
       await req.cb({
         blobbed,
         chunked,
         contentType,
-        dataSize,
         id,
-        numChunks,
         type,
         /** @type {VMScriptResponseObject} */
         data: shouldNotify && {
           finalUrl: req.url || xhr.responseURL,
           ...getResponseHeaders(),
-          ...objectPick(xhr, ['readyState', 'status', 'statusText']),
-          ...('loaded' in evt) && objectPick(evt, ['lengthComputable', 'loaded', 'total']),
+          ...objectPick(xhr, SEND_XHR_PROPS),
+          ...objectPick(evt, SEND_PROGRESS_PROPS),
           response: shouldSendResponse
             ? numChunks && await getChunk(response, 0) || response
             : null,
@@ -133,18 +149,6 @@ function xhrCallbackWrapper(req) {
             : null,
         },
       });
-      if (shouldSendResponse) {
-        for (let i = 1; i < numChunks; i += 1) {
-          await req.cb({
-            id,
-            chunk: {
-              pos: i * CHUNK_SIZE,
-              data: await getChunk(response, i),
-              last: i + 1 === numChunks,
-            },
-          });
-        }
-      }
       if (type === 'loadend') {
         clearRequest(req);
       }
@@ -154,10 +158,12 @@ function xhrCallbackWrapper(req) {
 
 /**
  * @param {GMReq.Message.Web} opts
+ * @param {GMReq.EventType[]} events
  * @param {MessageSender} src
  * @param {function} cb
+ * @returns {Promise<void>}
  */
-async function httpRequest(opts, src, cb) {
+async function httpRequest(opts, events, src, cb) {
   const { tab } = src;
   const { incognito } = tab;
   const { anonymous, id, overrideMimeType, xhrType, url } = opts;
@@ -169,12 +175,10 @@ async function httpRequest(opts, src, cb) {
   const vmHeaders = [];
   // Firefox can send Blob/ArrayBuffer directly
   const willStringifyBinaries = xhrType && !IS_FIREFOX;
+  // Chrome can't fetch Blob URL in incognito so we use chunks
   const chunked = willStringifyBinaries && incognito;
   const blobbed = willStringifyBinaries && !incognito;
   const [body, contentType] = decodeBody(opts.data);
-  // Chrome can't fetch Blob URL in incognito so we use chunks
-  req.blobbed = blobbed;
-  req.chunked = chunked;
   // Firefox doesn't send cookies, https://github.com/violentmonkey/violentmonkey/issues/606
   // Both Chrome & FF need explicit routing of cookies in containers or incognito
   let shouldSendCookies = !anonymous && (incognito || IS_FIREFOX);
@@ -220,8 +224,9 @@ async function httpRequest(opts, src, cb) {
     }
   }
   toggleHeaderInjector(id, vmHeaders);
-  const callback = xhrCallbackWrapper(req);
-  req.events.forEach(evt => { xhr[`on${evt}`] = callback; });
+  // Sending as params to avoid storing one-time init data in `requests`
+  const callback = xhrCallbackWrapper(req, events, blobbed, chunked);
+  events.forEach(evt => { xhr[`on${evt}`] = callback; });
   xhr.onloadend = callback; // always send it for the internal cleanup
   xhr.send(body);
 }

src/injected/content/requests.js

@@ -14,7 +14,6 @@ const getReaderResult = describeProperty(SafeFileReader[PROTO], 'result').get;
 const readAsDataURL = SafeFileReader[PROTO].readAsDataURL;
 const fdAppend = SafeFormData[PROTO].append;
 const PROPS_TO_COPY = [
-  'events',
   'fileName',
 ];
 /** @type {GMReq.Content} */
@@ -29,10 +28,9 @@ addHandlers({
    * @returns {Promise<void>}
    */
   async HttpRequest(msg, realm) {
-    /** @type {GMReq.Content} */
     requests[msg.id] = safePickInto({
       realm,
-      wantsBlob: msg.xhrType === 'blob',
+      asBlob: msg.xhrType === 'blob',
     }, msg, PROPS_TO_COPY);
     msg.url = getFullUrl(msg.url);
     let { data } = msg;
@@ -52,63 +50,48 @@ addBackgroundHandlers({
    * @returns {Promise<void>}
    */
   async HttpRequested(msg) {
-    const { id } = msg;
+    const { id, data } = msg;
     const req = requests[id];
-    if (!req) return;
-    if (hasOwnProperty(msg, 'chunk')) {
-      receiveChunk(req, /** @type {BGChunk} */msg);
+    if (!req) {
+      if (process.env.DEV) console.warn('[HttpRequested][content]: no request for id', id);
       return;
     }
-    if (hasOwnProperty(msg, 'error')) {
-      bridge.post('HttpRequested', msg, req.realm);
+    if (hasOwnProperty(msg, 'chunk')) {
+      processChunk(req, data, msg);
       return;
     }
-    if ((msg.numChunks || 1) === 1) {
-      req.gotChunks = true;
-    }
-    const { blobbed, data, chunked, type } = msg;
-    // only CONTENT realm can read blobs from an extension:// URL
-    const response = data
-      && req.events::includes(type)
-      && data.response;
-    // messages will come while blob is fetched so we'll temporarily store the Promise
-    const importing = response && (blobbed || chunked);
-    if (importing) {
-      req.bin = blobbed
-        ? importBlob(req, response)
-        : receiveAllChunks(req, msg);
+    let response = data?.response;
+    if (response && !IS_FIREFOX) {
+      if (msg.blobbed) {
+        response = await importBlob(req, response);
+      }
+      if (msg.chunked) {
+        response = processChunk(req, response);
+        response = req.asBlob
+          ? new SafeBlob([response], { type: msg.contentType })
+          : response.buffer;
+        delete req.arr;
+      }
+      data.response = response;
     }
-    // ...which can be awaited in these subsequent messages
-    if (isPromise(req.bin)) {
-      req.bin = await req.bin;
+    if (msg.type === 'load' && req.fileName) {
+      await downloadBlob(response, req.fileName);
     }
-    // If the user in incognito supplied only `onloadend` then it arrives first, followed by chunks
-    // If the user supplied any event before `loadend`, all chunks finish before `loadend` arrives
-    if (type === 'loadend') {
-      req.gotLoadEnd = true;
-    }
-    if (importing) {
-      data.response = req.bin;
-    }
-    const fileName = type === 'load' && req.fileName;
-    if (fileName) {
-      req.fileName = '';
-      await downloadBlob(IS_FIREFOX ? response : req.bin, fileName);
+    if (msg.type === 'loadend') {
+      delete requests[msg.id];
     }
     bridge.post('HttpRequested', msg, req.realm);
-    if (req.gotLoadEnd && req.gotChunks) {
-      delete requests[id];
-    }
   },
 });
 
 /**
+ * Only a content script can read blobs from an extension:// URL
  * @param {GMReq.Content} req
  * @param {string} url
  * @returns {Promise<Blob|ArrayBuffer>}
  */
 async function importBlob(req, url) {
-  const data = await (await safeFetch(url))::(req.wantsBlob ? getBlob : getArrayBuffer)();
+  const data = await (await safeFetch(url))::(req.asBlob ? getBlob : getArrayBuffer)();
   sendCmd('RevokeBlob', url);
   return data;
 }
@@ -133,57 +116,20 @@ async function revokeBlobAfterTimeout(url) {
   revokeObjectURL(url);
 }
 
-/**
- * ArrayBuffer/Blob in Chrome incognito is transferred in string chunks
- * @param {GMReq.Content} req
- * @param {GMReq.Message.BG} msg
- * @return {Promise<Blob|ArrayBuffer>}
- */
-function receiveAllChunks(req, msg) {
-  safePickInto(req, msg, ['dataSize', 'contentType']);
-  req.arr = new SafeUint8Array(req.dataSize);
-  processChunk(req, msg.data.response, 0);
-  return !req.gotChunks
-    ? new SafePromise(resolve => { req.resolve = resolve; })
-    : finishChunks(req);
-}
-
-/**
- * @param {GMReq.Content} req
- * @param {GMReq.Message.BGChunk} msg
- */
-function receiveChunk(req, { chunk: { data, pos, last } }) {
-  processChunk(req, data, pos);
-  if (last) {
-    req.gotChunks = true;
-    req.resolve(finishChunks(req));
-    delete req.resolve;
-  }
-}
-
 /**
  * @param {GMReq.Content} req
  * @param {string} data
- * @param {number} pos
+ * @param {GMReq.Message.BGChunk} [msg]
+ * @returns {Uint8Array}
  */
-function processChunk(req, data, pos) {
-  const { arr } = req;
+function processChunk(req, data, msg) {
   data = safeAtob(data);
-  for (let len = data.length, i = 0; i < len; i += 1, pos += 1) {
-    arr[pos] = data::charCodeAt(i);
+  const len = data.length;
+  const arr = req.arr || (req.arr = new SafeUint8Array(msg ? msg.size : len));
+  for (let pos = msg?.chunk || 0, i = 0; i < len;) {
+    arr[pos++] = data::charCodeAt(i++);
   }
-}
-
-/**
- * @param {GMReq.Content} req
- * @return {Blob|ArrayBuffer}
- */
-function finishChunks(req) {
-  const { arr } = req;
-  delete req.arr;
-  return req.wantsBlob
-    ? new SafeBlob([arr], { type: req.contentType })
-    : arr.buffer;
+  return arr;
 }
 
 /** Doing it here because vault's SafeResponse+blob() doesn't work in injected-web */

src/types.d.ts

@@ -29,11 +29,8 @@ declare namespace GMReq {
   type UserOpts = VMScriptGMDownloadOptions | VMScriptGMXHRDetails;
   interface BG {
     anonymous: boolean;
-    blobbed: boolean;
     cb: (data: GMReq.Message.BGAny) => Promise<void>;
-    chunked: boolean;
     coreId: number;
-    events: EventType[];
     frameId: number;
     id: string;
     noNativeCookie: boolean;
@@ -44,15 +41,10 @@ declare namespace GMReq {
     xhr: XMLHttpRequest;
   }
   interface Content {
-    realm: VMScriptInjectInto;
-    wantsBlob: boolean;
-    events: EventType[];
-    fileName: string;
     arr?: Uint8Array;
-    resolve?: (data: any) => void;
-    dataSize?: number;
-    contentType?: string;
-    gotChunks?: boolean;
+    asBlob: boolean;
+    fileName: string;
+    realm: VMScriptInjectInto;
   }
   interface Web {
     id: string;
@@ -71,22 +63,19 @@ declare namespace GMReq {
       chunked: boolean;
       contentType: string;
       data: VMScriptResponseObject;
-      dataSize: number;
       id: string;
       type: EventType;
-      numChunks: number;
     }
     interface BGChunk {
       id: string;
-      chunk: {
-        pos: number;
-        data: string;
-        last: boolean;
-      };
+      chunk: number;
+      data: string;
+      size: number;
     }
     interface BGError {
       id: string;
       type: 'error';
+      data: null; // helps avoid the need for hasOwnProperty in HttpRequested
       error: string;
     }
     /** From web/content bridge */
@@ -119,7 +108,7 @@ declare type VMBridgeContentIds = {
 declare type VMBridgePostFunc = (
   cmd: string,
   data: any, // all types supported by structuredClone algo
-  realm?: string,
+  realm?: VMBridgeMode,
   node?: Node,
 ) => void;