fix(extension): capture initial-page-load XHR/fetch bodies (injection-timing race)

browser-extension/common/src/constants.ts

@@ -85,6 +85,9 @@ export const CLIENT_MESSAGES = {
   NETWORK_EVENT_CAPTURED: "networkEventCaptured",
   NETWORK_RECORDING_ENDED: "networkRecordingEnded",
   NETWORK_BODY_CAPTURED: "networkBodyCaptured",
+  // Page body-recorder → SW: "I'm armed and buffering, send me START (with caps) now." Pull-based
+  // handshake so START isn't sent before the page (and the content-script relay) is listening.
+  NETWORK_BODY_RECORDER_READY: "networkBodyRecorderReady",
   // Floating widget: SW → content script to show/hide the on-page reopen widget.
   SHOW_NETWORK_RECORDING_WIDGET: "showNetworkRecordingWidget",
   HIDE_NETWORK_RECORDING_WIDGET: "hideNetworkRecordingWidget",

browser-extension/mv3/src/content-scripts/client/pageScriptMessageListener.ts

@@ -63,6 +63,11 @@ export const initPageScriptMessageListener = () => {
           payload: event.data.payload,
         });
         break;
+      case CLIENT_MESSAGES.NETWORK_BODY_RECORDER_READY:
+        // Network Interceptor v2: the page body-recorder is armed and listening — forward to the SW
+        // so it replies with START (resolved caps) now. Pull-based handshake; tabId added in the SW.
+        chrome.runtime.sendMessage({ action: CLIENT_MESSAGES.NETWORK_BODY_RECORDER_READY });
+        break;
     }
   });
 };

browser-extension/mv3/src/page-scripts/networkBodyRecorder.js

@@ -67,16 +67,49 @@ const applyCaps = (data, cfg) => {
   if (window.__rqNetworkBodyRecorderInstalled) return;
   window.__rqNetworkBodyRecorderInstalled = true;
 
-  let enabled = false;
+  // Capture lifecycle (see the injection-timing-race fix below):
+  //   "buffering" — interceptor is armed and captures, but we hold events in `buffer` until START
+  //                 arrives with the resolved caps. This is the state at injection time.
+  //   "live"      — START received; flush the buffer and stream every capture immediately.
+  //   "stopped"   — STOP received, OR the READY handshake gave up with no START (this tab is not
+  //                 being recorded). Drop captures and free the buffer. We do NOT call
+  //                 clearInterceptors — that would nuke other SDK consumers (e.g. session recording).
+  let state = "buffering";
   let registered = false;
+  let buffer = [];
   // Init default; overwritten by the SW's resolved value on the START signal (keep in sync with
   // DEFAULT_MAX_PAYLOAD_SIZE in networkRecording/index.ts).
-  let cfg = { maxPayloadSize: 200 * 1024, ignoreMediaResponse: true };
+  let cfg = { maxPayloadSize: 10 * 1024 * 1024, ignoreMediaResponse: true };
+
+  // The body recorder is registered broadly (every http(s) tab) for the recording's duration, but
+  // only the recorded tab ever receives START. On every OTHER tab the interceptor sits in
+  // "buffering" until the READY handshake gives up. Two guards bound memory there (and on a
+  // slow-to-START recorded tab):
+  //   1. the pre-START buffer is hard-capped to MAX_BUFFERED_ENTRIES (drop-oldest), so it can't grow
+  //      without bound while waiting;
+  //   2. when the READY handshake gives up with no START, the interceptor transitions to "stopped"
+  //      and frees the buffer (see startReadyHandshake's give-up branch).
+  // START normally arrives in ms, so the cap is only ever exercised on non-recorded tabs.
+  const MAX_BUFFERED_ENTRIES = 100;
 
   const postToExtension = (action, payload) => {
     window.postMessage({ source: "requestly:client", action, payload }, window.location.href);
   };
 
+  const emit = (data) => postToExtension(CLIENT_MESSAGES.NETWORK_BODY_CAPTURED, applyCaps(data, cfg));
+
+  // Arm the interceptor IMMEDIATELY at injection — do NOT wait for the START signal.
+  //
+  // Injection-timing race fix: the web-sdk fetch/XHR override is installed at module-eval, but it
+  // only emits for URLs that have a registered intercept record. Previously we registered that
+  // record only on START, which arrives several async hops after the document commits (SW
+  // executeScript x2 + an SW→content-script→page relay). By then the SPA's bootstrap GET /api/...
+  // requests have already fired and were passed through un-emitted — so their response bodies (the
+  // ones LTS correlates IDs/slugs from) were lost, and downstream auto-correlation chained 0 rows.
+  //
+  // Now we register synchronously and BUFFER every capture from t=0; START only flips us to live
+  // (flushing the buffer first). Paired with document_start MAIN-world injection (registerContent-
+  // Scripts in clientHandler/networkRecording), this guarantees request #1 is observed.
   const registerInterceptorOnce = () => {
     if (registered) return;
     // The web-sdk UMD declares a top-level `var Requestly`; reference it bare (same as
@@ -88,12 +121,79 @@ const applyCaps = (data, cfg) => {
     Requestly.Network.intercept(
       /.*/,
       (data) => {
-        if (!enabled) return;
-        postToExtension(CLIENT_MESSAGES.NETWORK_BODY_CAPTURED, applyCaps(data, cfg));
+        if (state === "stopped") return;
+        if (state === "buffering") {
+          // Hold until START resolves the caps, then flush in order. Hard-cap the buffer so a tab
+          // that is slow to START — or never will (a non-recorded tab the broad registration landed
+          // in, before the give-up below terminates it) — cannot grow memory without bound. Keep the
+          // most recent MAX_BUFFERED_ENTRIES; drop the oldest. (Kept raw, not pre-capped, so the
+          // flush applies the START-resolved maxPayloadSize — capping here would use the default.)
+          buffer.push(data);
+          if (buffer.length > MAX_BUFFERED_ENTRIES) buffer.shift();
+          return;
+        }
+        emit(data); // live
       },
       false
     );
+    // Pull-based handshake: announce readiness so the SW sends START (with resolved caps) now that
+    // we're armed and listening. We do NOT rely on the SW push-sending START — that can race the
+    // content-script relay's listener and be dropped.
+    startReadyHandshake();
+  };
+
+  // Announce READY and RETRY until START arrives. A single READY is not reliable: on a fast page
+  // load the MAIN-world page script (document_start) can post READY before the isolated-world
+  // content-script relay has attached its window 'message' listener — so that READY (or its START
+  // reply) is lost and the buffer would never flush (capture silently dies). We re-post READY on a
+  // short interval until state flips to "live", with a cap so we don't loop forever on a page where
+  // the SW genuinely isn't recording (e.g. a non-recorded tab the broadly-registered script landed
+  // in — it correctly never gets START and just stops asking).
+  let readyTimer;
+  let readyAttempts = 0;
+  const READY_RETRY_MS = 300;
+  const READY_MAX_ATTEMPTS = 20; // ~6s of retries
+  const stopReadyHandshake = () => {
+    if (readyTimer !== undefined) {
+      clearInterval(readyTimer);
+      readyTimer = undefined;
+    }
   };
+  const announceReady = () => {
+    postToExtension(CLIENT_MESSAGES.NETWORK_BODY_RECORDER_READY);
+  };
+  const startReadyHandshake = () => {
+    if (readyTimer !== undefined || state !== "buffering") return; // already handshaking or past it
+    announceReady();
+    readyAttempts = 1;
+    readyTimer = setInterval(() => {
+      if (state !== "buffering") {
+        stopReadyHandshake(); // START (or STOP) already moved us out of buffering
+        return;
+      }
+      if (readyAttempts >= READY_MAX_ATTEMPTS) {
+        // Gave up: no START ever arrived (this tab is not being recorded, or its relay never
+        // attached). Terminate capture so the interceptor stops accumulating — go "stopped" (the
+        // callback early-returns on it) and free the buffered entries. Without this the buffer would
+        // be retained, and keep growing up to MAX_BUFFERED_ENTRIES, for the page's whole lifetime.
+        stopReadyHandshake();
+        state = "stopped";
+        buffer = [];
+        return;
+      }
+      readyAttempts += 1;
+      announceReady();
+    }, READY_RETRY_MS);
+  };
+
+  // Try to arm now. If the UMD isn't evaluated yet in this scope, retry on microtask/next tick —
+  // both scripts are injected together at document_start, so this resolves within the same frame,
+  // still before the page's own bundles run their first request.
+  registerInterceptorOnce();
+  if (!registered) {
+    Promise.resolve().then(registerInterceptorOnce);
+    setTimeout(registerInterceptorOnce, 0);
+  }
 
   window.addEventListener("message", (event) => {
     if (event.source !== window || event.data?.source !== "requestly:extension") return;
@@ -102,12 +202,24 @@ const applyCaps = (data, cfg) => {
       const incoming = event.data.payload || {};
       if (typeof incoming.maxPayloadSize === "number") cfg.maxPayloadSize = incoming.maxPayloadSize;
       if (typeof incoming.ignoreMediaResponse === "boolean") cfg.ignoreMediaResponse = incoming.ignoreMediaResponse;
-      enabled = true;
-      registerInterceptorOnce();
+      // Only the first START matters. "live" = already started (ignore the handshake's retried
+      // READYs' duplicate STARTs); "stopped" = STOPped, or the handshake gave up before START
+      // arrived (>6s, effectively never for a recorded tab) — don't resurrect either.
+      if (state !== "buffering") return;
+      registerInterceptorOnce(); // belt-and-suspenders: ensure armed even if the eager attempts raced the UMD
+      stopReadyHandshake(); // got START — stop re-posting READY
+      // Flush everything captured before START (the bootstrap requests), in arrival order, with the
+      // now-resolved caps — then go live.
+      const pending = buffer;
+      buffer = [];
+      state = "live";
+      pending.forEach(emit);
     } else if (event.data.action === EXTENSION_MESSAGES.STOP_NETWORK_BODY_CAPTURE) {
-      // Gate the callback off — do NOT call Network.clearInterceptors() (it would nuke every
-      // SDK consumer on the page, e.g. session recording).
-      enabled = false;
+      // Stop capturing — do NOT call Network.clearInterceptors() (it would nuke every SDK consumer
+      // on the page, e.g. session recording). Drop any unflushed buffer.
+      stopReadyHandshake();
+      state = "stopped";
+      buffer = [];
     }
   });
 })();

browser-extension/mv3/src/service-worker/services/clientHandler.ts

@@ -164,6 +164,11 @@ export const initClientSideCaching = async () => {
   });
 
   chrome.webNavigation.onCommitted.addListener(async (navigatedTabData) => {
+    // Skip non-http(s) commits (e.g. about:blank, chrome://). The extension has no host access to
+    // them, so executeScript/messaging would throw "Cannot access contents of url ...". This also
+    // covers the about:blank tab the network recorder briefly creates before navigating to the
+    // recorded URL (see startNetworkRecording's about:blank hack).
+    if (navigatedTabData.url && !/^https?:\/\//.test(navigatedTabData.url)) return;
     if (isExtensionStatusEnabled) {
       updateTabRuleCache(navigatedTabData.tabId, navigatedTabData.frameId);
       globalStateManager.initSharedStateCaching(navigatedTabData.tabId);

browser-extension/mv3/src/service-worker/services/messageHandler/listener.ts

@@ -40,6 +40,7 @@ import {
   getNetworkRecordingSummary,
   handleNetworkRecordingOnClientPageLoad,
   onNetworkBodyCaptured,
+  onBodyRecorderReady,
   reopenNetworkRecordingPanel,
 } from "../networkRecording";
 
@@ -110,6 +111,11 @@ export const initMessageHandler = () => {
         onNetworkBodyCaptured(sender.tab?.id, message.payload);
         break;
 
+      case CLIENT_MESSAGES.NETWORK_BODY_RECORDER_READY:
+        // Network Interceptor v2: the page body-recorder is armed — reply with START (resolved caps).
+        onBodyRecorderReady(sender.tab?.id);
+        break;
+
       case EXTENSION_MESSAGES.REOPEN_NETWORK_RECORDING_PANEL:
         // Floating widget asked to reopen the closed side panel for this tab.
         reopenNetworkRecordingPanel(sender.tab?.id);