diff --git a/browser-extension/common/src/constants.ts b/browser-extension/common/src/constants.ts index bb57ecf91c..18349f2f34 100644 --- a/browser-extension/common/src/constants.ts +++ b/browser-extension/common/src/constants.ts @@ -85,6 +85,9 @@ export const CLIENT_MESSAGES = { NETWORK_EVENT_CAPTURED: "networkEventCaptured", NETWORK_RECORDING_ENDED: "networkRecordingEnded", NETWORK_BODY_CAPTURED: "networkBodyCaptured", + // Page body-recorder → SW: "I'm armed and buffering, send me START (with caps) now." Pull-based + // handshake so START isn't sent before the page (and the content-script relay) is listening. + NETWORK_BODY_RECORDER_READY: "networkBodyRecorderReady", // Floating widget: SW → content script to show/hide the on-page reopen widget. SHOW_NETWORK_RECORDING_WIDGET: "showNetworkRecordingWidget", HIDE_NETWORK_RECORDING_WIDGET: "hideNetworkRecordingWidget", diff --git a/browser-extension/mv3/src/content-scripts/client/pageScriptMessageListener.ts b/browser-extension/mv3/src/content-scripts/client/pageScriptMessageListener.ts index 5b311410df..ebff0ed037 100644 --- a/browser-extension/mv3/src/content-scripts/client/pageScriptMessageListener.ts +++ b/browser-extension/mv3/src/content-scripts/client/pageScriptMessageListener.ts @@ -63,6 +63,11 @@ export const initPageScriptMessageListener = () => { payload: event.data.payload, }); break; + case CLIENT_MESSAGES.NETWORK_BODY_RECORDER_READY: + // Network Interceptor v2: the page body-recorder is armed and listening — forward to the SW + // so it replies with START (resolved caps) now. Pull-based handshake; tabId added in the SW. + chrome.runtime.sendMessage({ action: CLIENT_MESSAGES.NETWORK_BODY_RECORDER_READY }); + break; } }); }; diff --git a/browser-extension/mv3/src/page-scripts/networkBodyRecorder.js b/browser-extension/mv3/src/page-scripts/networkBodyRecorder.js index f1fa8879b7..60db9ac752 100644 --- a/browser-extension/mv3/src/page-scripts/networkBodyRecorder.js +++ b/browser-extension/mv3/src/page-scripts/networkBodyRecorder.js @@ -67,16 +67,49 @@ const applyCaps = (data, cfg) => { if (window.__rqNetworkBodyRecorderInstalled) return; window.__rqNetworkBodyRecorderInstalled = true; - let enabled = false; + // Capture lifecycle (see the injection-timing-race fix below): + // "buffering" — interceptor is armed and captures, but we hold events in `buffer` until START + // arrives with the resolved caps. This is the state at injection time. + // "live" — START received; flush the buffer and stream every capture immediately. + // "stopped" — STOP received, OR the READY handshake gave up with no START (this tab is not + // being recorded). Drop captures and free the buffer. We do NOT call + // clearInterceptors — that would nuke other SDK consumers (e.g. session recording). + let state = "buffering"; let registered = false; + let buffer = []; // Init default; overwritten by the SW's resolved value on the START signal (keep in sync with // DEFAULT_MAX_PAYLOAD_SIZE in networkRecording/index.ts). - let cfg = { maxPayloadSize: 200 * 1024, ignoreMediaResponse: true }; + let cfg = { maxPayloadSize: 10 * 1024 * 1024, ignoreMediaResponse: true }; + + // The body recorder is registered broadly (every http(s) tab) for the recording's duration, but + // only the recorded tab ever receives START. On every OTHER tab the interceptor sits in + // "buffering" until the READY handshake gives up. Two guards bound memory there (and on a + // slow-to-START recorded tab): + // 1. the pre-START buffer is hard-capped to MAX_BUFFERED_ENTRIES (drop-oldest), so it can't grow + // without bound while waiting; + // 2. when the READY handshake gives up with no START, the interceptor transitions to "stopped" + // and frees the buffer (see startReadyHandshake's give-up branch). + // START normally arrives in ms, so the cap is only ever exercised on non-recorded tabs. + const MAX_BUFFERED_ENTRIES = 100; const postToExtension = (action, payload) => { window.postMessage({ source: "requestly:client", action, payload }, window.location.href); }; + const emit = (data) => postToExtension(CLIENT_MESSAGES.NETWORK_BODY_CAPTURED, applyCaps(data, cfg)); + + // Arm the interceptor IMMEDIATELY at injection — do NOT wait for the START signal. + // + // Injection-timing race fix: the web-sdk fetch/XHR override is installed at module-eval, but it + // only emits for URLs that have a registered intercept record. Previously we registered that + // record only on START, which arrives several async hops after the document commits (SW + // executeScript x2 + an SW→content-script→page relay). By then the SPA's bootstrap GET /api/... + // requests have already fired and were passed through un-emitted — so their response bodies (the + // ones LTS correlates IDs/slugs from) were lost, and downstream auto-correlation chained 0 rows. + // + // Now we register synchronously and BUFFER every capture from t=0; START only flips us to live + // (flushing the buffer first). Paired with document_start MAIN-world injection (registerContent- + // Scripts in clientHandler/networkRecording), this guarantees request #1 is observed. const registerInterceptorOnce = () => { if (registered) return; // The web-sdk UMD declares a top-level `var Requestly`; reference it bare (same as @@ -88,12 +121,79 @@ const applyCaps = (data, cfg) => { Requestly.Network.intercept( /.*/, (data) => { - if (!enabled) return; - postToExtension(CLIENT_MESSAGES.NETWORK_BODY_CAPTURED, applyCaps(data, cfg)); + if (state === "stopped") return; + if (state === "buffering") { + // Hold until START resolves the caps, then flush in order. Hard-cap the buffer so a tab + // that is slow to START — or never will (a non-recorded tab the broad registration landed + // in, before the give-up below terminates it) — cannot grow memory without bound. Keep the + // most recent MAX_BUFFERED_ENTRIES; drop the oldest. (Kept raw, not pre-capped, so the + // flush applies the START-resolved maxPayloadSize — capping here would use the default.) + buffer.push(data); + if (buffer.length > MAX_BUFFERED_ENTRIES) buffer.shift(); + return; + } + emit(data); // live }, false ); + // Pull-based handshake: announce readiness so the SW sends START (with resolved caps) now that + // we're armed and listening. We do NOT rely on the SW push-sending START — that can race the + // content-script relay's listener and be dropped. + startReadyHandshake(); + }; + + // Announce READY and RETRY until START arrives. A single READY is not reliable: on a fast page + // load the MAIN-world page script (document_start) can post READY before the isolated-world + // content-script relay has attached its window 'message' listener — so that READY (or its START + // reply) is lost and the buffer would never flush (capture silently dies). We re-post READY on a + // short interval until state flips to "live", with a cap so we don't loop forever on a page where + // the SW genuinely isn't recording (e.g. a non-recorded tab the broadly-registered script landed + // in — it correctly never gets START and just stops asking). + let readyTimer; + let readyAttempts = 0; + const READY_RETRY_MS = 300; + const READY_MAX_ATTEMPTS = 20; // ~6s of retries + const stopReadyHandshake = () => { + if (readyTimer !== undefined) { + clearInterval(readyTimer); + readyTimer = undefined; + } }; + const announceReady = () => { + postToExtension(CLIENT_MESSAGES.NETWORK_BODY_RECORDER_READY); + }; + const startReadyHandshake = () => { + if (readyTimer !== undefined || state !== "buffering") return; // already handshaking or past it + announceReady(); + readyAttempts = 1; + readyTimer = setInterval(() => { + if (state !== "buffering") { + stopReadyHandshake(); // START (or STOP) already moved us out of buffering + return; + } + if (readyAttempts >= READY_MAX_ATTEMPTS) { + // Gave up: no START ever arrived (this tab is not being recorded, or its relay never + // attached). Terminate capture so the interceptor stops accumulating — go "stopped" (the + // callback early-returns on it) and free the buffered entries. Without this the buffer would + // be retained, and keep growing up to MAX_BUFFERED_ENTRIES, for the page's whole lifetime. + stopReadyHandshake(); + state = "stopped"; + buffer = []; + return; + } + readyAttempts += 1; + announceReady(); + }, READY_RETRY_MS); + }; + + // Try to arm now. If the UMD isn't evaluated yet in this scope, retry on microtask/next tick — + // both scripts are injected together at document_start, so this resolves within the same frame, + // still before the page's own bundles run their first request. + registerInterceptorOnce(); + if (!registered) { + Promise.resolve().then(registerInterceptorOnce); + setTimeout(registerInterceptorOnce, 0); + } window.addEventListener("message", (event) => { if (event.source !== window || event.data?.source !== "requestly:extension") return; @@ -102,12 +202,24 @@ const applyCaps = (data, cfg) => { const incoming = event.data.payload || {}; if (typeof incoming.maxPayloadSize === "number") cfg.maxPayloadSize = incoming.maxPayloadSize; if (typeof incoming.ignoreMediaResponse === "boolean") cfg.ignoreMediaResponse = incoming.ignoreMediaResponse; - enabled = true; - registerInterceptorOnce(); + // Only the first START matters. "live" = already started (ignore the handshake's retried + // READYs' duplicate STARTs); "stopped" = STOPped, or the handshake gave up before START + // arrived (>6s, effectively never for a recorded tab) — don't resurrect either. + if (state !== "buffering") return; + registerInterceptorOnce(); // belt-and-suspenders: ensure armed even if the eager attempts raced the UMD + stopReadyHandshake(); // got START — stop re-posting READY + // Flush everything captured before START (the bootstrap requests), in arrival order, with the + // now-resolved caps — then go live. + const pending = buffer; + buffer = []; + state = "live"; + pending.forEach(emit); } else if (event.data.action === EXTENSION_MESSAGES.STOP_NETWORK_BODY_CAPTURE) { - // Gate the callback off — do NOT call Network.clearInterceptors() (it would nuke every - // SDK consumer on the page, e.g. session recording). - enabled = false; + // Stop capturing — do NOT call Network.clearInterceptors() (it would nuke every SDK consumer + // on the page, e.g. session recording). Drop any unflushed buffer. + stopReadyHandshake(); + state = "stopped"; + buffer = []; } }); })(); diff --git a/browser-extension/mv3/src/service-worker/services/clientHandler.ts b/browser-extension/mv3/src/service-worker/services/clientHandler.ts index b9cc1bf722..002f18f71f 100644 --- a/browser-extension/mv3/src/service-worker/services/clientHandler.ts +++ b/browser-extension/mv3/src/service-worker/services/clientHandler.ts @@ -164,6 +164,11 @@ export const initClientSideCaching = async () => { }); chrome.webNavigation.onCommitted.addListener(async (navigatedTabData) => { + // Skip non-http(s) commits (e.g. about:blank, chrome://). The extension has no host access to + // them, so executeScript/messaging would throw "Cannot access contents of url ...". This also + // covers the about:blank tab the network recorder briefly creates before navigating to the + // recorded URL (see startNetworkRecording's about:blank hack). + if (navigatedTabData.url && !/^https?:\/\//.test(navigatedTabData.url)) return; if (isExtensionStatusEnabled) { updateTabRuleCache(navigatedTabData.tabId, navigatedTabData.frameId); globalStateManager.initSharedStateCaching(navigatedTabData.tabId); diff --git a/browser-extension/mv3/src/service-worker/services/messageHandler/listener.ts b/browser-extension/mv3/src/service-worker/services/messageHandler/listener.ts index da6b7c5029..886e238c81 100644 --- a/browser-extension/mv3/src/service-worker/services/messageHandler/listener.ts +++ b/browser-extension/mv3/src/service-worker/services/messageHandler/listener.ts @@ -40,6 +40,7 @@ import { getNetworkRecordingSummary, handleNetworkRecordingOnClientPageLoad, onNetworkBodyCaptured, + onBodyRecorderReady, reopenNetworkRecordingPanel, } from "../networkRecording"; @@ -110,6 +111,11 @@ export const initMessageHandler = () => { onNetworkBodyCaptured(sender.tab?.id, message.payload); break; + case CLIENT_MESSAGES.NETWORK_BODY_RECORDER_READY: + // Network Interceptor v2: the page body-recorder is armed — reply with START (resolved caps). + onBodyRecorderReady(sender.tab?.id); + break; + case EXTENSION_MESSAGES.REOPEN_NETWORK_RECORDING_PANEL: // Floating widget asked to reopen the closed side panel for this tab. reopenNetworkRecordingPanel(sender.tab?.id); diff --git a/browser-extension/mv3/src/service-worker/services/networkRecording/index.ts b/browser-extension/mv3/src/service-worker/services/networkRecording/index.ts index 6772ce2598..4daeac7c5a 100644 --- a/browser-extension/mv3/src/service-worker/services/networkRecording/index.ts +++ b/browser-extension/mv3/src/service-worker/services/networkRecording/index.ts @@ -1,7 +1,6 @@ import { tabService, TAB_SERVICE_DATA } from "../tabService"; import { CLIENT_MESSAGES, EXTENSION_MESSAGES } from "common/constants"; import { ChangeType } from "common/storage"; -import { injectWebAccessibleScript } from "../utils"; import { isExtensionEnabled } from "../../../utils"; import { onVariableChange, Variable } from "../../variable"; import { @@ -55,7 +54,11 @@ export interface NetworkRecordingConfig { requestScope?: RequestScope; } -const DEFAULT_MAX_PAYLOAD_SIZE = 200 * 1024; // 200 KB per-body cap (LTS-overridable via config.maxPayloadSize) +// 10 MB per-body cap (bytes), LTS-overridable via config.maxPayloadSize. Sized as a "safe maximum": +// large enough that realistic API/JSON bodies are never truncated, while staying well under Chrome's +// ~32 MB chrome.runtime.sendMessage ceiling (bodies stream one-per-message) so an oversized body +// truncates gracefully (RESPONSE_TOO_LARGE flag) instead of throwing and losing the whole entry. +const DEFAULT_MAX_PAYLOAD_SIZE = 10 * 1024 * 1024; interface NetworkRecordingState { targetTabId: number; @@ -206,18 +209,19 @@ const stopKeepaliveIfIdle = () => { // // XHR/Fetch ("xmlhttprequest" is the resource type for both) are NEVER recorded via the webRequest // path — this guard always suppresses them there: -// - recordAjax !== false (default): they're captured by the web-sdk Network interceptor (page -// script) instead, which carries headers AND bodies. Single source, no correlation needed. -// - recordAjax === false: the SDK page script is NOT injected (see injectBodyRecorder), and we -// still suppress them here — so xhr/fetch are not recorded AT ALL. "Record Ajax Requests" is a -// yes/no on recording them, not a bodies-vs-no-bodies toggle. +// - recordAjax !== false (default): they're captured by the web-sdk Network interceptor (the +// document_start MAIN-world body-recorder content scripts), which carries headers AND bodies. +// Single source, no correlation needed. +// - recordAjax === false: the SDK body-recorder scripts are NOT registered (see +// registerBodyRecorderScripts), and we still suppress them here — so xhr/fetch are not recorded +// AT ALL. "Record Ajax Requests" is a yes/no on recording them, not a bodies-vs-no toggle. // Either way, webRequest must not emit an xhr/fetch entry, so this predicate ignores recordAjax. const isAjaxRequest = (type: chrome.webRequest.ResourceType): boolean => type === "xmlhttprequest"; // requestScope "top-level": drop sub-frame (iframe-originated) requests on the webRequest path. -// frameId 0 is the main frame. The SDK (xhr/fetch) path needs no equivalent guard: injectBodyRecorder -// is only ever invoked with frameId 0 (the webNavigation.onCommitted re-inject early-returns on -// frameId !== 0), so SDK-sourced entries are inherently top-level-only. +// frameId 0 is the main frame. The SDK (xhr/fetch) path needs no equivalent guard: the body-recorder +// content scripts register without allFrames (main frame only), so SDK-sourced entries are inherently +// top-level-only. const isExcludedByScope = (recording: NetworkRecordingState, frameId: number): boolean => recording.config.requestScope === RequestScope.TOP_LEVEL && frameId !== 0; @@ -400,7 +404,7 @@ const isValidUrl = (url: string): boolean => { // CRITICAL: fire-and-forget — NEVER awaited. startNetworkRecording must stay synchronous up to // chrome.tabs.create so chrome.sidePanel.open() keeps its user gesture; an await here would break it. // Start-time only (no teardown — the browser owns this state). Failures are swallowed (best-effort, -// like injectBodyRecorder); a cache wipe that doesn't land just means a few warm-cache entries. +// like the body-recorder script registration); a cache wipe that doesn't land just means a few warm-cache entries. const wipeOriginBrowsingData = (url: string, config: NetworkRecordingConfig) => { const remove = (chrome as any).browsingData?.remove; if (typeof remove !== "function") return; // Firefox/Safari: no browsingData → no-op @@ -456,6 +460,12 @@ const removePortFromAllSubscriptions = (port: chrome.runtime.Port) => { * onCompleted can interleave, so there is no gap or duplicate. */ export const initNetworkRecordingPort = () => { + // Defensive: on SW startup activeRecordings is always empty (no rehydration), so any body-recorder + // content scripts still registered from a prior SW session that died without teardown are orphans + // injecting into every browsed tab. Clear them. A live recording re-registers via + // registerBodyRecorderScripts on its next start. + unregisterBodyRecorderScripts(); + chrome.runtime.onConnectExternal.addListener((port) => { if (port.name !== NETWORK_RECORDING_PORT) return; @@ -499,25 +509,60 @@ export const initNetworkRecordingPort = () => { // --- v2 body capture: inject the web-sdk Network interceptor into the recorded tab ---------- // The web-sdk UMD exposes the global `Requestly` (incl. Network); networkBodyRecorder.ps.js uses -// it. Both are MAIN-world. executeScript is one-shot, so we re-inject on each navigation of the -// recorded tab (handled by chrome.webNavigation.onCommitted below). The content-script relay -// forwards the start/stop control signals to the page script. +// it. Both run MAIN-world at document_start. +// +// Injection-timing-race fix: previously we injected imperatively via executeScript from +// webNavigation.onCommitted, which fires AFTER the document commits and its early scripts run — so +// an SPA's bootstrap requests fired before the interceptor was armed and their bodies were lost +// (breaking LTS auto-correlation, which needs those bootstrap response IDs). We now register both +// scripts as document_start MAIN-world CONTENT SCRIPTS (registerContentScripts) for the duration of +// the recording, mirroring the always-on ajaxRequestInterceptor (clientHandler.ts). That guarantees +// they load before the page's own scripts on every navigation. The page script arms the interceptor +// synchronously and buffers captures from t=0 (see networkBodyRecorder.js); the START signal only +// delivers the resolved caps and flushes the buffer to live. +// +// registerContentScripts is URL-pattern scoped (no tabId), so while registered the scripts inject +// into every browsed tab. That's harmless: a non-recorded tab buffers and never flushes (no START), +// and the SW's onNetworkBodyCaptured drops captures whose tabId isn't an active recording. The +// scripts are unregistered once the last recording stops. + +// One registered content script bundling the UMD + recorder (ordered js array), so the UMD is +// guaranteed evaluated before the recorder reads the global `Requestly`. +const BODY_RECORDER_SCRIPT_ID = "network-recording-body-recorder"; + +// Registered while ≥1 recording with recordAjax !== false is active. Idempotent: getRegistered → +// register only the missing ids, so a second concurrent recording doesn't double-register. +const registerBodyRecorderScripts = async () => { + try { + const existing = await chrome.scripting.getRegisteredContentScripts({ ids: [BODY_RECORDER_SCRIPT_ID] }); + if (existing.length) return; // already registered (another concurrent recording) + // ONE registration with both files in order: the UMD MUST evaluate before the recorder reads the + // global `Requestly`. Multiple files in a single RegisteredContentScript.js array run in array + // order, in the same injection — so this guarantees Requestly is defined when the recorder runs. + // (Two separate registrations did NOT guarantee cross-script order, which left `Requestly + // present? false` and forced a late retry that missed early requests.) + await chrome.scripting.registerContentScripts([ + { + id: BODY_RECORDER_SCRIPT_ID, + js: ["libs/requestly-web-sdk.js", "page-scripts/networkBodyRecorder.ps.js"], + world: "MAIN", + runAt: "document_start", + matches: ["http://*/*", "https://*/*"], + // No rehydration of recordings across SW restarts, so don't persist; cleaned up on SW init. + persistAcrossSessions: false, + }, + ]); + } catch { + // Best-effort: if registration fails (e.g. id already present from a race), body capture + // degrades but the recording continues. webRequest still covers non-xhr/fetch. + } +}; -const injectBodyRecorder = async (tabId: number, frameId = 0) => { +const unregisterBodyRecorderScripts = async () => { try { - // recordAjax === false: skip SDK injection so xhr/fetch have no SDK source; the webRequest path - // also drops them (isAjaxRequest is recordAjax-agnostic), so ajax is not recorded at all. - // A missing recording must NOT skip (recordAjax defaults to true). - if (activeRecordings.get(tabId)?.config.recordAjax === false) return; - // 1) web-sdk UMD lib (exposes global Requestly.Network) - await injectWebAccessibleScript("libs/requestly-web-sdk.js", { tabId, frameIds: [frameId] }); - // 2) our page script that registers the interceptor - await injectWebAccessibleScript("page-scripts/networkBodyRecorder.ps.js", { tabId, frameIds: [frameId] }); - // 3) start signal with the resolved caps (relayed by the content script to the page) - sendBodyCaptureSignal(tabId, EXTENSION_MESSAGES.START_NETWORK_BODY_CAPTURE); + await chrome.scripting.unregisterContentScripts({ ids: [BODY_RECORDER_SCRIPT_ID] }); } catch { - // Injection can fail on restricted pages (e.g. chrome://, strict CSP) — body capture is - // best-effort; webRequest still covers non-xhr/fetch. Don't break the recording. + // Already unregistered / never registered — ignore. } }; @@ -531,13 +576,35 @@ const sendBodyCaptureSignal = (tabId: number, action: string) => { chrome.tabs.sendMessage(tabId, { action, payload }).catch(() => {}); }; -// Re-inject on navigation of a recorded tab (executeScript is one-shot). Single-tab scoped, -// matching v1's model. Gated to active recordings; main frame only. -chrome.webNavigation.onCommitted.addListener((details) => { - if (details.frameId !== 0) return; - if (!activeRecordings.has(details.tabId)) return; - injectBodyRecorder(details.tabId, 0); -}); +/** + * Pull-based START handshake. The page body-recorder posts NETWORK_BODY_RECORDER_READY once it has + * armed the interceptor and attached its message listener; this replies with START (resolved caps), + * which flushes the page's pre-START buffer (the bootstrap requests) and flips it to live. + * + * Why pull, not push: previously START was sent on chrome.webNavigation.onCommitted, which fires + * before the content-script relay is guaranteed to be listening — so START was dropped and the + * buffer never flushed (0 entries reached the panel). The page asking for START when it's actually + * ready removes that race entirely (and is the same "START only after the recorder is set up" + * guarantee the original post-injection sendBodyCaptureSignal had). + * + * DELIVERY DEPENDENCY (known limitation): READY (page→here) and START (here→page) both ride the + * client content-script relay (content-scripts/client/index.ts → initPageScriptMessageListener), + * which attaches only on HTML top documents and only while the extension is enabled. For a recorded + * tab whose top document is NOT HTML (a raw JSON/XML/no-doctype URL), the relay never attaches: + * READY is never delivered, START never arrives, and xhr/fetch BODIES are not captured for that tab + * (the page buffers and gives up after its ~6s READY-retry window). webRequest still captures + * non-xhr/fetch skeletons. This is acceptable because LTS records web applications (HTML pages), not + * raw non-HTML endpoints. If that assumption ever changes, add an SW-side arming watchdog (expect a + * READY within ~8s of navigating; if none, surface a degraded state to the panel) — deliberately + * NOT added now to avoid machinery for a case real LTS targets don't hit. + */ +export const onBodyRecorderReady = (tabId: number | undefined) => { + if (tabId === undefined) return; + const recording = activeRecordings.get(tabId); + if (!recording) return; // not a recorded tab (scripts inject broadly; only recorded tabs get START) + if (recording.config.recordAjax === false) return; // ajax recording off → never arm emission + sendBodyCaptureSignal(tabId, EXTENSION_MESSAGES.START_NETWORK_BODY_CAPTURE); +}; // Synchronously-readable copy of IS_EXTENSION_ENABLED, so startNetworkRecording can reject a start // while the extension is off WITHOUT an async storage read — an await there would push @@ -600,35 +667,77 @@ export const reopenNetworkRecordingPanel = (tabId: number | undefined) => { openPanel(tabId); }; -export const startNetworkRecording = ( +/** + * Start a network recording in a fresh tab. + * + * THE about:blank HACK — read before changing the tab-creation order. Two hard constraints collide: + * + * (A) chrome.sidePanel.open() requires a live USER GESTURE and must be called synchronously within + * the gesture's call stack. The gesture comes from the LTS chrome.runtime.sendMessage and + * survives exactly ONE async hop — the chrome.tabs.create callback — as long as nothing is + * `await`ed before openPanel(). Any await before it forfeits the gesture and the panel never + * opens (verified: the CLIENT_PAGE_LOADED backstop can't open it either, since that's a + * gesture-less programmatic message). + * + * (B) The v2 body-recorder content scripts (web-sdk UMD + networkBodyRecorder) must be REGISTERED + * (chrome.scripting.registerContentScripts, an async call we must await) BEFORE the recorded + * URL starts loading — otherwise the SPA's bootstrap requests fire before the fetch/XHR + * interceptor is armed and their response bodies are lost (the injection-timing race; LTS + * auto-correlation then chains 0 rows because the IDs live in those missed responses). + * + * (A) needs NO await before the open; (B) needs an await before the load. Mutually exclusive on one + * tab created directly at the URL. The resolution: create the tab at **about:blank** first. + * 1. about:blank loads instantly and does NOT consume the gesture, so openPanel() in the + * tabs.create callback still succeeds (constraint A satisfied). + * 2. The real URL has NOT started loading, so we can await registerBodyRecorderScripts() AFTER + * openPanel/resolve, then chrome.tabs.update(tabId, {url}) to navigate — the scripts are now in + * place for that navigation's document_start (constraint B satisfied). + * 3. Because the recorded URL only ever loads ONCE (the post-registration navigation), nothing is + * captured twice — no webRequest/SDK double-send. (A reload-after-load approach would have + * double-counted the webRequest-sourced doc/js/css/img entries; about:blank avoids that.) + * + * COST / SUPPRESSION: the about:blank commit fires the SW's various webNavigation.onCommitted / + * page-load handlers against a URL the extension has no host access to ("about:blank"), which would + * otherwise log "Cannot access contents of url about:blank" / "Receiving end does not exist" noise. + * Those handlers are guarded to skip non-http(s) URLs (see clientHandler.ts and below). Our own + * networkRecording onCommitted/CLIENT_PAGE_LOADED paths are inert on about:blank too: the body + * recorder isn't registered for it (registered after), and capture is no-op until the page posts + * READY (which the about:blank document never does). + */ +export const startNetworkRecording = async ( url: string, config: NetworkRecordingConfig = {}, sender?: { tabId?: number; windowId?: number } ): Promise<{ success: boolean; targetTabId?: number; error?: string }> => { - // NOTE: kept synchronous up to chrome.tabs.create (no await) so the LTS sendMessage user gesture - // survives to the openPanel() call — chrome.sidePanel.open() requires an in-gesture call stack. // Reject a start while the extension is off, so the UI never says "disabled" with a live - // recording. Read from the in-memory cache (NOT an await) to keep that path synchronous. + // recording. Read from the in-memory cache (NOT an await). if (!isExtensionEnabledCache) { - return Promise.resolve({ + return { success: false, error: "Requestly extension is disabled. Enable it to start a recording.", - }); + }; } if (!url || !isValidUrl(url)) { - return Promise.resolve({ success: false, error: "Invalid URL. Must be a valid http or https URL." }); + return { success: false, error: "Invalid URL. Must be a valid http or https URL." }; } return new Promise((resolve) => { - chrome.tabs.create({ url }, (tab) => { + // Create the tab at about:blank FIRST. This keeps the synchronous gesture path to + // sidePanel.open() intact (about:blank loads instantly and does NOT consume the gesture) AND + // means the recorded URL hasn't started loading yet — so we can register the document_start + // body-recorder scripts and ONLY THEN navigate to the URL, guaranteeing the interceptor arms + // before the recorded page's first request (the injection-timing-race fix). Because the real + // URL never loads pre-registration, nothing is captured twice (no double-send on reload). + chrome.tabs.create({ url: "about:blank" }, (tab) => { if (chrome.runtime.lastError || !tab?.id) { resolve({ success: false, error: chrome.runtime.lastError?.message || "Failed to create tab" }); return; } + const tabId = tab.id; const state: NetworkRecordingState = { - targetTabId: tab.id, + targetTabId: tabId, url, startTime: Date.now(), // Resolve maxPayloadSize to its default now so the body page script (v2) can read a @@ -638,9 +747,9 @@ export const startNetworkRecording = ( senderWindowId: sender?.windowId, }; - activeRecordings.set(tab.id, state); - recordingEntries.set(tab.id, []); - tabService.setData(tab.id, TAB_SERVICE_DATA.NETWORK_RECORDING, { active: true }); + activeRecordings.set(tabId, state); + recordingEntries.set(tabId, []); + tabService.setData(tabId, TAB_SERVICE_DATA.NETWORK_RECORDING, { active: true }); // Advanced settings: start-time cache / service-worker wipe for the recorded origin (Chrome/ // Edge only; feature-guarded no-op elsewhere). Fire-and-forget — NOT awaited — so the @@ -651,22 +760,29 @@ export const startNetworkRecording = ( // inline isOverMaxDuration check in onCompleted is the fast path on a busy page. (See the // sleep/wake caveat in the keepalive comment — the only case this timer can be late.) if (config.maxDuration !== undefined) { - state.maxDurationTimer = setTimeout(() => stopNetworkRecording(tab.id!, "max-duration"), config.maxDuration); + state.maxDurationTimer = setTimeout(() => stopNetworkRecording(tabId, "max-duration"), config.maxDuration); } addWebRequestListeners(); startKeepalive(); - // Open the panel here, synchronously on the external-message path. chrome.sidePanel.open() - // requires a user gesture and must run within its call stack — the LTS sendMessage provides - // that gesture, but only as long as nothing awaits before this point (hence no async - // isExtensionEnabled check above). handleNetworkRecordingOnClientPageLoad re-opens it on - // later navigations of the recorded tab as a backstop. - openPanel(tab.id); - // v2: the body recorder is injected via webNavigation.onCommitted, which fires for this new - // tab's initial navigation (and every later one). No explicit inject here — it would be too - // early (the document isn't committed yet). - - resolve({ success: true, targetTabId: tab.id }); + // Open the panel synchronously in this callback — the LTS sendMessage gesture survives the + // single tabs.create hop (no await before here), so sidePanel.open() succeeds. The tab is on + // about:blank, so nothing is loading yet. + openPanel(tabId); + + resolve({ success: true, targetTabId: tabId }); + + // Now register the body-recorder scripts, THEN navigate the blank tab to the real URL. The + // await is AFTER openPanel/resolve, so it costs neither the gesture nor the LTS response. + // Skipped when recordAjax === false (just navigate). + // active:true re-asserts the tab as focused as it navigates — nudges Chrome to put focus on + // the page rather than leaving it in the omnibox (the blank tab held no focus). Best-effort. + const navigate = () => chrome.tabs.update(tabId, { url, active: true }).catch(() => {}); + if (config.recordAjax !== false) { + registerBodyRecorderScripts().then(navigate, navigate); + } else { + navigate(); + } }); }); }; @@ -761,6 +877,8 @@ export const stopNetworkRecording = ( if (activeRecordings.size === 0) { removeWebRequestListeners(); + // v2: no recordings left — stop injecting the body-recorder content scripts into every browsed tab. + unregisterBodyRecorderScripts(); } stopKeepaliveIfIdle(); @@ -842,6 +960,7 @@ const cleanupRecording = (tabId: number) => { recordingEntries.delete(tabId); if (activeRecordings.size === 0) { removeWebRequestListeners(); + unregisterBodyRecorderScripts(); // no recordings left — stop injecting the body-recorder scripts } stopKeepaliveIfIdle(); }; diff --git a/browser-extension/mv3/src/service-worker/services/tabService.ts b/browser-extension/mv3/src/service-worker/services/tabService.ts index 1b42baf669..0e36a927c5 100644 --- a/browser-extension/mv3/src/service-worker/services/tabService.ts +++ b/browser-extension/mv3/src/service-worker/services/tabService.ts @@ -74,6 +74,11 @@ class TabService { }); chrome.webNavigation.onDOMContentLoaded.addListener((navigatedTabData) => { + // Skip non-http(s) documents (e.g. about:blank, chrome://) — they have no client content + // script, so sendMessage would fail with "Receiving end does not exist". This covers the + // about:blank tab the network recorder briefly opens before navigating (see + // startNetworkRecording's about:blank hack). + if (navigatedTabData.url && !/^https?:\/\//.test(navigatedTabData.url)) return; if (navigatedTabData.frameId === 0) { const tab = this.getTab(navigatedTabData.tabId);