From f6425b0ca12dadb10763d7b216a2f0df4b825714 Mon Sep 17 00:00:00 2001 From: julianharty Date: Fri, 8 May 2026 16:15:14 +0100 Subject: [PATCH 1/2] Resolve CLI test suite IDs against the live dropdown Running `npm run test:api @` for any suite whose dropdown ID is the bare name (e.g. `@sensing`, `@materials`, `@physics`) failed with a 30s Playwright timeout and a misleading "did not find some options" message. Three suites in tests/tests.html use `@`-prefixed dropdown IDs (`@new`, `@notslow`, `@onlyslow`), but for ~20 others the `@`-prefix is the mocha grep pattern, not the dropdown ID. The runner passed the user's argument straight to `page.selectOption`, so either form had to match the dropdown exactly. Replace the static AVAILABLE_SUITES lookup with `resolveSuite(id, dropdownIds)` that: - Reads the actual dropdown options after the page loads. - Accepts either `@` or `` and resolves to whichever matches. - Polls for up to 30s, matching Playwright's default selectOption timeout, because tests.html's loadAllTests populates the dropdown incrementally (each option is appended before its module is awaited). - On miss, throws immediately with the full list of valid IDs and a hint about the `@`/grep-pattern distinction, replacing the opaque "did not find some options" timeout. Also remove the `movement: "translation"` alias. `movement` and `translation` are separate suites with separate test files (movement.test.js vs transform.translate.test.js); the alias caused `npm run test:api movement` to silently run translation tests instead of the 10 movement tests. Fixes flipcomputing/flock#619 Co-Authored-By: Claude Opus 4.7 --- scripts/run-api-tests.mjs | 63 +++++++++++++++++++++++++++++++++++---- 1 file changed, 58 insertions(+), 5 deletions(-) diff --git a/scripts/run-api-tests.mjs b/scripts/run-api-tests.mjs index efb1d8ff..d4afca9c 100644 --- a/scripts/run-api-tests.mjs +++ b/scripts/run-api-tests.mjs @@ -151,9 +151,26 @@ const logAll = args.includes("--log-all"); const requestedSuite = args.find((arg) => !arg.startsWith("--")) || "all"; const suiteAliases = { translate: "translation", - movement: "translation", }; -const suite = suiteAliases[requestedSuite] || requestedSuite; + +// Resolve a requested suite ID against the live dropdown contents from +// tests.html. Some dropdown entries are "@"-prefixed (e.g. "@new", +// "@notslow"); most use a bare name and keep "@" only for the mocha grep +// pattern. Accept either form so a user passing "@sensing" finds the +// "sensing" entry rather than timing out for 30s on a missing dropdown +// option. Returns the resolved ID if found, or null if neither the input +// (after alias) nor its bare-form fallback matches a dropdown option. +function resolveSuite(id, dropdownIds) { + const aliased = suiteAliases[id] || id; + if (dropdownIds.includes(aliased)) return aliased; + if (aliased.startsWith("@")) { + const bare = aliased.slice(1); + if (dropdownIds.includes(bare)) return bare; + } + return null; +} +// Resolution against the dropdown happens after the page loads โ€” see runTests. +const suite = requestedSuite; // Enable both logs if --log-all is specified if (logAll) { @@ -575,19 +592,55 @@ async function runTests(suiteId = "all") { throw error; } + // Resolve the requested suite against the dropdown's actual options. The + // dropdown in tests.html fills incrementally (loadAllTests awaits each + // suite's import()), so a one-shot read may miss entries near the end of + // the list. Poll up to 30s โ€” matching Playwright's default selectOption + // timeout โ€” but on miss we throw an immediate, descriptive error instead + // of Playwright's misleading "did not find some options" message. Lets + // users pass either "@sensing" or "sensing" interchangeably. + const readDropdownIds = () => + page.evaluate(() => { + const sel = document.getElementById("testSelect"); + return sel + ? Array.from(sel.options) + .map((o) => o.value) + .filter((v) => v) + : []; + }); + let dropdownIds = await readDropdownIds(); + let resolvedSuiteId = resolveSuite(suiteId, dropdownIds); + const resolveDeadline = Date.now() + 30000; + while (!resolvedSuiteId && Date.now() < resolveDeadline) { + await page.waitForTimeout(200); + dropdownIds = await readDropdownIds(); + resolvedSuiteId = resolveSuite(suiteId, dropdownIds); + } + if (!resolvedSuiteId) { + throw new Error( + `Suite "${suiteId}" is not in the test page dropdown after waiting 30s. ` + + `Available IDs: ${dropdownIds.join(", ")}. ` + + `Tip: pass the bare suite ID (e.g. "sensing") rather than its mocha tag ("@sensing").`, + ); + } + // Find suite info for diagnostics - const suiteInfo = AVAILABLE_SUITES.find((s) => s.id === suiteId); + const suiteInfo = AVAILABLE_SUITES.find((s) => s.id === resolvedSuiteId); const patternInfo = suiteInfo ? suiteInfo.pattern : "unknown"; // Select test suite - console.log(`๐Ÿงช Running test suite: ${suiteId}`); + if (resolvedSuiteId !== suiteId) { + console.log(`๐Ÿงช Running test suite: ${suiteId} โ†’ ${resolvedSuiteId}`); + } else { + console.log(`๐Ÿงช Running test suite: ${resolvedSuiteId}`); + } if (verbose) { console.log(` Suite: ${suiteInfo?.name || "Unknown"}`); console.log(` Pattern: ${patternInfo || "none (all tests)"}`); } console.log(); - await page.selectOption("#testSelect", suiteId); + await page.selectOption("#testSelect", resolvedSuiteId); // Wait for the selection to register and grep filter to be applied await page.waitForTimeout(1000); From eaa1299516e6178522649686f381c325d66e8cbe Mon Sep 17 00:00:00 2001 From: julianharty Date: Fri, 8 May 2026 17:30:26 +0100 Subject: [PATCH 2/2] Added the inverse lookup for test suites as CodeRabbit suggested. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CodeRabbit noted: ``` resolveSuite only resolves @foo โ†’ foo, not the reverse foo โ†’ @foo``. The function handles the primary PR use-case (@sensing โ†’ sensing) but has no path for the inverse: a user who types notslow, new, or onlyslow (without @) will hit the 30-second poll and receive a confusing "not in the dropdown" error, because the three special dropdown entries are @notslow, @onlyslow, @new. ``` I've added the code it suggested and tested the resolution in both directions - they worked. --- scripts/run-api-tests.mjs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/run-api-tests.mjs b/scripts/run-api-tests.mjs index d4afca9c..429e67a9 100644 --- a/scripts/run-api-tests.mjs +++ b/scripts/run-api-tests.mjs @@ -166,7 +166,11 @@ function resolveSuite(id, dropdownIds) { if (aliased.startsWith("@")) { const bare = aliased.slice(1); if (dropdownIds.includes(bare)) return bare; + } else { + const withAt = "@" + aliased; + if (dropdownIds.includes(withAt)) return withAt; } + return null; } // Resolution against the dropdown happens after the page loads โ€” see runTests.