diff --git a/apps/vscode-e2e/fixtures/read-file.json b/apps/vscode-e2e/fixtures/read-file.json new file mode 100644 index 00000000000..48b9c3ba74f --- /dev/null +++ b/apps/vscode-e2e/fixtures/read-file.json @@ -0,0 +1,107 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "READ_FILE_SIMPLE_SMOKE" + }, + "response": { + "toolCalls": [ + { + "name": "read_file", + "arguments": "{\"path\":\"simple-read-file-smoke.txt\",\"mode\":\"slice\",\"offset\":1,\"limit\":2000,\"indentation\":{\"anchor_line\":1,\"max_levels\":0,\"include_siblings\":false,\"include_header\":true,\"max_lines\":2000}}", + "id": "call_read_file_simple_001" + } + ] + } + }, + { + "match": { + "userMessage": "READ_FILE_MULTILINE_SMOKE" + }, + "response": { + "toolCalls": [ + { + "name": "read_file", + "arguments": "{\"path\":\"multiline-read-file.txt\",\"mode\":\"slice\",\"offset\":1,\"limit\":20,\"indentation\":{\"anchor_line\":1,\"max_levels\":0,\"include_siblings\":false,\"include_header\":true,\"max_lines\":20}}", + "id": "call_read_file_multiline_001" + } + ] + } + }, + { + "match": { + "userMessage": "READ_FILE_SLICE_SMOKE" + }, + "response": { + "toolCalls": [ + { + "name": "read_file", + "arguments": "{\"path\":\"multiline-read-file.txt\",\"mode\":\"slice\",\"offset\":2,\"limit\":3,\"indentation\":{\"anchor_line\":1,\"max_levels\":0,\"include_siblings\":false,\"include_header\":true,\"max_lines\":2000}}", + "id": "call_read_file_slice_001" + } + ] + } + }, + { + "match": { + "userMessage": "READ_FILE_MISSING_SMOKE" + }, + "response": { + "toolCalls": [ + { + "name": "read_file", + "arguments": "{\"path\":\"non-existent-read-file.txt\",\"mode\":\"slice\",\"offset\":1,\"limit\":2000,\"indentation\":{\"anchor_line\":1,\"max_levels\":0,\"include_siblings\":false,\"include_header\":true,\"max_lines\":2000}}", + "id": "call_read_file_missing_001" + } + ] + } + }, + { + "match": { + "userMessage": "READ_FILE_XML_SMOKE" + }, + "response": { + "toolCalls": [ + { + "name": "read_file", + "arguments": "{\"path\":\"xml-content-read-file.xml\",\"mode\":\"slice\",\"offset\":1,\"limit\":50,\"indentation\":{\"anchor_line\":1,\"max_levels\":0,\"include_siblings\":false,\"include_header\":true,\"max_lines\":50}}", + "id": "call_read_file_xml_001" + } + ] + } + }, + { + "match": { + "userMessage": "READ_FILE_MULTIPLE_SMOKE" + }, + "response": { + "toolCalls": [ + { + "name": "read_file", + "arguments": "{\"path\":\"simple-read-file-smoke.txt\",\"mode\":\"slice\",\"offset\":1,\"limit\":20,\"indentation\":{\"anchor_line\":1,\"max_levels\":0,\"include_siblings\":false,\"include_header\":true,\"max_lines\":20}}", + "id": "call_read_file_multiple_simple_001" + }, + { + "name": "read_file", + "arguments": "{\"path\":\"multiline-read-file.txt\",\"mode\":\"slice\",\"offset\":1,\"limit\":20,\"indentation\":{\"anchor_line\":1,\"max_levels\":0,\"include_siblings\":false,\"include_header\":true,\"max_lines\":20}}", + "id": "call_read_file_multiple_multiline_001" + } + ] + } + }, + { + "match": { + "userMessage": "READ_FILE_LARGE_SMOKE" + }, + "response": { + "toolCalls": [ + { + "name": "read_file", + "arguments": "{\"path\":\"large-read-file.txt\",\"mode\":\"slice\",\"offset\":1,\"limit\":1000,\"indentation\":{\"anchor_line\":1,\"max_levels\":0,\"include_siblings\":false,\"include_header\":true,\"max_lines\":2000}}", + "id": "call_read_file_large_001" + } + ] + } + } + ] +} diff --git a/apps/vscode-e2e/src/fixtures/read-file.ts b/apps/vscode-e2e/src/fixtures/read-file.ts new file mode 100644 index 00000000000..0740ada31e8 --- /dev/null +++ b/apps/vscode-e2e/src/fixtures/read-file.ts @@ -0,0 +1,114 @@ +import { LLMock } from "@copilotkit/aimock" +import type { ChatCompletionRequest, ChatMessage } from "@copilotkit/aimock" + +type ToolResultExpectation = { toolCallId: string; expected: string[] } + +type ReadFileResultFixture = { + toolCallId: string + expected: string[] | ToolResultExpectation[] + result: string + id: string +} + +function isToolResultExpectation(value: unknown): value is ToolResultExpectation { + return typeof value === "object" && value !== null && "toolCallId" in value && "expected" in value +} + +function toolResultContains(req: ChatCompletionRequest, toolCallId: string, expected: string[]) { + const messages = Array.isArray(req?.messages) ? req.messages : [] + const toolMessage = messages.find( + (message: ChatMessage) => message?.role === "tool" && message.tool_call_id === toolCallId, + ) + + const content = toolMessage?.content + if (typeof content !== "string") { + return false + } + + return expected.every((text) => content.includes(text)) +} + +function toolResultsContain(req: ChatCompletionRequest, expectations: ToolResultExpectation[]) { + return expectations.every(({ toolCallId, expected }) => toolResultContains(req, toolCallId, expected)) +} + +export function addReadFileResultFixtures(mock: InstanceType) { + const fixtures: ReadFileResultFixture[] = [ + { + toolCallId: "call_read_file_simple_001", + expected: ["File: simple-read-file-smoke.txt", "1 | Hello, World!"], + result: 'The file [`simple-read-file-smoke.txt`](simple-read-file-smoke.txt) contains the text: "Hello, World!"', + id: "call_read_file_simple_002", + }, + { + toolCallId: "call_read_file_multiline_001", + expected: ["File: multiline-read-file.txt", "1 | Line 1", "5 | Line 5"], + result: "The file [`multiline-read-file.txt`](multiline-read-file.txt) contains 5 lines: Line 1, Line 2, Line 3, Line 4, and Line 5.", + id: "call_read_file_multiline_002", + }, + { + toolCallId: "call_read_file_slice_001", + expected: ["File: multiline-read-file.txt", "2 | Line 2", "3 | Line 3", "4 | Line 4"], + result: "The three lines read from [`multiline-read-file.txt`](multiline-read-file.txt) starting at offset 2 are:\n\n- Line 2\n- Line 3\n- Line 4", + id: "call_read_file_slice_002", + }, + { + toolCallId: "call_read_file_missing_001", + expected: ["non-existent-read-file.txt", "ENOENT", "no such file or directory"], + result: "Attempting to read [`non-existent-read-file.txt`](non-existent-read-file.txt) resulted in an error: the file does not exist. This error was handled appropriately and no file contents were returned.", + id: "call_read_file_missing_002", + }, + { + toolCallId: "call_read_file_xml_001", + expected: ["File: xml-content-read-file.xml", "Test content", "Some data"], + result: "The XML file [`xml-content-read-file.xml`](xml-content-read-file.xml) contains the following elements:\n- `` (root element)\n- `` (child of root)\n- `` (child of root)\n\nThe structure is:\n```xml\n\n Test content\n Some data\n\n```", + id: "call_read_file_xml_002", + }, + { + toolCallId: "call_read_file_multiple_multiline_001", + expected: [ + { + toolCallId: "call_read_file_multiple_simple_001", + expected: ["File: simple-read-file-smoke.txt", "1 | Hello, World!"], + }, + { + toolCallId: "call_read_file_multiple_multiline_001", + expected: ["File: multiline-read-file.txt", "1 | Line 1", "5 | Line 5"], + }, + ], + result: "Contents of [`simple-read-file-smoke.txt`](simple-read-file-smoke.txt):\n\n```\nHello, World!\n```\n\nContents of [`multiline-read-file.txt`](multiline-read-file.txt):\n\n```\nLine 1\nLine 2\nLine 3\nLine 4\nLine 5\n```", + id: "call_read_file_multiple_002", + }, + { + toolCallId: "call_read_file_large_001", + expected: [ + "File: large-read-file.txt", + "1 | Line 1: This is a test line with some content", + "100 | Line 100: This is a test line with some content", + ], + result: "The file [`large-read-file.txt`](large-read-file.txt) contains 100 lines, each following the pattern: `Line N: This is a test line with some content`, where `N` is the line number (from 1 to 100). The structure is consistent throughout the file, with only the line number changing on each line.", + id: "call_read_file_large_002", + }, + ] + + for (const fixture of fixtures) { + mock.addFixture({ + match: { + toolCallId: fixture.toolCallId, + predicate: (req) => + isToolResultExpectation(fixture.expected[0]) + ? toolResultsContain(req, fixture.expected as ToolResultExpectation[]) + : toolResultContains(req, fixture.toolCallId, fixture.expected as string[]), + }, + response: { + toolCalls: [ + { + name: "attempt_completion", + arguments: JSON.stringify({ result: fixture.result }), + id: fixture.id, + }, + ], + }, + }) + } +} diff --git a/apps/vscode-e2e/src/runTest.ts b/apps/vscode-e2e/src/runTest.ts index 5efa409a9ab..dfe84ee4c6e 100644 --- a/apps/vscode-e2e/src/runTest.ts +++ b/apps/vscode-e2e/src/runTest.ts @@ -5,6 +5,8 @@ import * as fs from "fs/promises" import { runTests } from "@vscode/test-electron" import { LLMock } from "@copilotkit/aimock" +import { addReadFileResultFixtures } from "./fixtures/read-file" + async function main() { const isRecord = process.env.AIMOCK_RECORD === "true" @@ -55,6 +57,8 @@ async function main() { mock.loadFixtureDir(fixturesDir) if (!isRecord) { + addReadFileResultFixtures(mock) + // The modes test (switch_mode → ask) triggers a second API call whose last // user message starts with directly — no // wrapper. JSON fixtures use substring matching so a bare "" diff --git a/apps/vscode-e2e/src/suite/tools/read-file.test.ts b/apps/vscode-e2e/src/suite/tools/read-file.test.ts index 6f3e28f60fc..a8b646842e5 100644 --- a/apps/vscode-e2e/src/suite/tools/read-file.test.ts +++ b/apps/vscode-e2e/src/suite/tools/read-file.test.ts @@ -9,7 +9,7 @@ import { RooCodeEventName, type ClineMessage } from "@roo-code/types" import { waitFor, sleep } from "../utils" import { setDefaultSuiteTimeout } from "../test-utils" -suite.skip("Roo Code read_file Tool", function () { +suite("Roo Code read_file Tool", function () { setDefaultSuiteTimeout(this) let tempDir: string @@ -31,12 +31,12 @@ suite.skip("Roo Code read_file Tool", function () { // Create test files with different content types testFiles = { - simple: path.join(workspaceDir, `simple-${Date.now()}.txt`), - multiline: path.join(workspaceDir, `multiline-${Date.now()}.txt`), - empty: path.join(workspaceDir, `empty-${Date.now()}.txt`), - large: path.join(workspaceDir, `large-${Date.now()}.txt`), - xmlContent: path.join(workspaceDir, `xml-content-${Date.now()}.xml`), - nested: path.join(workspaceDir, "nested", "deep", `nested-${Date.now()}.txt`), + simple: path.join(workspaceDir, "simple-read-file-smoke.txt"), + multiline: path.join(workspaceDir, "multiline-read-file.txt"), + empty: path.join(workspaceDir, "empty-read-file.txt"), + large: path.join(workspaceDir, "large-read-file.txt"), + xmlContent: path.join(workspaceDir, "xml-content-read-file.xml"), + nested: path.join(workspaceDir, "nested", "deep", "nested-read-file.txt"), } // Create files with content @@ -126,49 +126,11 @@ suite.skip("Roo Code read_file Tool", function () { let taskStarted = false let taskCompleted = false let errorOccurred: string | null = null - let toolExecuted = false - let toolResult: string | null = null // Listen for messages const messageHandler = ({ message }: { message: ClineMessage }) => { messages.push(message) - // Check for tool execution and extract result - if (message.type === "say" && message.say === "api_req_started") { - const text = message.text || "" - if (text.includes("read_file")) { - toolExecuted = true - console.log("Tool executed:", text.substring(0, 200)) - - // Parse the tool result from the api_req_started message - try { - const requestData = JSON.parse(text) - if (requestData.request && requestData.request.includes("[read_file")) { - console.log("Full request for debugging:", requestData.request) - // Try multiple patterns to extract the content - // Pattern 1: Content between triple backticks - let resultMatch = requestData.request.match(/```[^`]*\n([\s\S]*?)\n```/) - if (!resultMatch) { - // Pattern 2: Content after "Result:" with line numbers - resultMatch = requestData.request.match(/Result:[\s\S]*?\n((?:\d+\s*\|[^\n]*\n?)+)/) - } - if (!resultMatch) { - // Pattern 3: Simple content after Result: - resultMatch = requestData.request.match(/Result:\s*\n([\s\S]+?)(?:\n\n|$)/) - } - if (resultMatch) { - toolResult = resultMatch[1] - console.log("Extracted tool result:", toolResult) - } else { - console.log("Could not extract tool result from request") - } - } - } catch (e) { - console.log("Failed to parse tool result:", e) - } - } - } - // Log important messages for debugging if (message.type === "say" && message.say === "error") { errorOccurred = message.text || "Unknown error" @@ -211,7 +173,7 @@ suite.skip("Roo Code read_file Tool", function () { alwaysAllowReadOnly: true, alwaysAllowReadOnlyOutsideWorkspace: true, }, - text: `Please use the read_file tool to read the file named "${fileName}". This file contains the text "Hello, World!" and is located in the current workspace directory. Assume the file exists and you can read it directly. After reading it, tell me what the file contains.`, + text: `READ_FILE_SIMPLE_SMOKE: Please use the read_file tool to read the file named "${fileName}". This file contains the text "Hello, World!" and is located in the current workspace directory. Assume the file exists and you can read it directly. After reading it, tell me what the file contains.`, }) console.log("Task ID:", taskId) @@ -229,24 +191,11 @@ suite.skip("Roo Code read_file Tool", function () { // Wait for task completion await waitFor(() => taskCompleted, { timeout: 60_000 }) - // Verify the read_file tool was executed - assert.ok(toolExecuted, "The read_file tool should have been executed") - // Check that no errors occurred assert.strictEqual(errorOccurred, null, "No errors should have occurred") - // Verify the tool returned the correct content - assert.ok(toolResult !== null, "Tool should have returned a result") - // The tool returns content with line numbers, so we need to extract just the content - // For single line, the format is "1 | Hello, World!" - const actualContent = (toolResult as string).replace(/^\d+\s*\|\s*/, "") - assert.strictEqual( - actualContent.trim(), - "Hello, World!", - "Tool should have returned the exact file content", - ) - - // Also verify the AI mentioned the content in its response + // The committed aimock fixture drives this through a read_file tool call. + // The public e2e event stream only exposes the final assistant response. const hasContent = messages.some( (m) => m.type === "say" && @@ -269,46 +218,11 @@ suite.skip("Roo Code read_file Tool", function () { const api = globalThis.api const messages: ClineMessage[] = [] let taskCompleted = false - let toolExecuted = false - let toolResult: string | null = null // Listen for messages const messageHandler = ({ message }: { message: ClineMessage }) => { messages.push(message) - // Check for tool execution and extract result - if (message.type === "say" && message.say === "api_req_started") { - const text = message.text || "" - if (text.includes("read_file")) { - toolExecuted = true - console.log("Tool executed for multiline file") - - // Parse the tool result - try { - const requestData = JSON.parse(text) - if (requestData.request && requestData.request.includes("[read_file")) { - console.log("Full request for debugging:", requestData.request) - // Try multiple patterns to extract the content - let resultMatch = requestData.request.match(/```[^`]*\n([\s\S]*?)\n```/) - if (!resultMatch) { - resultMatch = requestData.request.match(/Result:[\s\S]*?\n((?:\d+\s*\|[^\n]*\n?)+)/) - } - if (!resultMatch) { - resultMatch = requestData.request.match(/Result:\s*\n([\s\S]+?)(?:\n\n|$)/) - } - if (resultMatch) { - toolResult = resultMatch[1] - console.log("Extracted multiline tool result") - } else { - console.log("Could not extract tool result from request") - } - } - } catch (e) { - console.log("Failed to parse tool result:", e) - } - } - } - // Log AI responses if (message.type === "say" && (message.say === "text" || message.say === "completion_result")) { console.log("AI response:", message.text?.substring(0, 200)) @@ -335,31 +249,14 @@ suite.skip("Roo Code read_file Tool", function () { alwaysAllowReadOnly: true, alwaysAllowReadOnlyOutsideWorkspace: true, }, - text: `Use the read_file tool to read the file "${fileName}" which contains 5 lines of text (Line 1, Line 2, Line 3, Line 4, Line 5). Assume the file exists and you can read it directly. Count how many lines it has and tell me the result.`, + text: `READ_FILE_MULTILINE_SMOKE: Use the read_file tool to read the file "${fileName}" which contains 5 lines of text (Line 1, Line 2, Line 3, Line 4, Line 5). Assume the file exists and you can read it directly. Count how many lines it has and tell me the result.`, }) // Wait for task completion await waitFor(() => taskCompleted, { timeout: 60_000 }) - // Verify the read_file tool was executed - assert.ok(toolExecuted, "The read_file tool should have been executed") - - // Verify the tool returned the correct multiline content - assert.ok(toolResult !== null, "Tool should have returned a result") - // The tool returns content with line numbers, so we need to extract just the content - const lines = (toolResult as string).split("\n").map((line) => { - const match = line.match(/^\d+\s*\|\s*(.*)$/) - return match ? match[1] : line - }) - const actualContent = lines.join("\n") - const expectedContent = "Line 1\nLine 2\nLine 3\nLine 4\nLine 5" - assert.strictEqual( - actualContent.trim(), - expectedContent, - "Tool should have returned the exact multiline content", - ) - - // Also verify the AI mentioned the correct number of lines + // The replay fixture only completes after aimock sees a tool response + // containing the first and final expected lines. const hasLineCount = messages.some( (m) => m.type === "say" && @@ -380,46 +277,11 @@ suite.skip("Roo Code read_file Tool", function () { const api = globalThis.api const messages: ClineMessage[] = [] let taskCompleted = false - let toolExecuted = false - let toolResult: string | null = null // Listen for messages const messageHandler = ({ message }: { message: ClineMessage }) => { messages.push(message) - // Check for tool execution and extract result - if (message.type === "say" && message.say === "api_req_started") { - const text = message.text || "" - if (text.includes("read_file")) { - toolExecuted = true - console.log("Tool executed:", text.substring(0, 300)) - - // Parse the tool result - try { - const requestData = JSON.parse(text) - if (requestData.request && requestData.request.includes("[read_file")) { - console.log("Full request for debugging:", requestData.request) - // Try multiple patterns to extract the content - let resultMatch = requestData.request.match(/```[^`]*\n([\s\S]*?)\n```/) - if (!resultMatch) { - resultMatch = requestData.request.match(/Result:[\s\S]*?\n((?:\d+\s*\|[^\n]*\n?)+)/) - } - if (!resultMatch) { - resultMatch = requestData.request.match(/Result:\s*\n([\s\S]+?)(?:\n\n|$)/) - } - if (resultMatch) { - toolResult = resultMatch[1] - console.log("Extracted line range tool result") - } else { - console.log("Could not extract tool result from request") - } - } - } catch (e) { - console.log("Failed to parse tool result:", e) - } - } - } - // Log AI responses if (message.type === "say" && (message.say === "text" || message.say === "completion_result")) { console.log("AI response:", message.text?.substring(0, 200)) @@ -446,37 +308,19 @@ suite.skip("Roo Code read_file Tool", function () { alwaysAllowReadOnly: true, alwaysAllowReadOnlyOutsideWorkspace: true, }, - text: `Use the read_file tool to read the file "${fileName}" using slice mode with offset=2 and limit=3 (1-based offset). The file contains lines like "Line 1", "Line 2", etc. After reading, show me the three lines you read.`, + text: `READ_FILE_SLICE_SMOKE: Use the read_file tool to read the file "${fileName}" using slice mode with offset=2 and limit=3 (1-based offset). The file contains lines like "Line 1", "Line 2", etc. After reading, show me the three lines you read.`, }) // Wait for task completion await waitFor(() => taskCompleted, { timeout: 60_000 }) - // Verify tool was executed - assert.ok(toolExecuted, "The read_file tool should have been executed") - - // Verify the tool returned the correct lines (offset=2, limit=3 -> lines 2-4) - if (toolResult && (toolResult as string).includes(" | ")) { - assert.ok( - (toolResult as string).includes("2 | Line 2"), - "Tool result should include line 2 with line number", - ) - assert.ok( - (toolResult as string).includes("3 | Line 3"), - "Tool result should include line 3 with line number", - ) - assert.ok( - (toolResult as string).includes("4 | Line 4"), - "Tool result should include line 4 with line number", - ) - } - - // Also verify the AI mentioned the specific lines const hasLines = messages.some( (m) => m.type === "say" && (m.say === "completion_result" || m.say === "text") && - m.text?.includes("Line 2"), + m.text?.includes("Line 2") && + m.text?.includes("Line 3") && + m.text?.includes("Line 4"), ) assert.ok(hasLines, "AI should have mentioned the requested lines") @@ -492,24 +336,10 @@ suite.skip("Roo Code read_file Tool", function () { const api = globalThis.api const messages: ClineMessage[] = [] let taskCompleted = false - let toolExecuted = false - let _errorHandled = false // Listen for messages const messageHandler = ({ message }: { message: ClineMessage }) => { messages.push(message) - - // Check for tool execution - if (message.type === "say" && message.say === "api_req_started") { - const text = message.text || "" - if (text.includes("read_file")) { - toolExecuted = true - // Check if error was returned - if (text.includes("error") || text.includes("not found")) { - _errorHandled = true - } - } - } } api.on(RooCodeEventName.Message, messageHandler) @@ -524,7 +354,7 @@ suite.skip("Roo Code read_file Tool", function () { let taskId: string try { // Start task with non-existent file - const nonExistentFile = `non-existent-${Date.now()}.txt` + const nonExistentFile = "non-existent-read-file.txt" taskId = await api.startNewTask({ configuration: { mode: "code", @@ -532,15 +362,12 @@ suite.skip("Roo Code read_file Tool", function () { alwaysAllowReadOnly: true, alwaysAllowReadOnlyOutsideWorkspace: true, }, - text: `Try to read the file "${nonExistentFile}" and tell me what happens. This file does not exist, so I expect you to handle the error appropriately.`, + text: `READ_FILE_MISSING_SMOKE: Try to read the file "${nonExistentFile}" and tell me what happens. This file does not exist, so I expect you to handle the error appropriately.`, }) // Wait for task completion await waitFor(() => taskCompleted, { timeout: 60_000 }) - // Verify the read_file tool was executed - assert.ok(toolExecuted, "The read_file tool should have been executed") - // Verify the AI handled the error appropriately const completionMessage = messages.find( (m) => @@ -564,21 +391,11 @@ suite.skip("Roo Code read_file Tool", function () { const api = globalThis.api const messages: ClineMessage[] = [] let taskCompleted = false - let toolExecuted = false // Listen for messages const messageHandler = ({ message }: { message: ClineMessage }) => { messages.push(message) - // Check for tool execution - if (message.type === "say" && message.say === "api_req_started") { - const text = message.text || "" - if (text.includes("read_file")) { - toolExecuted = true - console.log("Tool executed for XML file") - } - } - // Log AI responses if (message.type === "say" && (message.say === "text" || message.say === "completion_result")) { console.log("AI response:", message.text?.substring(0, 200)) @@ -605,21 +422,20 @@ suite.skip("Roo Code read_file Tool", function () { alwaysAllowReadOnly: true, alwaysAllowReadOnlyOutsideWorkspace: true, }, - text: `Use the read_file tool to read the XML file "${fileName}". It contains XML elements including root, child, and data. Assume the file exists and you can read it directly. Tell me what elements you find.`, + text: `READ_FILE_XML_SMOKE: Use the read_file tool to read the XML file "${fileName}". It contains XML elements including root, child, and data. Assume the file exists and you can read it directly. Tell me what elements you find.`, }) // Wait for task completion await waitFor(() => taskCompleted, { timeout: 60_000 }) - // Verify the read_file tool was executed - assert.ok(toolExecuted, "The read_file tool should have been executed") - // Verify the AI mentioned the XML content - be more flexible const hasXMLContent = messages.some( (m) => m.type === "say" && (m.say === "completion_result" || m.say === "text") && - (m.text?.toLowerCase().includes("root") || m.text?.toLowerCase().includes("xml")), + m.text?.toLowerCase().includes("root") && + m.text?.toLowerCase().includes("child") && + m.text?.toLowerCase().includes("data"), ) assert.ok(hasXMLContent, "AI should have mentioned the XML elements") @@ -635,20 +451,10 @@ suite.skip("Roo Code read_file Tool", function () { const api = globalThis.api const messages: ClineMessage[] = [] let taskCompleted = false - let readFileCount = 0 // Listen for messages const messageHandler = ({ message }: { message: ClineMessage }) => { messages.push(message) - - // Count read_file executions - if (message.type === "say" && message.say === "api_req_started") { - const text = message.text || "" - if (text.includes("read_file")) { - readFileCount++ - console.log(`Read file execution #${readFileCount}`) - } - } } api.on(RooCodeEventName.Message, messageHandler) @@ -672,7 +478,7 @@ suite.skip("Roo Code read_file Tool", function () { alwaysAllowReadOnly: true, alwaysAllowReadOnlyOutsideWorkspace: true, }, - text: `Use the read_file tool to read these two files: + text: `READ_FILE_MULTIPLE_SMOKE: Use the read_file tool to read these two files: 1. "${simpleFileName}" - contains "Hello, World!" 2. "${multilineFileName}" - contains 5 lines of text Assume both files exist and you can read them directly. Read each file and tell me what you found in each one.`, @@ -681,18 +487,14 @@ Assume both files exist and you can read them directly. Read each file and tell // Wait for task completion await waitFor(() => taskCompleted, { timeout: 60_000 }) - // Verify multiple read_file executions - AI might read them together - assert.ok( - readFileCount >= 1, - `Should have executed read_file at least once, but executed ${readFileCount} times`, - ) - - // Verify the AI mentioned both file contents - be more flexible + // Verify the AI mentioned both file contents. const hasContent = messages.some( (m) => m.type === "say" && (m.say === "completion_result" || m.say === "text") && - m.text?.toLowerCase().includes("hello"), + m.text?.includes("Hello, World!") && + m.text?.includes("Line 1") && + m.text?.includes("Line 5"), ) assert.ok(hasContent, "AI should have mentioned contents of the files") @@ -708,21 +510,11 @@ Assume both files exist and you can read them directly. Read each file and tell const api = globalThis.api const messages: ClineMessage[] = [] let taskCompleted = false - let toolExecuted = false // Listen for messages const messageHandler = ({ message }: { message: ClineMessage }) => { messages.push(message) - // Check for tool execution - if (message.type === "say" && message.say === "api_req_started") { - const text = message.text || "" - if (text.includes("read_file")) { - toolExecuted = true - console.log("Reading large file...") - } - } - // Log AI responses if (message.type === "say" && (message.say === "text" || message.say === "completion_result")) { console.log("AI response:", message.text?.substring(0, 200)) @@ -749,21 +541,19 @@ Assume both files exist and you can read them directly. Read each file and tell alwaysAllowReadOnly: true, alwaysAllowReadOnlyOutsideWorkspace: true, }, - text: `Use the read_file tool to read the file "${fileName}" which has 100 lines. Each line follows the pattern "Line N: This is a test line with some content". Assume the file exists and you can read it directly. Tell me about the pattern you see.`, + text: `READ_FILE_LARGE_SMOKE: Use the read_file tool to read the file "${fileName}" which has 100 lines. Each line follows the pattern "Line N: This is a test line with some content". Assume the file exists and you can read it directly. Tell me about the pattern you see.`, }) // Wait for task completion await waitFor(() => taskCompleted, { timeout: 60_000 }) - // Verify the read_file tool was executed - assert.ok(toolExecuted, "The read_file tool should have been executed") - // Verify the AI mentioned the line pattern - be more flexible const hasPattern = messages.some( (m) => m.type === "say" && (m.say === "completion_result" || m.say === "text") && - (m.text?.toLowerCase().includes("line") || m.text?.toLowerCase().includes("pattern")), + m.text?.toLowerCase().includes("100 lines") && + m.text?.toLowerCase().includes("line n"), ) assert.ok(hasPattern, "AI should have identified the line pattern")