From b5974d3e2ecf1308c43a1631c93aea2a065f5cb0 Mon Sep 17 00:00:00 2001 From: Joshua Tjhin Date: Wed, 10 Jun 2026 12:09:13 -0700 Subject: [PATCH] fix: log Anthropic system message first in span input The Anthropic instrumentation merges the top-level `system` param into the logged messages array, but appended it after the conversation messages. The model actually receives the system prompt first, and downstream consumers (e.g. opening a logged span in the playground) expect chat-message order, so a trailing system message had to be manually reordered before reuse. Prepend the system message instead, and update the e2e span-tree snapshots to match. --- .../anthropic-v0273.span-tree.json | 8 ++-- .../anthropic-v0273.span-tree.txt | 8 ++-- .../anthropic-v0390.span-tree.json | 8 ++-- .../anthropic-v0390.span-tree.txt | 8 ++-- .../anthropic-v0712.span-tree.json | 8 ++-- .../anthropic-v0712.span-tree.txt | 8 ++-- .../anthropic-v0730.span-tree.json | 8 ++-- .../anthropic-v0730.span-tree.txt | 8 ++-- .../anthropic-v0780.span-tree.json | 8 ++-- .../anthropic-v0780.span-tree.txt | 8 ++-- .../anthropic-v0800.span-tree.json | 8 ++-- .../anthropic-v0800.span-tree.txt | 8 ++-- .../plugins/anthropic-plugin.test.ts | 47 +++++++++++++++++++ .../plugins/anthropic-plugin.ts | 8 ++-- 14 files changed, 100 insertions(+), 51 deletions(-) diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0273.span-tree.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0273.span-tree.json index 91d38bb51..7f6197d6b 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0273.span-tree.json +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0273.span-tree.json @@ -57,10 +57,6 @@ "type": "llm", "children": [], "input": [ - { - "content": "Bonjour mon ami!", - "role": "user" - }, { "content": [ { @@ -77,6 +73,10 @@ } ], "role": "system" + }, + { + "content": "Bonjour mon ami!", + "role": "user" } ], "output": { diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0273.span-tree.txt b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0273.span-tree.txt index 1284b06c6..2c194b682 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0273.span-tree.txt +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0273.span-tree.txt @@ -49,10 +49,6 @@ span_tree: │ └── anthropic.messages.create [llm] │ input: [ │ { - │ "content": "Bonjour mon ami!", - │ "role": "user" - │ }, - │ { │ "content": [ │ { │ "text": "translate to english", @@ -68,6 +64,10 @@ span_tree: │ } │ ], │ "role": "system" + │ }, + │ { + │ "content": "Bonjour mon ami!", + │ "role": "user" │ } │ ] │ output: { diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0390.span-tree.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0390.span-tree.json index ea8c75692..480a3aa60 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0390.span-tree.json +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0390.span-tree.json @@ -57,10 +57,6 @@ "type": "llm", "children": [], "input": [ - { - "content": "Bonjour mon ami!", - "role": "user" - }, { "content": [ { @@ -77,6 +73,10 @@ } ], "role": "system" + }, + { + "content": "Bonjour mon ami!", + "role": "user" } ], "output": { diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0390.span-tree.txt b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0390.span-tree.txt index 5294510cd..490a4d0f3 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0390.span-tree.txt +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0390.span-tree.txt @@ -49,10 +49,6 @@ span_tree: │ └── anthropic.messages.create [llm] │ input: [ │ { - │ "content": "Bonjour mon ami!", - │ "role": "user" - │ }, - │ { │ "content": [ │ { │ "text": "translate to english", @@ -68,6 +64,10 @@ span_tree: │ } │ ], │ "role": "system" + │ }, + │ { + │ "content": "Bonjour mon ami!", + │ "role": "user" │ } │ ] │ output: { diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0712.span-tree.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0712.span-tree.json index 1eae56496..ba8dcaa03 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0712.span-tree.json +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0712.span-tree.json @@ -57,10 +57,6 @@ "type": "llm", "children": [], "input": [ - { - "content": "Bonjour mon ami!", - "role": "user" - }, { "content": [ { @@ -77,6 +73,10 @@ } ], "role": "system" + }, + { + "content": "Bonjour mon ami!", + "role": "user" } ], "output": { diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0712.span-tree.txt b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0712.span-tree.txt index 448d4bf5f..d8b0c4859 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0712.span-tree.txt +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0712.span-tree.txt @@ -49,10 +49,6 @@ span_tree: │ └── anthropic.messages.create [llm] │ input: [ │ { - │ "content": "Bonjour mon ami!", - │ "role": "user" - │ }, - │ { │ "content": [ │ { │ "text": "translate to english", @@ -68,6 +64,10 @@ span_tree: │ } │ ], │ "role": "system" + │ }, + │ { + │ "content": "Bonjour mon ami!", + │ "role": "user" │ } │ ] │ output: { diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0730.span-tree.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0730.span-tree.json index 5f2716071..70b0e16e9 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0730.span-tree.json +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0730.span-tree.json @@ -57,10 +57,6 @@ "type": "llm", "children": [], "input": [ - { - "content": "Bonjour mon ami!", - "role": "user" - }, { "content": [ { @@ -77,6 +73,10 @@ } ], "role": "system" + }, + { + "content": "Bonjour mon ami!", + "role": "user" } ], "output": { diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0730.span-tree.txt b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0730.span-tree.txt index 21afa203d..17fc49f44 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0730.span-tree.txt +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0730.span-tree.txt @@ -49,10 +49,6 @@ span_tree: │ └── anthropic.messages.create [llm] │ input: [ │ { - │ "content": "Bonjour mon ami!", - │ "role": "user" - │ }, - │ { │ "content": [ │ { │ "text": "translate to english", @@ -68,6 +64,10 @@ span_tree: │ } │ ], │ "role": "system" + │ }, + │ { + │ "content": "Bonjour mon ami!", + │ "role": "user" │ } │ ] │ output: { diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0780.span-tree.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0780.span-tree.json index 051e9c9ca..c6348050a 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0780.span-tree.json +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0780.span-tree.json @@ -57,10 +57,6 @@ "type": "llm", "children": [], "input": [ - { - "content": "Bonjour mon ami!", - "role": "user" - }, { "content": [ { @@ -77,6 +73,10 @@ } ], "role": "system" + }, + { + "content": "Bonjour mon ami!", + "role": "user" } ], "output": { diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0780.span-tree.txt b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0780.span-tree.txt index 782feeb64..f49bf60e4 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0780.span-tree.txt +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0780.span-tree.txt @@ -49,10 +49,6 @@ span_tree: │ └── anthropic.messages.create [llm] │ input: [ │ { - │ "content": "Bonjour mon ami!", - │ "role": "user" - │ }, - │ { │ "content": [ │ { │ "text": "translate to english", @@ -68,6 +64,10 @@ span_tree: │ } │ ], │ "role": "system" + │ }, + │ { + │ "content": "Bonjour mon ami!", + │ "role": "user" │ } │ ] │ output: { diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0800.span-tree.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0800.span-tree.json index 72207ab39..377171e7f 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0800.span-tree.json +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0800.span-tree.json @@ -57,10 +57,6 @@ "type": "llm", "children": [], "input": [ - { - "content": "Bonjour mon ami!", - "role": "user" - }, { "content": [ { @@ -77,6 +73,10 @@ } ], "role": "system" + }, + { + "content": "Bonjour mon ami!", + "role": "user" } ], "output": { diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0800.span-tree.txt b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0800.span-tree.txt index 7b7e87d79..91de1916f 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0800.span-tree.txt +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0800.span-tree.txt @@ -49,10 +49,6 @@ span_tree: │ └── anthropic.messages.create [llm] │ input: [ │ { - │ "content": "Bonjour mon ami!", - │ "role": "user" - │ }, - │ { │ "content": [ │ { │ "text": "translate to english", @@ -68,6 +64,10 @@ span_tree: │ } │ ], │ "role": "system" + │ }, + │ { + │ "content": "Bonjour mon ami!", + │ "role": "user" │ } │ ] │ output: { diff --git a/js/src/instrumentation/plugins/anthropic-plugin.test.ts b/js/src/instrumentation/plugins/anthropic-plugin.test.ts index f50426b59..efe50aedb 100644 --- a/js/src/instrumentation/plugins/anthropic-plugin.test.ts +++ b/js/src/instrumentation/plugins/anthropic-plugin.test.ts @@ -12,6 +12,7 @@ import { parseMetricsFromUsage, aggregateAnthropicStreamChunks, processAttachmentsInInput, + coalesceInput, } from "./anthropic-plugin"; import type { StartEvent } from "../core"; import { Attachment } from "../../logger"; @@ -918,3 +919,49 @@ describe("processAttachmentsInInput", () => { expect(result[0].type).toBe("image"); }); }); + +describe("coalesceInput", () => { + it("should place the system message before the conversation messages", () => { + const messages = [ + { role: "user", content: "Hello" }, + { role: "assistant", content: "Hi there" }, + { role: "user", content: "How are you?" }, + ]; + + const result = coalesceInput(messages, "You are a helpful assistant."); + + expect(result).toEqual([ + { role: "system", content: "You are a helpful assistant." }, + { role: "user", content: "Hello" }, + { role: "assistant", content: "Hi there" }, + { role: "user", content: "How are you?" }, + ]); + }); + + it("should support system content as an array of text blocks", () => { + const messages = [{ role: "user", content: "Bonjour" }]; + const system = [ + { type: "text" as const, text: "translate to english" }, + { type: "text" as const, text: "only the answer no other text" }, + ]; + + const result = coalesceInput(messages, system); + + expect(result[0]).toEqual({ role: "system", content: system }); + expect(result[1]).toEqual({ role: "user", content: "Bonjour" }); + }); + + it("should return messages unchanged when there is no system prompt", () => { + const messages = [{ role: "user", content: "Hello" }]; + + expect(coalesceInput(messages, undefined)).toEqual(messages); + }); + + it("should not mutate the original messages array", () => { + const messages = [{ role: "user", content: "Hello" }]; + + coalesceInput(messages, "system prompt"); + + expect(messages).toEqual([{ role: "user", content: "Hello" }]); + }); +}); diff --git a/js/src/instrumentation/plugins/anthropic-plugin.ts b/js/src/instrumentation/plugins/anthropic-plugin.ts index 132b6a001..e50a500ee 100644 --- a/js/src/instrumentation/plugins/anthropic-plugin.ts +++ b/js/src/instrumentation/plugins/anthropic-plugin.ts @@ -1051,16 +1051,18 @@ export function processAttachmentsInInput(input: unknown): unknown { /** * Convert Anthropic args to the single "input" field Braintrust expects. - * Combines messages array with system message if present. + * Combines messages array with system message if present. The system message + * is placed first to match the order the model sees and the chat-message + * convention used elsewhere (e.g. the playground). */ -function coalesceInput( +export function coalesceInput( messages: AnthropicInputMessage[], system: AnthropicCreateParams["system"], ): AnthropicInputMessage[] { // Make a copy because we're going to mutate it const input = (messages || []).slice(); if (system) { - input.push({ role: "system", content: system }); + input.unshift({ role: "system", content: system }); } return input; }