From 22b78cff073852859706c917e507d47a084673a2 Mon Sep 17 00:00:00 2001 From: st-gr <38470677+st-gr@users.noreply.github.com> Date: Mon, 1 Jun 2026 05:02:31 -0700 Subject: [PATCH 1/8] feat(sandbox): allow AWS Bedrock InvokeModel paths through the L7 router MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds two patterns to `default_patterns()` so the supervisor's L7 inference router recognizes the Bedrock InvokeModel URL shape and forwards matched requests to the registered upstream: - `POST /model/{modelId}/invoke` → aws_bedrock_invoke - `POST /model/{modelId}/invoke-with-response-stream` → aws_bedrock_invoke_stream The `{modelId}` segment is wildcarded by extending `detect_inference_pattern` to handle one middle `/*/` segment in addition to the existing trailing `/*`. The wildcard is constrained to a single non-empty path segment to avoid path-traversal liabilities — `/model//invoke` and `/model/a/b/invoke` both no-match. Without this, sandboxes running Claude Code in its native Bedrock mode (`CLAUDE_CODE_USE_BEDROCK=1`, `ANTHROPIC_BEDROCK_BASE_URL`, AWS-style auth) hit the supervisor with `403 connection not allowed by policy` because their URL doesn't match `/v1/*` shapes. The fix unblocks operators wanting to register direct AWS Bedrock, an in-cluster Bedrock-compatible bridge, or a Bedrock-emulating LiteLLM as `--type aws-bedrock` providers. Tests cover: positive matches for invoke + invoke-with-response-stream, query-string handling, GET rejection, empty-segment rejection, multi-segment rejection, and unknown-action rejection. Companion changes (provider discovery spec + YAML profile) follow in the next commit. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: st-gr <38470677+st-gr@users.noreply.github.com> --- crates/openshell-sandbox/src/l7/inference.rs | 98 +++++++++++++++++++- 1 file changed, 97 insertions(+), 1 deletion(-) diff --git a/crates/openshell-sandbox/src/l7/inference.rs b/crates/openshell-sandbox/src/l7/inference.rs index ec789ef95..60a7a7e64 100644 --- a/crates/openshell-sandbox/src/l7/inference.rs +++ b/crates/openshell-sandbox/src/l7/inference.rs @@ -60,7 +60,7 @@ impl InferenceApiPattern { } } -/// Default patterns for known inference APIs (`OpenAI`, Anthropic). +/// Default patterns for known inference APIs (`OpenAI`, Anthropic, AWS Bedrock). pub fn default_patterns() -> Vec { vec![ InferenceApiPattern { @@ -114,10 +114,31 @@ pub fn default_patterns() -> Vec { kind: "models_get".to_string(), framing: ResponseFraming::Buffered, }, + // AWS Bedrock InvokeModel + InvokeModelWithResponseStream. The `*` + // segment is the Bedrock model id (e.g. `anthropic.claude-opus-4-7`). + InferenceApiPattern { + method: "POST".to_string(), + path_glob: "/model/*/invoke".to_string(), + protocol: "aws_bedrock_invoke".to_string(), + kind: "messages".to_string(), + }, + InferenceApiPattern { + method: "POST".to_string(), + path_glob: "/model/*/invoke-with-response-stream".to_string(), + protocol: "aws_bedrock_invoke_stream".to_string(), + kind: "messages".to_string(), + }, ] } /// Check if an HTTP request matches a known inference API pattern. +/// +/// Path globs support two wildcard shapes (one per pattern, not both): +/// - **Trailing `/*`**: `/v1/models/*` matches `/v1/models` and any +/// `/v1/models/` (one or many path segments). +/// - **Middle `/*/`**: `/model/*/invoke` matches `/model//invoke` +/// for a single non-empty segment that contains no `/`. Used for +/// AWS Bedrock's `/model/{modelId}/invoke[-with-response-stream]`. pub fn detect_inference_pattern<'a>( method: &str, path: &str, @@ -137,6 +158,21 @@ pub fn detect_inference_pattern<'a>( .is_some_and(|suffix| suffix.starts_with('/')); } + if let Some((before, after)) = p.path_glob.split_once("/*/") { + let Some(rest) = path_only.strip_prefix(before) else { + return false; + }; + let Some(rest) = rest.strip_prefix('/') else { + return false; + }; + // rest must look like `/` where is non-empty + // and contains no `/` (single path segment). + let Some(slash_at) = rest.find('/') else { + return false; + }; + return slash_at > 0 && rest[slash_at + 1..] == *after; + } + path_only == p.path_glob }) } @@ -543,6 +579,66 @@ mod tests { } } + #[test] + fn detect_aws_bedrock_invoke() { + let patterns = default_patterns(); + let result = + detect_inference_pattern("POST", "/model/anthropic.claude-opus-4-7/invoke", &patterns); + assert!(result.is_some()); + assert_eq!(result.unwrap().protocol, "aws_bedrock_invoke"); + assert_eq!(result.unwrap().kind, "messages"); + } + + #[test] + fn detect_aws_bedrock_invoke_stream() { + let patterns = default_patterns(); + let result = detect_inference_pattern( + "POST", + "/model/anthropic.claude-opus-4-7/invoke-with-response-stream", + &patterns, + ); + assert!(result.is_some()); + assert_eq!(result.unwrap().protocol, "aws_bedrock_invoke_stream"); + } + + #[test] + fn aws_bedrock_invoke_with_query_string() { + let patterns = default_patterns(); + let result = detect_inference_pattern("POST", "/model/foo.bar/invoke?trace=1", &patterns); + assert!(result.is_some()); + assert_eq!(result.unwrap().protocol, "aws_bedrock_invoke"); + } + + #[test] + fn aws_bedrock_rejects_empty_model_id() { + let patterns = default_patterns(); + // `/model//invoke` — empty wildcard segment is not a valid Bedrock id. + assert!(detect_inference_pattern("POST", "/model//invoke", &patterns).is_none()); + } + + #[test] + fn aws_bedrock_rejects_multi_segment_model_id() { + let patterns = default_patterns(); + // The `*` matches a single path segment only; multi-segment ids must + // not match (would be a path-traversal liability otherwise). + assert!(detect_inference_pattern("POST", "/model/foo/bar/invoke", &patterns).is_none()); + } + + #[test] + fn aws_bedrock_rejects_get() { + let patterns = default_patterns(); + assert!( + detect_inference_pattern("GET", "/model/anthropic.claude-opus-4-7/invoke", &patterns) + .is_none() + ); + } + + #[test] + fn aws_bedrock_rejects_unknown_action() { + let patterns = default_patterns(); + assert!(detect_inference_pattern("POST", "/model/foo/converse", &patterns).is_none()); + } + #[test] fn parse_simple_post_request() { let body = b"{\"hello\":true}"; From dbfb60f445c700346ddcba99833efa80128b08bb Mon Sep 17 00:00:00 2001 From: st-gr <38470677+st-gr@users.noreply.github.com> Date: Mon, 1 Jun 2026 05:03:09 -0700 Subject: [PATCH 2/8] feat(providers): add aws-bedrock provider profile + discovery spec Adds `aws-bedrock` to the built-in provider catalog so operators can run `openshell provider create --type aws-bedrock --credential ...` and have the gateway treat it as a first-class inference provider alongside `anthropic`, `openai`, etc. - `providers/aws-bedrock.yaml`: YAML profile declaring four credentials (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN, AWS_REGION). Default endpoint is `bedrock-runtime.us-east-1.amazonaws.com:443`; operators in other regions or running against a Bedrock-compatible proxy override via the operator-supplied `BEDROCK_BASE_URL` config-key (mirrors `ANTHROPIC_BASE_URL` for the `anthropic` provider). - `crates/openshell-providers/src/providers/aws_bedrock.rs`: the `ProviderDiscoverySpec` so `openshell provider create --auto-providers` picks up AWS_* env vars from local credentials. - `crates/openshell-providers/src/providers/mod.rs`: register the module. - `crates/openshell-providers/src/lib.rs`: register the SPEC in the default registry alongside the other providers. - `crates/openshell-providers/src/profiles.rs`: include the new YAML in `BUILT_IN_PROFILE_YAMLS`. What this PR explicitly does NOT add (intentionally separated for review-size reasons; will follow up): - A SigV4 signer in `openshell-router`. The current change simply declares the protocol; a follow-up PR adds outbound SigV4 signing using the `aws-sigv4` crate and a new `auth_style: sigv4` validator branch in profiles.rs. Operators who don't need SigV4 (e.g. an in-cluster bridge that ignores it and authenticates separately to the upstream) can use this PR today. - Body translation between Bedrock InvokeModel shape and other inference shapes. The router treats Bedrock requests as opaque pass-through; if the operator's upstream is real AWS Bedrock it speaks Bedrock natively, if it's a translating bridge the bridge does any conversion server-side. - `BEDROCK_BASE_URL` placeholder substitution in the YAML loader. Today the YAML's `host` is a literal default; operators override with the config-key the same way `ANTHROPIC_BASE_URL` works. Tested: `cargo test -p openshell-providers` (35 tests green) and `cargo test -p openshell-sandbox --lib l7::inference` (40 tests green including the seven new aws_bedrock cases from the previous commit). Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: st-gr <38470677+st-gr@users.noreply.github.com> --- crates/openshell-providers/src/lib.rs | 1 + crates/openshell-providers/src/profiles.rs | 1 + .../src/providers/aws_bedrock.rs | 20 ++++++++++ .../openshell-providers/src/providers/mod.rs | 1 + providers/aws-bedrock.yaml | 38 +++++++++++++++++++ 5 files changed, 61 insertions(+) create mode 100644 crates/openshell-providers/src/providers/aws_bedrock.rs create mode 100644 providers/aws-bedrock.yaml diff --git a/crates/openshell-providers/src/lib.rs b/crates/openshell-providers/src/lib.rs index 1d0d5a192..891ccc48e 100644 --- a/crates/openshell-providers/src/lib.rs +++ b/crates/openshell-providers/src/lib.rs @@ -115,6 +115,7 @@ impl ProviderRegistry { registry.register(providers::generic::GenericProvider); registry.register(providers::openai::SPEC); registry.register(providers::anthropic::SPEC); + registry.register(providers::aws_bedrock::SPEC); registry.register(providers::nvidia::SPEC); registry.register(providers::gitlab::SPEC); registry.register(providers::github::SPEC); diff --git a/crates/openshell-providers/src/profiles.rs b/crates/openshell-providers/src/profiles.rs index 624ee0711..5dca23763 100644 --- a/crates/openshell-providers/src/profiles.rs +++ b/crates/openshell-providers/src/profiles.rs @@ -19,6 +19,7 @@ use std::sync::OnceLock; const PATH_TEMPLATE_CREDENTIAL_PLACEHOLDER: &str = "{credential}"; const BUILT_IN_PROFILE_YAMLS: &[&str] = &[ + include_str!("../../../providers/aws-bedrock.yaml"), include_str!("../../../providers/claude-code.yaml"), include_str!("../../../providers/codex.yaml"), include_str!("../../../providers/copilot.yaml"), diff --git a/crates/openshell-providers/src/providers/aws_bedrock.rs b/crates/openshell-providers/src/providers/aws_bedrock.rs new file mode 100644 index 000000000..d696774f8 --- /dev/null +++ b/crates/openshell-providers/src/providers/aws_bedrock.rs @@ -0,0 +1,20 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use crate::ProviderDiscoverySpec; + +pub const SPEC: ProviderDiscoverySpec = ProviderDiscoverySpec { + id: "aws-bedrock", + credential_env_vars: &[ + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + "AWS_REGION", + ], +}; + +test_discovers_env_credential!( + discovers_aws_bedrock_env_credentials, + "AWS_ACCESS_KEY_ID", + "AKIA-test-key" +); diff --git a/crates/openshell-providers/src/providers/mod.rs b/crates/openshell-providers/src/providers/mod.rs index dfe5935a1..57a8f2053 100644 --- a/crates/openshell-providers/src/providers/mod.rs +++ b/crates/openshell-providers/src/providers/mod.rs @@ -31,6 +31,7 @@ macro_rules! test_discovers_env_credential { }; } pub mod anthropic; +pub mod aws_bedrock; pub mod claude; pub mod codex; pub mod copilot; diff --git a/providers/aws-bedrock.yaml b/providers/aws-bedrock.yaml new file mode 100644 index 000000000..90bdcce1d --- /dev/null +++ b/providers/aws-bedrock.yaml @@ -0,0 +1,38 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +id: aws-bedrock +display_name: AWS Bedrock +description: Anthropic + Mistral + Llama models served via the AWS Bedrock InvokeModel API +category: inference +inference_capable: true +credentials: + - name: aws_access_key_id + description: AWS access key id used for SigV4 signing of outbound Bedrock requests + env_vars: [AWS_ACCESS_KEY_ID] + required: true + - name: aws_secret_access_key + description: AWS secret access key paired with aws_access_key_id + env_vars: [AWS_SECRET_ACCESS_KEY] + required: true + - name: aws_session_token + description: Optional session token for temporary credentials (STS, IAM Roles for Service Accounts) + env_vars: [AWS_SESSION_TOKEN] + required: false + - name: aws_region + description: AWS region the Bedrock endpoint resolves into (e.g. us-east-1) + env_vars: [AWS_REGION, AWS_DEFAULT_REGION] + required: true +discovery: + credentials: [aws_access_key_id, aws_secret_access_key, aws_region] +endpoints: + # Default endpoint targets us-east-1 since the YAML loader does not yet + # substitute the `{region}` placeholder. Operators in other regions + # override via the `BEDROCK_BASE_URL` config-key the same way the + # `anthropic` provider accepts `ANTHROPIC_BASE_URL`. + - host: bedrock-runtime.us-east-1.amazonaws.com + port: 443 + protocol: rest + access: read-write + enforcement: enforce +binaries: [/usr/bin/claude, /usr/local/bin/claude] From ee4afc0c9c9455fd0a958ba94e07bef98fa3bd73 Mon Sep 17 00:00:00 2001 From: st-gr <38470677+st-gr@users.noreply.github.com> Date: Fri, 5 Jun 2026 01:09:16 -0700 Subject: [PATCH 3/8] revert(providers): drop legacy aws-bedrock SPEC, rely on v2 YAML profile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses johntmyers's review on NVIDIA/OpenShell#1704: net-new providers should land via the v2 YAML profile only and should NOT require changes to the legacy `ProviderDiscoverySpec` registry. - Delete `crates/openshell-providers/src/providers/aws_bedrock.rs` (the legacy SPEC + `test_discovers_env_credential!` invocation). - Drop `pub mod aws_bedrock;` from `crates/openshell-providers/src/providers/mod.rs`. - Drop `registry.register(providers::aws_bedrock::SPEC)` from `crates/openshell-providers/src/lib.rs`. Kept: - `providers/aws-bedrock.yaml` and the `include_str!` in `BUILT_IN_PROFILE_YAMLS` (`profiles.rs`) — the v2 path. `discover_from_profile()` (`crates/openshell-providers/src/discovery.rs`) picks up AWS_* env vars via `discovery.credentials` in the YAML. - L7 router patterns in `crates/openshell-sandbox/src/l7/inference.rs` — orthogonal to the provider registry. The discovery test in the deleted file goes with it; v2 doesn't have an established per-provider env-var-pickup unit test pattern, and other YAML-only registrations (none today, but this is the new direction) won't carry one either. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: st-gr <38470677+st-gr@users.noreply.github.com> --- crates/openshell-providers/src/lib.rs | 1 - .../src/providers/aws_bedrock.rs | 20 ------------------- .../openshell-providers/src/providers/mod.rs | 1 - 3 files changed, 22 deletions(-) delete mode 100644 crates/openshell-providers/src/providers/aws_bedrock.rs diff --git a/crates/openshell-providers/src/lib.rs b/crates/openshell-providers/src/lib.rs index 891ccc48e..1d0d5a192 100644 --- a/crates/openshell-providers/src/lib.rs +++ b/crates/openshell-providers/src/lib.rs @@ -115,7 +115,6 @@ impl ProviderRegistry { registry.register(providers::generic::GenericProvider); registry.register(providers::openai::SPEC); registry.register(providers::anthropic::SPEC); - registry.register(providers::aws_bedrock::SPEC); registry.register(providers::nvidia::SPEC); registry.register(providers::gitlab::SPEC); registry.register(providers::github::SPEC); diff --git a/crates/openshell-providers/src/providers/aws_bedrock.rs b/crates/openshell-providers/src/providers/aws_bedrock.rs deleted file mode 100644 index d696774f8..000000000 --- a/crates/openshell-providers/src/providers/aws_bedrock.rs +++ /dev/null @@ -1,20 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -use crate::ProviderDiscoverySpec; - -pub const SPEC: ProviderDiscoverySpec = ProviderDiscoverySpec { - id: "aws-bedrock", - credential_env_vars: &[ - "AWS_ACCESS_KEY_ID", - "AWS_SECRET_ACCESS_KEY", - "AWS_SESSION_TOKEN", - "AWS_REGION", - ], -}; - -test_discovers_env_credential!( - discovers_aws_bedrock_env_credentials, - "AWS_ACCESS_KEY_ID", - "AKIA-test-key" -); diff --git a/crates/openshell-providers/src/providers/mod.rs b/crates/openshell-providers/src/providers/mod.rs index 57a8f2053..dfe5935a1 100644 --- a/crates/openshell-providers/src/providers/mod.rs +++ b/crates/openshell-providers/src/providers/mod.rs @@ -31,7 +31,6 @@ macro_rules! test_discovers_env_credential { }; } pub mod anthropic; -pub mod aws_bedrock; pub mod claude; pub mod codex; pub mod copilot; From d26bf13161d690743d730959e6b317fd1d5e085e Mon Sep 17 00:00:00 2001 From: st-gr <38470677+st-gr@users.noreply.github.com> Date: Sun, 7 Jun 2026 10:28:53 -0700 Subject: [PATCH 4/8] fix(providers): include aws_session_token in discovery + update profile assertion Two fixes from johntmyers's gator-agent re-check on NVIDIA/OpenShell#1704: 1. `providers/aws-bedrock.yaml`: add `aws_session_token` to `discovery.credentials`. The credential is declared in the profile but was missing from the discovery scan list, so Providers v2 `--from-existing` would silently drop temporary AWS credentials (STS / IRSA scenarios). 2. `crates/openshell-server/src/grpc/provider.rs`: update the static `list_provider_profiles_returns_built_in_profile_categories` assertion to include `aws-bedrock` at alphabetical position 0. Adding `providers/aws-bedrock.yaml` to BUILT_IN_PROFILE_YAMLS made the prior `["claude-code", "github", "nvidia"]` expectation stale. Remaining blockers from the same review (deferred to follow-up commits): `inference::profile_for` registration for aws-bedrock, user-facing provider + inference-routing docs, and an `upsert_cluster_inference_route` integration test. Co-Authored-By: Claude Opus 4.7 Signed-off-by: st-gr <38470677+st-gr@users.noreply.github.com> --- crates/openshell-server/src/grpc/provider.rs | 1 + providers/aws-bedrock.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/openshell-server/src/grpc/provider.rs b/crates/openshell-server/src/grpc/provider.rs index 4552fceae..10c10868e 100644 --- a/crates/openshell-server/src/grpc/provider.rs +++ b/crates/openshell-server/src/grpc/provider.rs @@ -1821,6 +1821,7 @@ mod tests { assert_eq!( ids, vec![ + "aws-bedrock", "claude-code", "codex", "copilot", diff --git a/providers/aws-bedrock.yaml b/providers/aws-bedrock.yaml index 90bdcce1d..01a51b9e8 100644 --- a/providers/aws-bedrock.yaml +++ b/providers/aws-bedrock.yaml @@ -24,7 +24,7 @@ credentials: env_vars: [AWS_REGION, AWS_DEFAULT_REGION] required: true discovery: - credentials: [aws_access_key_id, aws_secret_access_key, aws_region] + credentials: [aws_access_key_id, aws_secret_access_key, aws_session_token, aws_region] endpoints: # Default endpoint targets us-east-1 since the YAML loader does not yet # substitute the `{region}` placeholder. Operators in other regions From 96d4805726e5570229cc698aa4dfa71a9f354abb Mon Sep 17 00:00:00 2001 From: st-gr <38470677+st-gr@users.noreply.github.com> Date: Sun, 7 Jun 2026 11:23:01 -0700 Subject: [PATCH 5/8] feat(inference): register aws-bedrock profile (bridge-fronted) + docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses johntmyers's blocking review feedback on PR #1704: "aws-bedrock still is not wired into the managed inference.local route registry. profile_for only registers openai, anthropic, and nvidia, so inference set --provider will reject this provider before the new sandbox L7 patterns can be used." Approach: register aws-bedrock as a *bridge-fronted* upstream — the router does not inject any auth header on outbound requests; the configured BEDROCK_BASE_URL is expected to point at a translating bridge / Bedrock-compatible proxy that handles auth in its own pod. This is the shape the L7 patterns commit (8b30211a) and the YAML profile (6b51e1a6) were designed for. SigV4 signing for direct AWS Bedrock is a separate follow-up; see PR thread. Changes: - core::inference::AuthHeader: add `None` variant for upstreams that authenticate themselves. - core::inference: add AWS_BEDROCK_PROFILE static + register in profile_for. Default base URL is bedrock-runtime.us-east-1, override via BEDROCK_BASE_URL config-key (mirrors ANTHROPIC_BASE_URL pattern). Empty credential_key_names + auth: None means no router-side credential lookup at route time. - router::backend: handle AuthHeader::None as a no-op (skip auth injection). - server::inference::resolve_provider_route: gate find_provider_api_key on auth != None. aws-bedrock providers with empty credentials now resolve cleanly. Updated the unsupported-type error message to include aws-bedrock in the supported list. - server::inference tests: add positive upsert_cluster_route_succeeds_for_aws_bedrock_without_api_key test covering the new code path end-to-end (provider with empty creds + BEDROCK_BASE_URL config → upsert succeeds → resolved route has empty api_key + provider_type aws-bedrock + bridge URL). - core::inference tests: profile_for_known_types covers aws-bedrock, case-insensitive lookup, plus three new aws-bedrock-specific tests (auth: None, no credential keys, bedrock-specific protocols). - docs/sandboxes/inference-routing.mdx: header forwarding row mentions aws-bedrock has no passthrough headers; new tabs in Supported API Patterns (InvokeModel + InvokeModelWithResponseStream) and Create a Provider (with the bridge-fronted shape note + SigV4 deferral). - docs/sandboxes/manage-providers.mdx: new row in Supported Provider Types table; new row in Supported Inference Providers table. Verification (in dev container): - cargo check -p openshell-core -p openshell-router -p openshell-server: clean - cargo test -p openshell-core --lib inference: 14/14 pass (incl. 3 new) - cargo test -p openshell-server --lib inference::tests::upsert: 6/6 pass (incl. new aws-bedrock test) - cargo fmt --check: clean - cargo clippy --all-targets -D warnings: clean Co-Authored-By: Claude Opus 4.7 Signed-off-by: st-gr <38470677+st-gr@users.noreply.github.com> --- crates/openshell-core/src/inference.rs | 68 +++++++++++++++ crates/openshell-router/src/backend.rs | 7 ++ crates/openshell-server/src/inference.rs | 105 +++++++++++++++++++---- docs/sandboxes/inference-routing.mdx | 32 ++++++- docs/sandboxes/manage-providers.mdx | 2 + 5 files changed, 197 insertions(+), 17 deletions(-) diff --git a/crates/openshell-core/src/inference.rs b/crates/openshell-core/src/inference.rs index 3071d53cd..821d907ca 100644 --- a/crates/openshell-core/src/inference.rs +++ b/crates/openshell-core/src/inference.rs @@ -18,6 +18,15 @@ pub enum AuthHeader { Bearer, /// Custom header name (e.g. `x-api-key` for Anthropic). Custom(&'static str), + /// Do not inject any auth header on outgoing requests. The upstream + /// is expected to authenticate itself — used when the configured + /// `default_base_url` (or operator-supplied base-URL override) points + /// at a translating bridge / proxy that holds operator-side + /// credentials in its own pod and ignores caller-supplied auth. + /// Currently used by the `aws-bedrock` profile, where `SigV4` signing + /// is deferred to a follow-up PR; today the only supported shape is + /// a bridge-fronted upstream. + None, } // --------------------------------------------------------------------------- @@ -69,6 +78,8 @@ const ANTHROPIC_PROTOCOLS: &[&str] = &["anthropic_messages", "model_discovery"]; /// base-URL-override escape hatch path. const VERTEX_AI_PROTOCOLS: &[&str] = &["anthropic_messages", "model_discovery"]; +const AWS_BEDROCK_PROTOCOLS: &[&str] = &["aws_bedrock_invoke", "aws_bedrock_invoke_stream"]; + static OPENAI_PROFILE: InferenceProviderProfile = InferenceProviderProfile { provider_type: "openai", default_base_url: "https://api.openai.com/v1", @@ -155,6 +166,37 @@ static NVIDIA_PROFILE: InferenceProviderProfile = InferenceProviderProfile { passthrough_headers: &["x-model-id"], }; +// AWS Bedrock — registered as bridge-fronted (no router-side auth +// injection). Real AWS Bedrock requires `SigV4` signing of every request, +// which is deferred to a follow-up PR (see #1704 thread). Until then, +// operators point `BEDROCK_BASE_URL` at a translating bridge or +// Bedrock-compatible proxy that handles auth in its own pod. The router +// passes Bedrock InvokeModel requests through opaquely; the L7 patterns +// `/model/{modelId}/invoke` and `/model/{modelId}/invoke-with-response-stream` +// are wired up in `crates/openshell-sandbox/src/l7/inference.rs`. +// +// Note: `default_base_url` is intentionally an empty string. Without +// `BEDROCK_BASE_URL` config, route resolution rejects the provider +// rather than silently forwarding prompts to real AWS Bedrock with +// `auth: None` (which would fail upstream and risks operator +// surprise). Once the `SigV4` follow-up lands, the default can revert +// to `https://bedrock-runtime.us-east-1.amazonaws.com`. +static AWS_BEDROCK_PROFILE: InferenceProviderProfile = InferenceProviderProfile { + provider_type: "aws-bedrock", + default_base_url: "", + protocols: AWS_BEDROCK_PROTOCOLS, + // No single API key for Bedrock — `SigV4` takes four credentials + // (access key id, secret, session token, region) and signs requests + // rather than injecting a header. Until the `SigV4` follow-up lands + // the router-side auth shape is `None` and no credential lookup is + // required at route time. + credential_key_names: &[], + base_url_config_keys: &["BEDROCK_BASE_URL"], + auth: AuthHeader::None, + default_headers: &[], + passthrough_headers: &[], +}; + /// Canonicalize an inference provider type string to a well-known identifier. /// /// Returns `Some(canonical_name)` for recognized inference providers, @@ -167,6 +209,7 @@ pub fn normalize_inference_provider_type(input: &str) -> Option<&'static str> { "openai" => Some("openai"), "anthropic" => Some("anthropic"), "nvidia" => Some("nvidia"), + "aws-bedrock" => Some("aws-bedrock"), "google-vertex-ai" | "vertex" | "vertex-ai" | "google-vertex" | "gcp-vertex" => { Some("google-vertex-ai") } @@ -184,6 +227,7 @@ pub fn profile_for(provider_type: &str) -> Option<&'static InferenceProviderProf "anthropic" => Some(&ANTHROPIC_PROFILE), "nvidia" => Some(&NVIDIA_PROFILE), "google-vertex-ai" => Some(&VERTEX_AI_PROFILE), + "aws-bedrock" => Some(&AWS_BEDROCK_PROFILE), _ => None, } } @@ -303,7 +347,31 @@ mod tests { assert!(profile_for("openai").is_some()); assert!(profile_for("anthropic").is_some()); assert!(profile_for("nvidia").is_some()); + assert!(profile_for("aws-bedrock").is_some()); assert!(profile_for("OpenAI").is_some()); // case insensitive + assert!(profile_for("AWS-Bedrock").is_some()); // case insensitive + } + + #[test] + fn aws_bedrock_uses_no_auth_header() { + let (auth, headers) = auth_for_provider_type("aws-bedrock"); + assert_eq!(auth, AuthHeader::None); + assert!(headers.is_empty()); + } + + #[test] + fn aws_bedrock_profile_has_no_credential_keys() { + let profile = profile_for("aws-bedrock").expect("profile registered"); + // No router-side credential lookup until the `SigV4` follow-up. + assert!(profile.credential_key_names.is_empty()); + assert_eq!(profile.base_url_config_keys, &["BEDROCK_BASE_URL"]); + } + + #[test] + fn aws_bedrock_protocols_are_bedrock_specific() { + let profile = profile_for("aws-bedrock").expect("profile registered"); + assert!(profile.protocols.contains(&"aws_bedrock_invoke")); + assert!(profile.protocols.contains(&"aws_bedrock_invoke_stream")); } #[test] diff --git a/crates/openshell-router/src/backend.rs b/crates/openshell-router/src/backend.rs index 9eb63c88b..a0e9ba672 100644 --- a/crates/openshell-router/src/backend.rs +++ b/crates/openshell-router/src/backend.rs @@ -216,6 +216,13 @@ fn prepare_backend_request( AuthHeader::Custom(header_name) => { builder = builder.header(*header_name, &route.api_key); } + AuthHeader::None => { + // Bridge-fronted upstream: no router-side auth injection. + // The configured `endpoint` is expected to be a translating + // bridge / proxy whose own pod holds operator-side + // credentials. Used today by the `aws-bedrock` profile + // (SigV4 signing is a separate follow-up). + } } for (name, value) in &headers { builder = builder.header(name.as_str(), value.as_str()); diff --git a/crates/openshell-server/src/inference.rs b/crates/openshell-server/src/inference.rs index 13496cd99..7d5d4e8bf 100644 --- a/crates/openshell-server/src/inference.rs +++ b/crates/openshell-server/src/inference.rs @@ -620,26 +620,33 @@ fn resolve_provider_route( let profile = openshell_core::inference::profile_for(&provider_type).ok_or_else(|| { Status::invalid_argument(format!( "provider '{name}' has unsupported type '{raw_provider_type}' for cluster inference \ - (supported: openai, anthropic, nvidia, google-vertex-ai)", + (supported: openai, anthropic, nvidia, google-vertex-ai, aws-bedrock)", name = provider.object_name() )) })?; - let api_key = find_provider_api_key( - provider, - profile.credential_key_names, - if provider_type == "google-vertex-ai" { - CredentialLookup::PreferredOnly - } else { - CredentialLookup::PreferredThenAny - }, - ) - .ok_or_else(|| { - Status::invalid_argument(format!( - "provider '{name}' has no usable API key credential", - name = provider.object_name() - )) - })?; + // Profiles with `auth: None` are bridge-fronted — the upstream + // authenticates itself, so the router doesn't need a credential at + // route-resolution time. Today this is `aws-bedrock`. + let api_key = if matches!(profile.auth, openshell_core::inference::AuthHeader::None) { + String::new() + } else { + find_provider_api_key( + provider, + profile.credential_key_names, + if provider_type == "google-vertex-ai" { + CredentialLookup::PreferredOnly + } else { + CredentialLookup::PreferredThenAny + }, + ) + .ok_or_else(|| { + Status::invalid_argument(format!( + "provider '{name}' has no usable API key credential", + name = provider.object_name() + )) + })? + }; // Vertex AI requires a model-aware URL; delegate to specialised resolver. if provider_type == "google-vertex-ai" { @@ -1059,6 +1066,72 @@ mod tests { assert_eq!(config.model_id, "gpt-4.1"); } + #[tokio::test] + async fn upsert_cluster_route_succeeds_for_aws_bedrock_without_api_key() { + // aws-bedrock is registered with `auth: AuthHeader::None` (the + // bridge-fronted shape) so route resolution must NOT require a + // credential lookup. Create a provider with empty credentials + // and confirm upsert succeeds, the resolved route carries + // `provider_type: "aws-bedrock"`, the default Bedrock base URL, + // and an empty api_key. SigV4 signing for direct AWS Bedrock is + // a separate follow-up; this test pins down the bridge-fronted + // contract this PR delivers. + let store = test_store().await; + + let provider = Provider { + metadata: Some(openshell_core::proto::datamodel::v1::ObjectMeta { + id: "provider-bedrock-bridge".to_string(), + name: "bedrock-bridge".to_string(), + created_at_ms: 1_000_000, + labels: std::collections::HashMap::new(), + resource_version: 0, + }), + r#type: "aws-bedrock".to_string(), + // Bridge-fronted: no credential needed at route-resolution + // time. The bridge holds operator-side auth in its own pod. + credentials: std::collections::HashMap::new(), + config: std::iter::once(( + "BEDROCK_BASE_URL".to_string(), + "http://bedrock-bridge.demo.svc.cluster.local:8080".to_string(), + )) + .collect(), + credential_expires_at_ms: std::collections::HashMap::new(), + }; + store + .put_message(&provider) + .await + .expect("provider should persist"); + + let upserted = upsert_cluster_inference_route( + &store, + CLUSTER_INFERENCE_ROUTE_NAME, + "bedrock-bridge", + "anthropic.claude-3-5-sonnet-20241022-v2:0", + 0, + false, + ) + .await + .expect("upsert should succeed for aws-bedrock provider"); + + assert_eq!(upserted.route.object_name(), CLUSTER_INFERENCE_ROUTE_NAME); + let config = upserted.route.config.as_ref().expect("config"); + assert_eq!(config.provider_name, "bedrock-bridge"); + assert_eq!(config.model_id, "anthropic.claude-3-5-sonnet-20241022-v2:0"); + + // Verify the resolved route metadata reflects bridge-fronted + // auth (empty api_key + provider_type = "aws-bedrock"). + let managed = resolve_route_by_name(&store, CLUSTER_INFERENCE_ROUTE_NAME) + .await + .expect("route should resolve") + .expect("managed route should exist"); + assert_eq!(managed.provider_type, "aws-bedrock"); + assert_eq!( + managed.base_url, + "http://bedrock-bridge.demo.svc.cluster.local:8080" + ); + assert_eq!(managed.api_key, ""); + } + #[tokio::test] async fn resolve_managed_route_returns_none_when_missing() { let store = test_store().await; diff --git a/docs/sandboxes/inference-routing.mdx b/docs/sandboxes/inference-routing.mdx index 0a4e9d726..2c75d60c0 100644 --- a/docs/sandboxes/inference-routing.mdx +++ b/docs/sandboxes/inference-routing.mdx @@ -24,7 +24,7 @@ If code calls an external inference host directly, OpenShell evaluates that traf | Property | Detail | |---|---| | Credentials | No sandbox API keys needed. Credentials come from the configured provider record. The router strips caller-supplied `Authorization` before forwarding the request. | -| Header forwarding | `inference.local` forwards only a per-provider header allowlist. OpenAI routes allow `openai-organization` and `x-model-id`. Anthropic routes allow `anthropic-version` and `anthropic-beta`. Vertex Claude rawPredict routes strip `anthropic-beta` and do not forward `anthropic-version` as a header because the router injects `anthropic_version` into the Vertex request body. NVIDIA routes allow `x-model-id`. All other caller headers are stripped. | +| Header forwarding | `inference.local` forwards only a per-provider header allowlist. OpenAI routes allow `openai-organization` and `x-model-id`. Anthropic routes allow `anthropic-version` and `anthropic-beta`. Vertex Claude rawPredict routes strip `anthropic-beta` and do not forward `anthropic-version` as a header because the router injects `anthropic_version` into the Vertex request body. NVIDIA routes allow `x-model-id`. AWS Bedrock routes have no passthrough headers today. All other caller headers are stripped. | | Configuration | One provider and one model define sandbox inference for the active gateway. Every sandbox on that gateway sees the same `inference.local` backend. | | Provider support | NVIDIA, Anthropic, Google Vertex AI, and any OpenAI-compatible provider all work through the same endpoint. Vertex routes Claude models through `/v1/messages` and non-Anthropic models through `/v1/chat/completions`. The gateway resolves the upstream Vertex host from the provider config, including regional, global, and supported multi-region endpoints. | | Streaming reliability | The router tolerates idle gaps of up to 120 seconds between streamed chunks so long reasoning responses are not cut off mid-stream. | @@ -54,6 +54,21 @@ Supported request patterns depend on the provider configured for `inference.loca |---|---|---| | Messages | `POST` | `/v1/messages` | + + + + +| Pattern | Method | Path | +|---|---|---| +| InvokeModel | `POST` | `/model/{modelId}/invoke` | +| InvokeModelWithResponseStream | `POST` | `/model/{modelId}/invoke-with-response-stream` | + +The `{modelId}` segment is constrained to a single non-empty path segment to avoid path-traversal liabilities. `/model//invoke` and `/model/a/b/invoke` both no-match. + + +Today the `aws-bedrock` provider type is bridge-fronted only. The router does not inject any auth header on outbound requests; the configured `BEDROCK_BASE_URL` is expected to point at a translating bridge or Bedrock-compatible proxy whose own pod holds operator-side credentials. SigV4 signing for direct AWS Bedrock is deferred to a follow-up release. + + @@ -148,6 +163,21 @@ openshell provider create --name anthropic-prod --type anthropic --from-existing This reads `ANTHROPIC_API_KEY` from your environment. + + + + +```shell +openshell provider create \ + --name bedrock-bridge \ + --type aws-bedrock \ + --config BEDROCK_BASE_URL=http://your-bedrock-bridge.your-ns.svc.cluster.local:8080 +``` + +The `aws-bedrock` provider type is bridge-fronted: the router does not inject any auth header on outbound requests. Point `BEDROCK_BASE_URL` at a translating bridge or Bedrock-compatible proxy that handles authentication in its own pod. The bridge is expected to accept Bedrock InvokeModel requests on the patterns listed above and forward to the operator's real upstream. + +For direct AWS Bedrock with SigV4 signing, refer to a future release that adds the SigV4 router-side signer. Today the discovery scan picks up `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_SESSION_TOKEN`, and `AWS_REGION` from your environment via `--from-existing`, but the router does not yet sign requests with them. + diff --git a/docs/sandboxes/manage-providers.mdx b/docs/sandboxes/manage-providers.mdx index a6b9654d0..bc522e2ce 100644 --- a/docs/sandboxes/manage-providers.mdx +++ b/docs/sandboxes/manage-providers.mdx @@ -247,6 +247,7 @@ The following provider types are supported. | Type | Environment Variables Injected | Typical Use | |---|---|---| | `anthropic` | `ANTHROPIC_API_KEY` | Anthropic API | +| `aws-bedrock` | `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_SESSION_TOKEN`, `AWS_REGION` | AWS Bedrock InvokeModel via a translating bridge. Today the router does not inject any auth header; the configured `BEDROCK_BASE_URL` upstream is expected to handle auth itself. Refer to [Inference Routing](/sandboxes/inference-routing). | | `claude` | `ANTHROPIC_API_KEY`, `CLAUDE_API_KEY` | Claude Code, Anthropic API | | `codex` | `OPENAI_API_KEY` | OpenAI Codex | | `copilot` | `COPILOT_GITHUB_TOKEN`, `GH_TOKEN`, `GITHUB_TOKEN` | GitHub Copilot CLI | @@ -273,6 +274,7 @@ The following providers have been tested with `inference.local`. Any provider th | Provider | Name | Type | Base URL | API Key Variable | |---|---|---|---|---| +| AWS Bedrock (via bridge) | `bedrock-bridge` | `aws-bedrock` | Operator-supplied `BEDROCK_BASE_URL` | None at router level (bridge holds creds) | | NVIDIA API Catalog | `nvidia-prod` | `nvidia` | `https://integrate.api.nvidia.com/v1` | `NVIDIA_API_KEY` | | Anthropic | `anthropic-prod` | `anthropic` | `https://api.anthropic.com` | `ANTHROPIC_API_KEY` | | Google Vertex AI | `vertex-prod` | `google-vertex-ai` | Regional, global, or multi-region Vertex endpoint | `GOOGLE_VERTEX_AI_TOKEN` or `GOOGLE_VERTEX_AI_SERVICE_ACCOUNT_TOKEN` | From 424b1e2f3fc127eb708d82c5a65b500ae1f672fe Mon Sep 17 00:00:00 2001 From: st-gr <38470677+st-gr@users.noreply.github.com> Date: Mon, 8 Jun 2026 15:29:47 -0700 Subject: [PATCH 6/8] fix(aws-bedrock): bridge-only YAML, doc actual cmd shape, neg test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses four findings from gator-agent's #1704 re-check on 4ab587f1: - **Item 5** (YAML collects unused AWS creds): mark all four AWS credentials `required: false` and clear `discovery.credentials`. Bridge-fronted routing intentionally does not consume AWS credentials, so `--from-existing` no longer scans for them. The credentials remain in the schema (not deleted) so the SigV4 follow-up can flip them back without a schema migration. Added a multi-line description that names the bridge-fronted shape and the SigV4 deferral so readers don't have to cross-reference the PR thread. - **Item 3** (docs show command that the CLI rejects): rewrite the Create-a-Provider example for AWS Bedrock to use the actual required shape — placeholder `--credential AWS_ACCESS_KEY_ID= unused-bridge-fronted-shape` plus the `--config BEDROCK_BASE_URL`. The placeholder satisfies the gRPC handler's `provider.credentials.is_empty()` rejection without expanding server-side validation; the router ignores it on the outbound path because `auth: AuthHeader::None` skips header injection. Operators see a clearly-labeled placeholder in `provider get` output. - **Item 1** (validator probe): document `--no-verify` as required for `openshell inference set --provider ` since the default validation probe doesn't recognize the `aws_bedrock_invoke` / `aws_bedrock_invoke_stream` protocols. Doc now shows the full `provider create` + `inference set --no-verify` flow with rationale for both decisions inline. - **Item 6** (docs polish): `inference-routing.mdx` summary row now lists AWS Bedrock alongside NVIDIA, Anthropic, Vertex AI, and OpenAI-compatible providers, with the bridge-fronted caveat inline. Test additions in `crates/openshell-server/src/inference.rs`: - Renamed the existing aws-bedrock test from `..._without_api_key` to `..._with_bridge_url` and updated it to use a placeholder credential (mirroring the doc-recommended pattern operators will copy-paste). The `auth: None` path still produces an empty `api_key` on the resolved route — the test now documents that the credential is *stored* but not *used*. - Added `upsert_cluster_route_rejects_aws_bedrock_without_bedrock_base_url`: the negative half of johntmyers' "successfully used by upsert_cluster_inference_route or intentionally rejected with a clear documented error" ask. With `default_base_url: ""` and no `BEDROCK_BASE_URL` config, route resolution returns `InvalidArgument` naming the missing base_url rather than silently forwarding prompts to AWS Bedrock with no usable auth. Verification (in dev container): - cargo test -p openshell-core --lib inference: 18/18 (incl. 3 new) - cargo test -p openshell-server --lib inference::tests::upsert: 8/8 (incl. 2 new aws-bedrock cases — positive + negative) - cargo fmt --check: clean - cargo clippy --all-targets -D warnings: clean Item 2 (router-side enforcement of operator-configured Bedrock model path, replacing the current verbatim path forwarding + body-only model rewrite) is the remaining blocker and is genuinely separable — it touches the L7 router with streaming-aware test coverage. Deferring to its own commit so the security-critical change gets the review attention it deserves. Co-Authored-By: Claude Opus 4.7 Signed-off-by: st-gr <38470677+st-gr@users.noreply.github.com> --- crates/openshell-server/src/inference.rs | 91 ++++++++++++++++++++---- docs/sandboxes/inference-routing.mdx | 20 +++++- providers/aws-bedrock.yaml | 43 +++++++---- 3 files changed, 127 insertions(+), 27 deletions(-) diff --git a/crates/openshell-server/src/inference.rs b/crates/openshell-server/src/inference.rs index 7d5d4e8bf..519039c75 100644 --- a/crates/openshell-server/src/inference.rs +++ b/crates/openshell-server/src/inference.rs @@ -1067,15 +1067,19 @@ mod tests { } #[tokio::test] - async fn upsert_cluster_route_succeeds_for_aws_bedrock_without_api_key() { + async fn upsert_cluster_route_succeeds_for_aws_bedrock_with_bridge_url() { // aws-bedrock is registered with `auth: AuthHeader::None` (the - // bridge-fronted shape) so route resolution must NOT require a - // credential lookup. Create a provider with empty credentials - // and confirm upsert succeeds, the resolved route carries - // `provider_type: "aws-bedrock"`, the default Bedrock base URL, - // and an empty api_key. SigV4 signing for direct AWS Bedrock is - // a separate follow-up; this test pins down the bridge-fronted - // contract this PR delivers. + // bridge-fronted shape) so route resolution does NOT require a + // real API key — but `provider create` still requires a + // non-empty credentials map at the gRPC layer, so operators + // pass a placeholder credential per the docs. The router + // ignores it on the outbound path. + // + // The other half of the contract is `BEDROCK_BASE_URL`: with + // `default_base_url: ""` in the core profile, providers + // without it fail route resolution rather than silently + // forwarding prompts to AWS Bedrock with no usable auth. This + // test pins down the success path. let store = test_store().await; let provider = Provider { @@ -1087,9 +1091,14 @@ mod tests { resource_version: 0, }), r#type: "aws-bedrock".to_string(), - // Bridge-fronted: no credential needed at route-resolution - // time. The bridge holds operator-side auth in its own pod. - credentials: std::collections::HashMap::new(), + // Placeholder credential — the router ignores it because + // auth: None skips header injection. Mirrors the + // doc-recommended `--credential AWS_ACCESS_KEY_ID=unused-bridge-fronted-shape`. + credentials: std::iter::once(( + "AWS_ACCESS_KEY_ID".to_string(), + "unused-bridge-fronted-shape".to_string(), + )) + .collect(), config: std::iter::once(( "BEDROCK_BASE_URL".to_string(), "http://bedrock-bridge.demo.svc.cluster.local:8080".to_string(), @@ -1119,7 +1128,9 @@ mod tests { assert_eq!(config.model_id, "anthropic.claude-3-5-sonnet-20241022-v2:0"); // Verify the resolved route metadata reflects bridge-fronted - // auth (empty api_key + provider_type = "aws-bedrock"). + // auth (empty api_key + provider_type = "aws-bedrock"). Note + // the api_key is empty even though the provider has a + // credential — auth: None skips api-key lookup entirely. let managed = resolve_route_by_name(&store, CLUSTER_INFERENCE_ROUTE_NAME) .await .expect("route should resolve") @@ -1132,6 +1143,62 @@ mod tests { assert_eq!(managed.api_key, ""); } + #[tokio::test] + async fn upsert_cluster_route_rejects_aws_bedrock_without_bedrock_base_url() { + // The companion to upsert_cluster_route_succeeds_for_aws_bedrock_with_bridge_url: + // an aws-bedrock provider without BEDROCK_BASE_URL must be + // rejected at route resolution. This pins down the safety + // contract johntmyers asked for — until the SigV4 follow-up + // lands, the router must NOT silently forward prompts to AWS + // with auth: None. + // + // Mechanism: AWS_BEDROCK_PROFILE.default_base_url is "". When + // the provider has no BEDROCK_BASE_URL config, base_url + // resolves to empty, triggering the existing + // empty-base_url check in resolve_provider_route. + let store = test_store().await; + + let provider = Provider { + metadata: Some(openshell_core::proto::datamodel::v1::ObjectMeta { + id: "provider-bedrock-misconfigured".to_string(), + name: "bedrock-misconfigured".to_string(), + created_at_ms: 1_000_000, + labels: std::collections::HashMap::new(), + resource_version: 0, + }), + r#type: "aws-bedrock".to_string(), + credentials: std::iter::once(( + "AWS_ACCESS_KEY_ID".to_string(), + "unused-bridge-fronted-shape".to_string(), + )) + .collect(), + // Intentionally no BEDROCK_BASE_URL. + config: std::collections::HashMap::new(), + credential_expires_at_ms: std::collections::HashMap::new(), + }; + store + .put_message(&provider) + .await + .expect("provider should persist"); + + let err = upsert_cluster_inference_route( + &store, + CLUSTER_INFERENCE_ROUTE_NAME, + "bedrock-misconfigured", + "anthropic.claude-3-5-sonnet-20241022-v2:0", + 0, + false, + ) + .await + .expect_err("upsert should reject aws-bedrock provider without BEDROCK_BASE_URL"); + assert_eq!(err.code(), tonic::Code::InvalidArgument); + assert!( + err.message().contains("empty base_url"), + "error should name the missing base_url, got: {}", + err.message() + ); + } + #[tokio::test] async fn resolve_managed_route_returns_none_when_missing() { let store = test_store().await; diff --git a/docs/sandboxes/inference-routing.mdx b/docs/sandboxes/inference-routing.mdx index 2c75d60c0..e9ec37d85 100644 --- a/docs/sandboxes/inference-routing.mdx +++ b/docs/sandboxes/inference-routing.mdx @@ -26,7 +26,7 @@ If code calls an external inference host directly, OpenShell evaluates that traf | Credentials | No sandbox API keys needed. Credentials come from the configured provider record. The router strips caller-supplied `Authorization` before forwarding the request. | | Header forwarding | `inference.local` forwards only a per-provider header allowlist. OpenAI routes allow `openai-organization` and `x-model-id`. Anthropic routes allow `anthropic-version` and `anthropic-beta`. Vertex Claude rawPredict routes strip `anthropic-beta` and do not forward `anthropic-version` as a header because the router injects `anthropic_version` into the Vertex request body. NVIDIA routes allow `x-model-id`. AWS Bedrock routes have no passthrough headers today. All other caller headers are stripped. | | Configuration | One provider and one model define sandbox inference for the active gateway. Every sandbox on that gateway sees the same `inference.local` backend. | -| Provider support | NVIDIA, Anthropic, Google Vertex AI, and any OpenAI-compatible provider all work through the same endpoint. Vertex routes Claude models through `/v1/messages` and non-Anthropic models through `/v1/chat/completions`. The gateway resolves the upstream Vertex host from the provider config, including regional, global, and supported multi-region endpoints. | +| Provider support | NVIDIA, Anthropic, Google Vertex AI, AWS Bedrock (via a translating bridge — direct AWS with SigV4 signing is a separate follow-up), and any OpenAI-compatible provider all work through the same endpoint. Vertex routes Claude models through `/v1/messages` and non-Anthropic models through `/v1/chat/completions`. The gateway resolves the upstream Vertex host from the provider config, including regional, global, and supported multi-region endpoints. | | Streaming reliability | The router tolerates idle gaps of up to 120 seconds between streamed chunks so long reasoning responses are not cut off mid-stream. | | Hot refresh | OpenShell picks up provider credential changes and inference updates without recreating sandboxes. Changes propagate within about 5 seconds by default. | @@ -171,12 +171,26 @@ This reads `ANTHROPIC_API_KEY` from your environment. openshell provider create \ --name bedrock-bridge \ --type aws-bedrock \ + --credential AWS_ACCESS_KEY_ID=unused-bridge-fronted-shape \ --config BEDROCK_BASE_URL=http://your-bedrock-bridge.your-ns.svc.cluster.local:8080 ``` -The `aws-bedrock` provider type is bridge-fronted: the router does not inject any auth header on outbound requests. Point `BEDROCK_BASE_URL` at a translating bridge or Bedrock-compatible proxy that handles authentication in its own pod. The bridge is expected to accept Bedrock InvokeModel requests on the patterns listed above and forward to the operator's real upstream. +Then set the inference route, passing `--no-verify` because the validation probe does not yet support Bedrock protocols: -For direct AWS Bedrock with SigV4 signing, refer to a future release that adds the SigV4 router-side signer. Today the discovery scan picks up `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_SESSION_TOKEN`, and `AWS_REGION` from your environment via `--from-existing`, but the router does not yet sign requests with them. +```shell +openshell inference set \ + --provider bedrock-bridge \ + --model anthropic.claude-3-5-sonnet-20241022-v2:0 \ + --no-verify +``` + +**Why a placeholder credential?** `provider create` requires a non-empty `credentials` map. The bridge-fronted shape ignores AWS credentials at the router level (the bridge holds operator-side auth in its own pod), so any string value satisfies the structural requirement; `unused-bridge-fronted-shape` makes the intent obvious in `openshell provider get` output. When the SigV4 follow-up lands, this becomes a real key. + +**About the bridge-fronted shape.** The router does not inject any auth header on outbound requests. Point `BEDROCK_BASE_URL` at a translating bridge or Bedrock-compatible proxy that handles authentication in its own pod. The bridge is expected to accept Bedrock InvokeModel requests on the patterns listed above and forward to the operator's real upstream. + +**About `--no-verify`.** The default validation probe does not yet recognize the `aws_bedrock_invoke` and `aws_bedrock_invoke_stream` protocols, so without `--no-verify` the `inference set` call would fail before it could mint a route. The first sandbox round-trip is the real verification today. + +**For direct AWS Bedrock**, refer to a future release that adds the SigV4 router-side signer. Until then, a `BEDROCK_BASE_URL` is required at provider-create time — the core profile sets `default_base_url: ""`, so route resolution rejects providers without it rather than silently forwarding prompts to AWS with no usable auth. diff --git a/providers/aws-bedrock.yaml b/providers/aws-bedrock.yaml index 01a51b9e8..e284a3bd9 100644 --- a/providers/aws-bedrock.yaml +++ b/providers/aws-bedrock.yaml @@ -3,33 +3,52 @@ id: aws-bedrock display_name: AWS Bedrock -description: Anthropic + Mistral + Llama models served via the AWS Bedrock InvokeModel API +description: | + Anthropic + Mistral + Llama models served via the AWS Bedrock InvokeModel API. + + Today this profile supports a bridge-fronted shape only: operators + point `BEDROCK_BASE_URL` at a translating bridge or + Bedrock-compatible proxy that handles auth in its own pod. Direct + AWS Bedrock with router-side SigV4 signing is a separate follow-up; + until that lands the AWS_* credentials below are declarative + schema only — none are required, none are auto-discovered, and the + router does not consume them. category: inference inference_capable: true credentials: + # Declarative-only until the SigV4 follow-up lands. None of these are + # required for the bridge-fronted shape; the router does not inject + # them on outbound requests. - name: aws_access_key_id - description: AWS access key id used for SigV4 signing of outbound Bedrock requests + description: AWS access key id (used by the SigV4 signer follow-up; unused today) env_vars: [AWS_ACCESS_KEY_ID] - required: true + required: false - name: aws_secret_access_key - description: AWS secret access key paired with aws_access_key_id + description: AWS secret access key (used by the SigV4 signer follow-up; unused today) env_vars: [AWS_SECRET_ACCESS_KEY] - required: true + required: false - name: aws_session_token description: Optional session token for temporary credentials (STS, IAM Roles for Service Accounts) env_vars: [AWS_SESSION_TOKEN] required: false - name: aws_region - description: AWS region the Bedrock endpoint resolves into (e.g. us-east-1) + description: AWS region (used by the SigV4 signer follow-up; unused today) env_vars: [AWS_REGION, AWS_DEFAULT_REGION] - required: true + required: false discovery: - credentials: [aws_access_key_id, aws_secret_access_key, aws_session_token, aws_region] + # Bridge-fronted routing intentionally does not consume AWS + # credentials, so `--from-existing` does not scan for them today. + # The SigV4 follow-up will repopulate this list. + credentials: [] endpoints: - # Default endpoint targets us-east-1 since the YAML loader does not yet - # substitute the `{region}` placeholder. Operators in other regions - # override via the `BEDROCK_BASE_URL` config-key the same way the - # `anthropic` provider accepts `ANTHROPIC_BASE_URL`. + # Default endpoint targets us-east-1 for the SigV4-fronted shape that + # comes later. The YAML loader does not yet substitute a `{region}` + # placeholder; operators in other regions or running against a bridge + # override via the `BEDROCK_BASE_URL` config-key (mirrors how the + # `anthropic` profile accepts `ANTHROPIC_BASE_URL`). Until then the + # core profile sets `default_base_url: ""` so route resolution + # rejects providers without `BEDROCK_BASE_URL` rather than silently + # forwarding prompts to AWS with no usable auth. - host: bedrock-runtime.us-east-1.amazonaws.com port: 443 protocol: rest From a1055e21b73baf9a8f96ca8249fbe05b75ff000e Mon Sep 17 00:00:00 2001 From: st-gr <38470677+st-gr@users.noreply.github.com> Date: Mon, 8 Jun 2026 17:07:24 -0700 Subject: [PATCH 7/8] fix(router): enforce operator-configured Bedrock model in request path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the security-blocking item from gator-agent's #1704 re-check on 4ab587f1: "Bedrock carries the model id in /model/{modelId}/invoke, but the router currently forwards the caller's original path and only rewrites JSON body model. That lets sandbox code choose a different upstream model than the operator-configured route model, and may also mutate native Bedrock request bodies incorrectly." Two changes in `prepare_backend_request`: 1. **Path rewrite for Bedrock routes.** Before computing the upstream URL, parse the inbound path's `/model//invoke[-with-response-stream]` shape and substitute the operator-configured `route.model` for the caller-supplied model segment. Sandbox code that hardcodes a different model still works (we don't reject on mismatch), but the operator's configured model is what reaches the upstream / bridge. If the inbound path is somehow not a recognized Bedrock shape on a Bedrock route (the L7 pattern detector upstream of the router should never produce this combination), reject with RouterError::Internal naming the offending path rather than forwarding verbatim. 2. **Skip body-model injection for Bedrock routes.** The existing body rewriter unconditionally inserts `route.model` into the JSON body for non-Vertex routes. AWS Bedrock InvokeModel encodes the model in the URL path; the body is the raw provider-specific payload (Anthropic Messages for Claude, Mistral payload for Mistral, etc.) and must not be mutated. The branch ordering is now: needs_vertex_anthropic_version → strip body model + inject anthropic_version; route_is_bedrock → leave body alone; else → inject route.model (existing default). New helpers, all in `crates/openshell-router/src/backend.rs`: - `route_is_bedrock(route)` — true when route.protocols contains aws_bedrock_invoke or aws_bedrock_invoke_stream. - `parse_bedrock_invocation_path(path)` — returns Some((model_id, "/invoke" | "/invoke-with-response-stream")) for paths matching the recognized Bedrock shapes. Strips query strings. Rejects empty model ids and multi-segment ids (defense-in-depth matching the L7 pattern detector's existing guards). - `rewrite_bedrock_path(route, path)` — returns the path with the caller's model segment replaced by route.model. Test coverage in the same file (9 new tests): - parse_bedrock_invocation_path: positive cases for both invoke variants, query-string stripping; negative cases for empty model id, multi-segment id, unknown action, wrong prefix, missing slash. - route_is_bedrock: matches both protocol variants singly and combined; rejects openai_chat_completions. - rewrite_bedrock_path: substitutes operator model on both invoke variants; returns None for non-Bedrock paths. - bedrock_route_rewrites_model_in_path_and_preserves_body (wiremock end-to-end): caller sends /model/some-other-model/invoke with a body containing model: "caller-supplied-model-name". Mock asserts the upstream receives /model//invoke and the body's model field is the caller's value (NOT route.model) — proves both the path rewrite and the body preservation. - bedrock_route_streaming_rewrites_model_in_path: same contract for invoke-with-response-stream. - bedrock_route_rejects_non_bedrock_path: defense-in-depth coverage of the Internal-error path when a Bedrock route receives a path that doesn't match Bedrock shape. Verification (in dev container): - cargo test -p openshell-router --lib: 53/53 (incl. 9 new) - cargo fmt --check: clean - cargo clippy -p openshell-core -p openshell-router -p openshell-server --all-targets -- -D warnings: clean Co-Authored-By: Claude Opus 4.7 Signed-off-by: st-gr <38470677+st-gr@users.noreply.github.com> --- crates/openshell-router/src/backend.rs | 380 ++++++++++++++++++++++++- 1 file changed, 374 insertions(+), 6 deletions(-) diff --git a/crates/openshell-router/src/backend.rs b/crates/openshell-router/src/backend.rs index a0e9ba672..2bae7f869 100644 --- a/crates/openshell-router/src/backend.rs +++ b/crates/openshell-router/src/backend.rs @@ -199,6 +199,30 @@ fn prepare_backend_request( body: bytes::Bytes, stream_response: bool, ) -> Result<(reqwest::RequestBuilder, String), RouterError> { + // For AWS Bedrock routes the model id is encoded in the URL path + // (`/model/{modelId}/invoke[-with-response-stream]`), not in the + // JSON body. The caller's path can carry any model id; rewrite it + // to the operator-configured `route.model` so a sandbox cannot + // pick a different upstream model than what `inference set` + // configured. If the path is not a recognized Bedrock shape on a + // Bedrock route, reject the request rather than forwarding + // verbatim. + let rewritten_path: String; + let path = if route_is_bedrock(route) { + match rewrite_bedrock_path(route, path) { + Some(p) => { + rewritten_path = p; + rewritten_path.as_str() + } + None => { + return Err(RouterError::Internal(format!( + "AWS Bedrock route received non-Bedrock path '{path}'; expected /model//invoke[-with-response-stream]" + ))); + } + } + } else { + path + }; let url = build_provider_url(route, &route.model, path, stream_response); let headers = sanitize_request_headers(route, headers); @@ -259,6 +283,14 @@ fn prepare_backend_request( // in the body; strip it so Vertex AI does not reject the // request with "Extra inputs are not permitted". obj.remove("model"); + } else if route_is_bedrock(route) { + // AWS Bedrock InvokeModel encodes the model in the URL + // path; the request body is the raw provider-specific + // payload (e.g. an Anthropic Messages body for Claude + // models, a Mistral payload for Mistral models). The + // body must not be mutated — injecting a "model" field + // here would either be silently ignored or rejected as + // an unexpected key by the upstream / bridge. } else { obj.insert( "model".to_string(), @@ -782,6 +814,66 @@ fn build_backend_url(endpoint: &str, path: &str) -> String { format!("{base}{path}") } +/// Check whether a route targets an AWS Bedrock `InvokeModel` endpoint. +/// +/// Returns true when any of the route's protocols is one of the Bedrock +/// invocation protocols. Used to gate Bedrock-specific request shaping +/// (path-segment rewriting, skipped body-model injection) in +/// [`prepare_backend_request`]. +fn route_is_bedrock(route: &ResolvedRoute) -> bool { + route + .protocols + .iter() + .any(|p| p == "aws_bedrock_invoke" || p == "aws_bedrock_invoke_stream") +} + +/// Parse a Bedrock invocation path into its `(model_id, action_suffix)` +/// components. +/// +/// Recognized shapes (caller's path on the way into the router): +/// - `/model//invoke` → action `/invoke` +/// - `/model//invoke-with-response-stream` → action +/// `/invoke-with-response-stream` +/// +/// `` must be non-empty and contain no `/`. A trailing query +/// string is stripped before matching. Returns `None` when the path +/// does not match either shape — the caller treats that as a malformed +/// request and rejects rather than forwarding verbatim. +fn parse_bedrock_invocation_path(path: &str) -> Option<(&str, &'static str)> { + let path_only = path.split('?').next().unwrap_or(path); + let rest = path_only.strip_prefix("/model/")?; + let slash_at = rest.find('/')?; + if slash_at == 0 { + return None; + } + let model_id = &rest[..slash_at]; + let suffix = &rest[slash_at..]; + let action: &'static str = match suffix { + "/invoke" => "/invoke", + "/invoke-with-response-stream" => "/invoke-with-response-stream", + _ => return None, + }; + Some((model_id, action)) +} + +/// Rewrite a Bedrock invocation path so the model segment is the +/// operator-configured `route.model` rather than whatever the caller +/// supplied. Returns the rewritten path on success, or `None` when the +/// inbound path is not a recognized Bedrock invocation shape. +/// +/// Why rewrite rather than reject: the inbound L7 pattern detector +/// already accepts only `/model/{x}/invoke[-with-response-stream]` +/// shapes for Bedrock routes, so a caller-supplied model segment that +/// differs from the operator-configured one is the only case this +/// function changes — and changing it (vs. rejecting) lets sandbox +/// code that hardcodes a different model continue to work, while still +/// guaranteeing the operator's chosen model is what reaches the +/// upstream. +fn rewrite_bedrock_path(route: &ResolvedRoute, path: &str) -> Option { + let (_caller_model, action) = parse_bedrock_invocation_path(path)?; + Some(format!("/model/{}{}", route.model, action)) +} + /// Check whether a route targets a Vertex AI Anthropic rawPredict endpoint. /// /// The predicate is purely structural — it tests `model_in_path`, @@ -807,10 +899,13 @@ fn is_vertex_anthropic_rawpredict_route(route: &ResolvedRoute) -> bool { mod tests { use super::{ ValidationFailure, ValidationFailureKind, build_backend_url, build_provider_url, - verify_backend_endpoint, + parse_bedrock_invocation_path, prepare_backend_request, rewrite_bedrock_path, + route_is_bedrock, verify_backend_endpoint, }; + use crate::RouterError; use crate::config::{DEFAULT_ROUTE_TIMEOUT, ResolvedRoute}; use openshell_core::inference::AuthHeader; + use std::time::Duration; use wiremock::matchers::{body_partial_json, header, method, path}; use wiremock::{Mock, MockServer, ResponseTemplate}; @@ -1677,7 +1772,7 @@ mod tests { ); let headers = vec![("content-type".to_string(), "application/json".to_string())]; - let (builder, _url) = super::prepare_backend_request( + let (builder, _url) = prepare_backend_request( &client, &route, "POST", @@ -1748,7 +1843,7 @@ mod tests { ); let headers = vec![("content-type".to_string(), "application/json".to_string())]; - let (builder, _url) = super::prepare_backend_request( + let (builder, _url) = prepare_backend_request( &client, &route, "POST", @@ -1870,7 +1965,7 @@ mod tests { ); let headers = vec![("content-type".to_string(), "application/json".to_string())]; - let (builder, _url) = super::prepare_backend_request( + let (builder, _url) = prepare_backend_request( &client, &route, "POST", @@ -1932,7 +2027,7 @@ mod tests { ); let headers = vec![("content-type".to_string(), "application/json".to_string())]; - let (builder, _url) = super::prepare_backend_request( + let (builder, _url) = prepare_backend_request( &client, &route, "POST", @@ -1996,7 +2091,7 @@ mod tests { ); let headers = vec![("content-type".to_string(), "application/json".to_string())]; - let (builder, _url) = super::prepare_backend_request( + let (builder, _url) = prepare_backend_request( &client, &route, "POST", @@ -2029,4 +2124,277 @@ mod tests { "Vertex Gemini route must still rewrite the model field, got: {received_body}" ); } + + // ============================================================ + // AWS Bedrock route shaping (path rewriting + body preservation) + // ============================================================ + + /// `parse_bedrock_invocation_path` rejects malformed paths. + #[test] + fn parse_bedrock_invocation_path_rejects_malformed() { + // Empty model id: `/model//invoke` + assert!(parse_bedrock_invocation_path("/model//invoke").is_none()); + // Multi-segment model id: `/model/a/b/invoke` + assert!(parse_bedrock_invocation_path("/model/a/b/invoke").is_none()); + // Unknown action: `/model/foo/converse` + assert!(parse_bedrock_invocation_path("/model/foo/converse").is_none()); + // Wrong prefix: `/v1/messages` + assert!(parse_bedrock_invocation_path("/v1/messages").is_none()); + // Missing slash before action + assert!(parse_bedrock_invocation_path("/model/foo").is_none()); + } + + #[test] + fn parse_bedrock_invocation_path_accepts_invoke() { + let parsed = parse_bedrock_invocation_path( + "/model/anthropic.claude-3-5-sonnet-20241022-v2:0/invoke", + ); + assert_eq!( + parsed, + Some(("anthropic.claude-3-5-sonnet-20241022-v2:0", "/invoke")) + ); + } + + #[test] + fn parse_bedrock_invocation_path_accepts_invoke_with_response_stream() { + let parsed = parse_bedrock_invocation_path( + "/model/anthropic.claude-opus-4-7/invoke-with-response-stream", + ); + assert_eq!( + parsed, + Some(("anthropic.claude-opus-4-7", "/invoke-with-response-stream")) + ); + } + + #[test] + fn parse_bedrock_invocation_path_strips_query_string() { + let parsed = + parse_bedrock_invocation_path("/model/anthropic.claude-opus-4-7/invoke?trace=1"); + assert_eq!(parsed, Some(("anthropic.claude-opus-4-7", "/invoke"))); + } + + /// `route_is_bedrock` matches both Bedrock protocol variants. + #[test] + fn route_is_bedrock_matches_invoke_protocols() { + let invoke_only = test_route( + "https://example.com", + &["aws_bedrock_invoke"], + AuthHeader::None, + ); + assert!(route_is_bedrock(&invoke_only)); + + let stream_only = test_route( + "https://example.com", + &["aws_bedrock_invoke_stream"], + AuthHeader::None, + ); + assert!(route_is_bedrock(&stream_only)); + + let both = test_route( + "https://example.com", + &["aws_bedrock_invoke", "aws_bedrock_invoke_stream"], + AuthHeader::None, + ); + assert!(route_is_bedrock(&both)); + + let openai = test_route( + "https://example.com", + &["openai_chat_completions"], + AuthHeader::Bearer, + ); + assert!(!route_is_bedrock(&openai)); + } + + /// `rewrite_bedrock_path` swaps caller's model segment for the + /// route-configured model on both invoke variants. + #[test] + fn rewrite_bedrock_path_substitutes_operator_model() { + let mut route = test_route( + "https://bedrock-bridge.example", + &["aws_bedrock_invoke", "aws_bedrock_invoke_stream"], + AuthHeader::None, + ); + route.model = "anthropic.claude-opus-4-7".to_string(); + + let rewritten = rewrite_bedrock_path(&route, "/model/some-other-model/invoke"); + assert_eq!( + rewritten, + Some("/model/anthropic.claude-opus-4-7/invoke".to_string()) + ); + + let rewritten_stream = rewrite_bedrock_path(&route, "/model/x/invoke-with-response-stream"); + assert_eq!( + rewritten_stream, + Some("/model/anthropic.claude-opus-4-7/invoke-with-response-stream".to_string()) + ); + } + + #[test] + fn rewrite_bedrock_path_returns_none_for_non_bedrock_path() { + let route = test_route( + "https://bedrock-bridge.example", + &["aws_bedrock_invoke"], + AuthHeader::None, + ); + assert_eq!(rewrite_bedrock_path(&route, "/v1/messages"), None); + assert_eq!(rewrite_bedrock_path(&route, "/model//invoke"), None); + assert_eq!(rewrite_bedrock_path(&route, "/model/a/b/invoke"), None); + } + + /// End-to-end: an inbound Bedrock request that names a different + /// model in the path arrives at the upstream/bridge with the + /// operator's model, and the body is unchanged (no `"model"` + /// injection). + #[tokio::test] + async fn bedrock_route_rewrites_model_in_path_and_preserves_body() { + let mock_server = MockServer::start().await; + let mut route = test_route( + &mock_server.uri(), + &["aws_bedrock_invoke", "aws_bedrock_invoke_stream"], + AuthHeader::None, + ); + route.model = "anthropic.claude-opus-4-7".to_string(); + + // The mock asserts the upstream sees the operator's model in + // the path, NOT the caller's model. + Mock::given(method("POST")) + .and(path("/model/anthropic.claude-opus-4-7/invoke")) + // Caller body has a "model" key; we expect it to pass + // through unchanged. The mock uses body_partial_json so + // additional fields are OK; the assertion below pins the + // body more tightly. + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({"ok": true}))) + .mount(&mock_server) + .await; + + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(5)) + .build() + .expect("client"); + + // Caller-supplied body — we deliberately include a "model" + // field naming a DIFFERENT model than the operator's, to + // verify the router does not inject route.model on top of + // it. The body should pass through verbatim because Bedrock + // encodes the model in the path. + let caller_body = serde_json::json!({ + "model": "caller-supplied-model-name", + "messages": [{"role": "user", "content": "hi"}], + }); + + let (builder, url) = prepare_backend_request( + &client, + &route, + "POST", + "/model/some-other-model/invoke", + &[], + bytes::Bytes::from(caller_body.to_string()), + false, + ) + .expect("prepare should succeed"); + + // URL should target the operator's model, not the caller's. + assert!( + url.ends_with("/model/anthropic.claude-opus-4-7/invoke"), + "URL must use operator model, got: {url}" + ); + + let resp = builder.send().await.expect("send"); + assert_eq!(resp.status(), 200); + + // Inspect what wiremock actually received. + let received = mock_server.received_requests().await.expect("requests"); + assert_eq!(received.len(), 1); + let req = &received[0]; + let received_body: serde_json::Value = + serde_json::from_slice(&req.body).expect("json body"); + // Caller's model name should pass through (NOT replaced by + // route.model). This proves the body is untouched. + assert_eq!( + received_body.get("model").and_then(|v| v.as_str()), + Some("caller-supplied-model-name"), + "Bedrock route must NOT rewrite body model, got: {received_body}" + ); + assert!( + received_body.get("messages").is_some(), + "messages field should pass through unchanged" + ); + } + + /// Streaming variant: the same path-rewrite + body-preservation + /// contract applies to invoke-with-response-stream. + #[tokio::test] + async fn bedrock_route_streaming_rewrites_model_in_path() { + let mock_server = MockServer::start().await; + let mut route = test_route( + &mock_server.uri(), + &["aws_bedrock_invoke", "aws_bedrock_invoke_stream"], + AuthHeader::None, + ); + route.model = "anthropic.claude-opus-4-7".to_string(); + + Mock::given(method("POST")) + .and(path( + "/model/anthropic.claude-opus-4-7/invoke-with-response-stream", + )) + .respond_with(ResponseTemplate::new(200).set_body_string("event: ok\n\n")) + .mount(&mock_server) + .await; + + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(5)) + .build() + .expect("client"); + + let (builder, url) = prepare_backend_request( + &client, + &route, + "POST", + "/model/another-model/invoke-with-response-stream", + &[], + bytes::Bytes::from(r#"{"messages":[]}"#), + true, + ) + .expect("prepare should succeed"); + + assert!( + url.ends_with("/model/anthropic.claude-opus-4-7/invoke-with-response-stream"), + "Streaming URL must use operator model, got: {url}" + ); + + let resp = builder.send().await.expect("send"); + assert_eq!(resp.status(), 200); + } + + /// Defense-in-depth: a Bedrock route receiving a non-Bedrock path + /// is rejected rather than forwarded. The L7 pattern detector + /// upstream of the router should never produce this combination, + /// but if it ever did, we must not silently forward. + #[test] + fn bedrock_route_rejects_non_bedrock_path() { + let client = reqwest::Client::new(); + let route = test_route( + "https://bedrock-bridge.example", + &["aws_bedrock_invoke"], + AuthHeader::None, + ); + let result = prepare_backend_request( + &client, + &route, + "POST", + "/v1/messages", + &[], + bytes::Bytes::from(r"{}"), + false, + ); + match result { + Err(RouterError::Internal(msg)) => { + assert!( + msg.contains("Bedrock") && msg.contains("/v1/messages"), + "error must name the offending path, got: {msg}" + ); + } + other => panic!("expected RouterError::Internal, got {other:?}"), + } + } } From af7609bed69e016317aaee18fea0654893943c12 Mon Sep 17 00:00:00 2001 From: st-gr <38470677+st-gr@users.noreply.github.com> Date: Mon, 8 Jun 2026 23:46:12 -0700 Subject: [PATCH 8/8] fix(sandbox/l7): declare framing for Bedrock patterns MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the buffered-vs-streaming framing warning from gator-agent's re-check on 4ab587f1: "Bedrock InvokeModel should be buffered while InvokeModelWithResponseStream is streaming. Please add framing/coverage so /model/{id}/invoke cannot be corrupted by the streaming proxy's truncation/error-frame behavior." The InferenceApiPattern struct gained a `framing: ResponseFraming` field upstream after the original Bedrock-patterns commit (#22b78cff) landed; the cherry-pick onto current upstream/main left the two Bedrock entries without the new field. Fixed here: - aws_bedrock_invoke (POST /model/{id}/invoke): framing = ResponseFraming::Buffered InvokeModel returns one JSON object the caller decodes whole. Sending it through the streaming proxy would risk a mid-body size-cap truncation or idle-timeout failure appending an SSE error event onto bytes the caller decodes as one JSON body — the same corruption mode that drove the existing embeddings + model-discovery to Buffered. - aws_bedrock_invoke_stream (POST /model/{id}/invoke-with-response-stream): framing = ResponseFraming::Streaming InvokeModelWithResponseStream returns an AWS event-stream of binary chunks; the caller wants chunks incrementally, so the streaming proxy path is correct. Two new tests in `crates/openshell-sandbox/src/l7/inference.rs` pin down the contract: - aws_bedrock_invoke_is_buffered — detect_inference_pattern returns a Buffered pattern for /model//invoke, with explanatory message naming the corruption mode being prevented. - aws_bedrock_invoke_stream_is_streaming — same shape, asserting Streaming for /model//invoke-with-response-stream. Verification (in dev container): - cargo check -p openshell-sandbox: clean (was failing on missing `framing` field before this commit) - cargo test -p openshell-sandbox --lib l7::inference::tests::aws_bedrock: 7/7 (incl. 2 new framing tests) - cargo fmt --check: clean - cargo clippy --all-targets -- -D warnings: clean Co-Authored-By: Claude Opus 4.7 Signed-off-by: st-gr <38470677+st-gr@users.noreply.github.com> --- crates/openshell-sandbox/src/l7/inference.rs | 48 ++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/crates/openshell-sandbox/src/l7/inference.rs b/crates/openshell-sandbox/src/l7/inference.rs index 60a7a7e64..10b58cc36 100644 --- a/crates/openshell-sandbox/src/l7/inference.rs +++ b/crates/openshell-sandbox/src/l7/inference.rs @@ -116,17 +116,28 @@ pub fn default_patterns() -> Vec { }, // AWS Bedrock InvokeModel + InvokeModelWithResponseStream. The `*` // segment is the Bedrock model id (e.g. `anthropic.claude-opus-4-7`). + // + // Framing differs between the two endpoints. InvokeModel returns ONE + // JSON object the client decodes whole — it must be served buffered + // with an accurate `Content-Length`, otherwise the streaming proxy's + // size-cap or idle-timeout failure mode would append an SSE error + // event to bytes the caller decodes as one JSON object, silently + // corrupting it. InvokeModelWithResponseStream returns an + // AWS event-stream of binary chunks and must go through the + // streaming path so chunks reach the agent incrementally. InferenceApiPattern { method: "POST".to_string(), path_glob: "/model/*/invoke".to_string(), protocol: "aws_bedrock_invoke".to_string(), kind: "messages".to_string(), + framing: ResponseFraming::Buffered, }, InferenceApiPattern { method: "POST".to_string(), path_glob: "/model/*/invoke-with-response-stream".to_string(), protocol: "aws_bedrock_invoke_stream".to_string(), kind: "messages".to_string(), + framing: ResponseFraming::Streaming, }, ] } @@ -639,6 +650,43 @@ mod tests { assert!(detect_inference_pattern("POST", "/model/foo/converse", &patterns).is_none()); } + /// `InvokeModel` returns one JSON object — must be served buffered. + /// Sending it through the streaming proxy would risk truncation or an + /// appended SSE error event corrupting the JSON body the caller decodes. + #[test] + fn aws_bedrock_invoke_is_buffered() { + let patterns = default_patterns(); + let invoke = + detect_inference_pattern("POST", "/model/anthropic.claude-opus-4-7/invoke", &patterns) + .expect("InvokeModel pattern must match"); + assert_eq!(invoke.protocol, "aws_bedrock_invoke"); + assert!( + invoke.is_buffered(), + "InvokeModel must be Buffered (one JSON object, accurate Content-Length); \ + streaming would risk corrupting the response" + ); + } + + /// `InvokeModelWithResponseStream` returns an AWS event-stream of + /// binary chunks — must go through the streaming proxy so chunks + /// reach the agent incrementally. + #[test] + fn aws_bedrock_invoke_stream_is_streaming() { + let patterns = default_patterns(); + let stream = detect_inference_pattern( + "POST", + "/model/anthropic.claude-opus-4-7/invoke-with-response-stream", + &patterns, + ) + .expect("InvokeModelWithResponseStream pattern must match"); + assert_eq!(stream.protocol, "aws_bedrock_invoke_stream"); + assert!( + !stream.is_buffered(), + "InvokeModelWithResponseStream must be Streaming so the AWS \ + event-stream chunks reach the agent incrementally" + ); + } + #[test] fn parse_simple_post_request() { let body = b"{\"hello\":true}";