From 469c01cc46f0e89f6b56069a5adf3dd74c607591 Mon Sep 17 00:00:00 2001 From: waldekmastykarz Date: Sat, 6 Jun 2026 15:40:22 +0200 Subject: [PATCH 1/2] Fixes bug in har responses encoding --- .../Generation/HarGeneratorPlugin.cs | 11 ++++--- DevProxy.Plugins/Inspection/DevToolsPlugin.cs | 32 ++----------------- DevProxy.Plugins/Utils/HttpUtils.cs | 28 ++++++++++++++++ 3 files changed, 37 insertions(+), 34 deletions(-) diff --git a/DevProxy.Plugins/Generation/HarGeneratorPlugin.cs b/DevProxy.Plugins/Generation/HarGeneratorPlugin.cs index f01e608a..217f9a23 100644 --- a/DevProxy.Plugins/Generation/HarGeneratorPlugin.cs +++ b/DevProxy.Plugins/Generation/HarGeneratorPlugin.cs @@ -6,6 +6,7 @@ using DevProxy.Abstractions.Proxy; using DevProxy.Abstractions.Utils; using DevProxy.Plugins.Models; +using DevProxy.Plugins.Utils; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; using System.Diagnostics; @@ -128,11 +129,11 @@ private HarEntry CreateHarEntry(RequestLog log) return new HarCookie { Name = parts[0].Trim(), Value = parts.Length > 1 ? parts[1].Trim() : "" }; })], HeadersSize = request.Headers?.ToString()?.Length ?? 0, - BodySize = request.HasBody ? (request.BodyString?.Length ?? 0) : 0, + BodySize = request.HasBody ? (request.Body?.Length ?? 0) : 0, PostData = request.HasBody ? new HarPostData { MimeType = request.ContentType, - Text = request.BodyString ?? "" + Text = request.Body is not null ? HttpUtils.GetBodyString(request.ContentType, request.Body) : "" } : null }, @@ -152,12 +153,12 @@ private HarEntry CreateHarEntry(RequestLog log) })], Content = new HarContent { - Size = response.HasBody ? (response.BodyString?.Length ?? 0) : 0, + Size = response.HasBody ? (response.Body?.Length ?? 0) : 0, MimeType = response.ContentType ?? "", - Text = Configuration.IncludeResponse && response.HasBody ? response.BodyString : null + Text = Configuration.IncludeResponse && response.HasBody && response.Body is not null ? HttpUtils.GetBodyString(response.ContentType, response.Body) : null }, HeadersSize = response.Headers?.ToString()?.Length ?? 0, - BodySize = response.HasBody ? (response.BodyString?.Length ?? 0) : 0 + BodySize = response.HasBody ? (response.Body?.Length ?? 0) : 0 } : null }; diff --git a/DevProxy.Plugins/Inspection/DevToolsPlugin.cs b/DevProxy.Plugins/Inspection/DevToolsPlugin.cs index 0cf6cd44..29ba4f72 100644 --- a/DevProxy.Plugins/Inspection/DevToolsPlugin.cs +++ b/DevProxy.Plugins/Inspection/DevToolsPlugin.cs @@ -6,6 +6,7 @@ using DevProxy.Abstractions.Plugins; using DevProxy.Abstractions.Utils; using DevProxy.Plugins.Inspection.CDP; +using DevProxy.Plugins.Utils; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; using System.Collections.Concurrent; @@ -119,7 +120,7 @@ public override async Task BeforeRequestAsync(ProxyRequestArgs e, CancellationTo Url = e.Session.HttpClient.Request.Url, Method = e.Session.HttpClient.Request.Method, Headers = headers, - PostData = e.Session.HttpClient.Request.HasBody ? GetBodyString(e.Session.HttpClient.Request.ContentType, e.Session.HttpClient.Request.Body) : null + PostData = e.Session.HttpClient.Request.HasBody ? HttpUtils.GetBodyString(e.Session.HttpClient.Request.ContentType, e.Session.HttpClient.Request.Body) : null }, Timestamp = (double)DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() / 1000, WallTime = DateTimeOffset.UtcNow.ToUnixTimeSeconds(), @@ -173,7 +174,7 @@ public override async Task AfterResponseAsync(ProxyResponseArgs e, CancellationT { if (IsTextResponse(e.Session.HttpClient.Response.ContentType)) { - body.Body = GetBodyString(e.Session.HttpClient.Response.ContentType, e.Session.HttpClient.Response.Body); + body.Body = HttpUtils.GetBodyString(e.Session.HttpClient.Response.ContentType, e.Session.HttpClient.Response.Body); body.Base64Encoded = false; } else @@ -1062,33 +1063,6 @@ private static bool IsTextResponse(string? contentType) return isTextResponse; } - // Decodes an HTTP message body (request or response) to a string. - // If the Content-Type header specifies a charset, that encoding is used. - // Otherwise, the body is decoded as UTF-8. The underlying proxy library - // defaults to ISO-8859-1 per the obsolete RFC 2616, but modern standards - // (RFC 7231, RFC 8259) treat UTF-8 as the default for JSON and most web content. - private static string GetBodyString(string? contentType, byte[] body) - { - if (contentType is not null) - { - try - { - var ct = new System.Net.Mime.ContentType(contentType); - if (!string.IsNullOrEmpty(ct.CharSet)) - { - return Encoding.GetEncoding(ct.CharSet).GetString(body); - } - } - catch - { - // Malformed Content-Type or unsupported charset; fall through - // to UTF-8 default - } - } - - return Encoding.UTF8.GetString(body); - } - protected override void Dispose(bool disposing) { if (disposing) diff --git a/DevProxy.Plugins/Utils/HttpUtils.cs b/DevProxy.Plugins/Utils/HttpUtils.cs index 14b9b37a..58a327b1 100644 --- a/DevProxy.Plugins/Utils/HttpUtils.cs +++ b/DevProxy.Plugins/Utils/HttpUtils.cs @@ -3,12 +3,40 @@ // See the LICENSE file in the project root for more information. using Microsoft.Extensions.Logging; +using System.Text; using Titanium.Web.Proxy.Http; namespace DevProxy.Plugins.Utils; internal sealed class HttpUtils { + // Decodes an HTTP message body (request or response) to a string. + // If the Content-Type header specifies a charset, that encoding is used. + // Otherwise, the body is decoded as UTF-8. The underlying proxy library + // defaults to ISO-8859-1 per the obsolete RFC 2616, but modern standards + // (RFC 7231, RFC 8259) treat UTF-8 as the default for JSON and most web content. + public static string GetBodyString(string? contentType, byte[] body) + { + if (contentType is not null) + { + try + { + var ct = new System.Net.Mime.ContentType(contentType); + if (!string.IsNullOrEmpty(ct.CharSet)) + { + return Encoding.GetEncoding(ct.CharSet).GetString(body); + } + } + catch + { + // Malformed Content-Type or unsupported charset; fall through + // to UTF-8 default + } + } + + return Encoding.UTF8.GetString(body); + } + public static string GetBodyFromStreamingResponse(Response response, ILogger logger) { logger.LogTrace("{Method} called", nameof(GetBodyFromStreamingResponse)); From 09b21854d4e54eb6b36e0fa265c0346c5d8a5dcb Mon Sep 17 00:00:00 2001 From: waldekmastykarz Date: Sat, 6 Jun 2026 15:48:59 +0200 Subject: [PATCH 2/2] Use strict UTF-8 with Latin-1 fallback for body decoding --- DevProxy.Plugins/Utils/HttpUtils.cs | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/DevProxy.Plugins/Utils/HttpUtils.cs b/DevProxy.Plugins/Utils/HttpUtils.cs index 58a327b1..500290ec 100644 --- a/DevProxy.Plugins/Utils/HttpUtils.cs +++ b/DevProxy.Plugins/Utils/HttpUtils.cs @@ -12,9 +12,12 @@ internal sealed class HttpUtils { // Decodes an HTTP message body (request or response) to a string. // If the Content-Type header specifies a charset, that encoding is used. - // Otherwise, the body is decoded as UTF-8. The underlying proxy library - // defaults to ISO-8859-1 per the obsolete RFC 2616, but modern standards - // (RFC 7231, RFC 8259) treat UTF-8 as the default for JSON and most web content. + // Otherwise, tries strict UTF-8 decoding first. If the body contains + // invalid UTF-8 sequences, falls back to Latin-1 (ISO-8859-1) which is a + // lossless 1:1 byte-to-char mapping that preserves raw byte values. + // The underlying proxy library defaults to ISO-8859-1 per the obsolete + // RFC 2616, but modern standards (RFC 7231, RFC 8259) treat UTF-8 as the + // default for JSON and most web content. public static string GetBodyString(string? contentType, byte[] body) { if (contentType is not null) @@ -30,11 +33,18 @@ public static string GetBodyString(string? contentType, byte[] body) catch { // Malformed Content-Type or unsupported charset; fall through - // to UTF-8 default + // to UTF-8/Latin-1 default } } - return Encoding.UTF8.GetString(body); + try + { + return new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true).GetString(body); + } + catch (DecoderFallbackException) + { + return Encoding.Latin1.GetString(body); + } } public static string GetBodyFromStreamingResponse(Response response, ILogger logger)