From ed773a4d385e0429757a0ebbe8aa8900ce10b44b Mon Sep 17 00:00:00 2001 From: Amit Mishra Date: Sat, 4 Jul 2026 02:47:23 +0530 Subject: [PATCH] fix(render): encode non-BMP characters as UTF-16 surrogate pairs in AsciiJSON MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AsciiJSON used fmt.Appendf with "\u%04x" to escape all non-ASCII runes. For code points above U+FFFF the format produces a 5+ digit sequence (e.g. ὠ0 for U+1F600), which is syntactically valid JSON but wrong: a JSON \u escape is exactly 4 hex digits, so decoders interpret the 5th digit as literal text and the recovered string is silently corrupted. Per RFC 8259 section 7, code points above U+FFFF must be encoded as a UTF-16 surrogate pair (\uD800-\uDBFF followed by \uDC00-\uDFFF). Use unicode/utf16.EncodeRune to compute the pair and emit two \uXXXX escapes. Add TestRenderAsciiJSONNonBMP to verify that emoji and other non-BMP characters survive an AsciiJSON -> json.Unmarshal round-trip unchanged. Fixes #4688 Co-Authored-By: Claude Sonnet 4.6 --- render/json.go | 14 ++++++++++++-- render/render_test.go | 31 +++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/render/json.go b/render/json.go index 2f98676c..5f132a8a 100644 --- a/render/json.go +++ b/render/json.go @@ -10,6 +10,7 @@ import ( "html/template" "net/http" "unicode" + "unicode/utf16" "github.com/gin-gonic/gin/codec/json" "github.com/gin-gonic/gin/internal/bytesconv" @@ -160,11 +161,20 @@ func (r AsciiJSON) Render(w http.ResponseWriter) error { } var buffer bytes.Buffer - escapeBuf := make([]byte, 0, 6) // Preallocate 6 bytes for Unicode escape sequences + escapeBuf := make([]byte, 0, 13) // Preallocate for worst case: two \uXXXX surrogate pair escapes for _, r := range bytesconv.BytesToString(ret) { if r > unicode.MaxASCII { - escapeBuf = fmt.Appendf(escapeBuf[:0], "\\u%04x", r) // Reuse escapeBuf + if r > 0xFFFF { + // Non-BMP character (above U+FFFF): encode as a UTF-16 surrogate pair. + // A JSON \u escape is exactly 4 hex digits, so code points requiring more + // than 4 digits must be split into a high/low surrogate pair per RFC 8259 §7. + high, low := utf16.EncodeRune(r) + escapeBuf = fmt.Appendf(escapeBuf[:0], "\\u%04x\\u%04x", high, low) + } else { + // BMP character (U+0080–U+FFFF): a single \uXXXX escape suffices. + escapeBuf = fmt.Appendf(escapeBuf[:0], "\\u%04x", r) + } buffer.Write(escapeBuf) } else { buffer.WriteByte(byte(r)) diff --git a/render/render_test.go b/render/render_test.go index f63878b9..c35ed757 100644 --- a/render/render_test.go +++ b/render/render_test.go @@ -5,6 +5,7 @@ package render import ( + "encoding/json" "encoding/xml" "errors" "html/template" @@ -261,6 +262,36 @@ func TestRenderAsciiJSON(t *testing.T) { assert.Equal(t, "3.1415926", w2.Body.String()) } +func TestRenderAsciiJSONNonBMP(t *testing.T) { + // Non-BMP characters (code points above U+FFFF) must be encoded as UTF-16 + // surrogate pairs in JSON \u escapes (RFC 8259 §7). Previously, AsciiJSON + // emitted a single \uXXXXX escape with 5+ hex digits, which is syntactically + // valid JSON but silently decodes to the wrong character. + inputs := []string{ + "\U0001F600", // U+1F600 GRINNING FACE + "\U0001F602", // U+1F602 FACE WITH TEARS OF JOY + "\U0001F680", // U+1F680 ROCKET + "\U00020000", // U+20000 CJK Extension B + } + for _, input := range inputs { + w := httptest.NewRecorder() + err := (AsciiJSON{input}).Render(w) + require.NoError(t, err) + + body := w.Body.String() + + // Output must be pure ASCII. + for i, b := range []byte(body) { + assert.Less(t, b, byte(128), "non-ASCII byte at index %d in rendered output for input %q", i, input) + } + + // Round-trip: json.Unmarshal must recover the original string. + var decoded string + require.NoError(t, json.Unmarshal([]byte(body), &decoded), "input %q", input) + assert.Equal(t, input, decoded, "round-trip failed for %q", input) + } +} + func TestRenderAsciiJSONFail(t *testing.T) { w := httptest.NewRecorder() data := make(chan int)