fix(render): encode non-BMP characters as UTF-16 surrogate pairs in AsciiJSON

AsciiJSON used fmt.Appendf with "\u%04x" to escape all non-ASCII runes.
For code points above U+FFFF the format produces a 5+ digit sequence (e.g.
ὠ0 for U+1F600), which is syntactically valid JSON but wrong: a JSON
\u escape is exactly 4 hex digits, so decoders interpret the 5th digit as
literal text and the recovered string is silently corrupted.

Per RFC 8259 section 7, code points above U+FFFF must be encoded as a
UTF-16 surrogate pair (\uD800-\uDBFF followed by \uDC00-\uDFFF). Use
unicode/utf16.EncodeRune to compute the pair and emit two \uXXXX escapes.

Add TestRenderAsciiJSONNonBMP to verify that emoji and other non-BMP
characters survive an AsciiJSON -> json.Unmarshal round-trip unchanged.

Fixes #4688

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Amit Mishra 2026-07-04 02:47:23 +05:30
parent 34dac209ff
commit ed773a4d38
2 changed files with 43 additions and 2 deletions

View File

@ -10,6 +10,7 @@ import (
"html/template"
"net/http"
"unicode"
"unicode/utf16"
"github.com/gin-gonic/gin/codec/json"
"github.com/gin-gonic/gin/internal/bytesconv"
@ -160,11 +161,20 @@ func (r AsciiJSON) Render(w http.ResponseWriter) error {
}
var buffer bytes.Buffer
escapeBuf := make([]byte, 0, 6) // Preallocate 6 bytes for Unicode escape sequences
escapeBuf := make([]byte, 0, 13) // Preallocate for worst case: two \uXXXX surrogate pair escapes
for _, r := range bytesconv.BytesToString(ret) {
if r > unicode.MaxASCII {
escapeBuf = fmt.Appendf(escapeBuf[:0], "\\u%04x", r) // Reuse escapeBuf
if r > 0xFFFF {
// Non-BMP character (above U+FFFF): encode as a UTF-16 surrogate pair.
// A JSON \u escape is exactly 4 hex digits, so code points requiring more
// than 4 digits must be split into a high/low surrogate pair per RFC 8259 §7.
high, low := utf16.EncodeRune(r)
escapeBuf = fmt.Appendf(escapeBuf[:0], "\\u%04x\\u%04x", high, low)
} else {
// BMP character (U+0080U+FFFF): a single \uXXXX escape suffices.
escapeBuf = fmt.Appendf(escapeBuf[:0], "\\u%04x", r)
}
buffer.Write(escapeBuf)
} else {
buffer.WriteByte(byte(r))

View File

@ -5,6 +5,7 @@
package render
import (
"encoding/json"
"encoding/xml"
"errors"
"html/template"
@ -261,6 +262,36 @@ func TestRenderAsciiJSON(t *testing.T) {
assert.Equal(t, "3.1415926", w2.Body.String())
}
func TestRenderAsciiJSONNonBMP(t *testing.T) {
// Non-BMP characters (code points above U+FFFF) must be encoded as UTF-16
// surrogate pairs in JSON \u escapes (RFC 8259 §7). Previously, AsciiJSON
// emitted a single \uXXXXX escape with 5+ hex digits, which is syntactically
// valid JSON but silently decodes to the wrong character.
inputs := []string{
"\U0001F600", // U+1F600 GRINNING FACE
"\U0001F602", // U+1F602 FACE WITH TEARS OF JOY
"\U0001F680", // U+1F680 ROCKET
"\U00020000", // U+20000 CJK Extension B
}
for _, input := range inputs {
w := httptest.NewRecorder()
err := (AsciiJSON{input}).Render(w)
require.NoError(t, err)
body := w.Body.String()
// Output must be pure ASCII.
for i, b := range []byte(body) {
assert.Less(t, b, byte(128), "non-ASCII byte at index %d in rendered output for input %q", i, input)
}
// Round-trip: json.Unmarshal must recover the original string.
var decoded string
require.NoError(t, json.Unmarshal([]byte(body), &decoded), "input %q", input)
assert.Equal(t, input, decoded, "round-trip failed for %q", input)
}
}
func TestRenderAsciiJSONFail(t *testing.T) {
w := httptest.NewRecorder()
data := make(chan int)