From efdd3c8b81a42e1769fe70483d4828d5bba7e87e Mon Sep 17 00:00:00 2001 From: aljun Date: Sun, 19 Aug 2018 10:45:56 +0800 Subject: [PATCH 1/5] Add support for Protobuf format response and unit test (#1479) `Gin` now have the `protobufBinding` function to check the request format, but didn't have a protobuf response function like `c.YAML()`. In our company [ByteDance](http://bytedance.com/), the largest internet company using golang in China, we use `gin` to transfer __Protobuf__ instead of __Json__, we have to write some internal library to make some wrappers to achieve that, and the code is not elegant. So we really want such a feature. --- README.md | 17 +++++++++++++++-- context.go | 6 ++++++ context_test.go | 27 +++++++++++++++++++++++++++ render/protobuf.go | 33 +++++++++++++++++++++++++++++++++ render/render.go | 1 + render/render_test.go | 31 +++++++++++++++++++++++++++++++ 6 files changed, 113 insertions(+), 2 deletions(-) create mode 100644 render/protobuf.go diff --git a/README.md b/README.md index 161ea28b..ae183f92 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ Gin is a web framework written in Go (Golang). It features a martini-like API wi - [Bind Query String or Post Data](#bind-query-string-or-post-data) - [Bind HTML checkboxes](#bind-html-checkboxes) - [Multipart/Urlencoded binding](#multiparturlencoded-binding) - - [XML, JSON and YAML rendering](#xml-json-and-yaml-rendering) + - [XML, JSON, YAML and ProtoBuf rendering](#xml-json-yaml-and-protobuf-rendering) - [JSONP rendering](#jsonp) - [Serving static files](#serving-static-files) - [Serving data from reader](#serving-data-from-reader) @@ -871,7 +871,7 @@ Test it with: $ curl -v --form user=user --form password=password http://localhost:8080/login ``` -### XML, JSON and YAML rendering +### XML, JSON, YAML and ProtoBuf rendering ```go func main() { @@ -905,6 +905,19 @@ func main() { c.YAML(http.StatusOK, gin.H{"message": "hey", "status": http.StatusOK}) }) + r.GET("/someProtoBuf", func(c *gin.Context) { + reps := []int64{int64(1), int64(2)} + label := "test" + // The specific definition of protobuf is written in the testdata/protoexample file. + data := &protoexample.Test{ + Label: &label, + Reps: reps, + } + // Note that data becomes binary data in the response + // Will output protoexample.Test protobuf serialized data + c.ProtoBuf(http.StatusOK, data) + }) + // Listen and serve on 0.0.0.0:8080 r.Run(":8080") } diff --git a/context.go b/context.go index bbdb7e4f..5cd52837 100644 --- a/context.go +++ b/context.go @@ -20,6 +20,7 @@ import ( "github.com/gin-contrib/sse" "github.com/gin-gonic/gin/binding" "github.com/gin-gonic/gin/render" + "github.com/golang/protobuf/proto" ) // Content-Type MIME of the most common data formats. @@ -845,6 +846,11 @@ func (c *Context) Stream(step func(w io.Writer) bool) { } } +// ProtoBuf serializes the given struct as ProtoBuf into the response body. +func (c *Context) ProtoBuf(code int, obj proto.Message) { + c.Render(code, render.ProtoBuf{Data: obj}) +} + /************************************/ /******** CONTENT NEGOTIATION *******/ /************************************/ diff --git a/context_test.go b/context_test.go index 13fb9099..675c2a49 100644 --- a/context_test.go +++ b/context_test.go @@ -20,8 +20,11 @@ import ( "github.com/gin-contrib/sse" "github.com/gin-gonic/gin/binding" + "github.com/golang/protobuf/proto" "github.com/stretchr/testify/assert" "golang.org/x/net/context" + + testdata "github.com/gin-gonic/gin/testdata/protoexample" ) var _ context.Context = &Context{} @@ -954,6 +957,30 @@ func TestContextRenderYAML(t *testing.T) { assert.Equal(t, "application/x-yaml; charset=utf-8", w.HeaderMap.Get("Content-Type")) } +// TestContextRenderProtoBuf tests that the response is serialized as ProtoBuf +// and Content-Type is set to application/x-protobuf +// and we just use the example protobuf to check if the response is correct +func TestContextRenderProtoBuf(t *testing.T) { + w := httptest.NewRecorder() + c, _ := CreateTestContext(w) + + reps := []int64{int64(1), int64(2)} + label := "test" + data := &testdata.Test{ + Label: &label, + Reps: reps, + } + + c.ProtoBuf(http.StatusCreated, data) + + protoData, err := proto.Marshal(data) + assert.NoError(t, err) + + assert.Equal(t, http.StatusCreated, w.Code) + assert.Equal(t, string(protoData[:]), w.Body.String()) + assert.Equal(t, "application/x-protobuf", w.HeaderMap.Get("Content-Type")) +} + func TestContextHeaders(t *testing.T) { c, _ := CreateTestContext(httptest.NewRecorder()) c.Header("Content-Type", "text/plain") diff --git a/render/protobuf.go b/render/protobuf.go new file mode 100644 index 00000000..fe826884 --- /dev/null +++ b/render/protobuf.go @@ -0,0 +1,33 @@ +// Copyright 2018 Gin Core Team. All rights reserved. +// Use of this source code is governed by a MIT style +// license that can be found in the LICENSE file. + +package render + +import ( + "net/http" + + "github.com/golang/protobuf/proto" +) + +type ProtoBuf struct { + Data proto.Message +} + +var protobufContentType = []string{"application/x-protobuf"} + +func (r ProtoBuf) Render(w http.ResponseWriter) error { + r.WriteContentType(w) + + bytes, err := proto.Marshal(r.Data) + if err != nil { + return err + } + + w.Write(bytes) + return nil +} + +func (r ProtoBuf) WriteContentType(w http.ResponseWriter) { + writeContentType(w, protobufContentType) +} diff --git a/render/render.go b/render/render.go index 4ff1c7b6..df0d1d7c 100755 --- a/render/render.go +++ b/render/render.go @@ -27,6 +27,7 @@ var ( _ Render = MsgPack{} _ Render = Reader{} _ Render = AsciiJSON{} + _ Render = ProtoBuf{} ) func writeContentType(w http.ResponseWriter, value []string) { diff --git a/render/render_test.go b/render/render_test.go index 47abf262..905b76a1 100755 --- a/render/render_test.go +++ b/render/render_test.go @@ -15,6 +15,8 @@ import ( "strings" "testing" + testdata "github.com/gin-gonic/gin/testdata/protoexample" + "github.com/golang/protobuf/proto" "github.com/stretchr/testify/assert" "github.com/ugorji/go/codec" ) @@ -265,6 +267,35 @@ func TestRenderYAMLFail(t *testing.T) { assert.Error(t, err) } +// test Protobuf rendering +func TestRenderProtoBuf(t *testing.T) { + w := httptest.NewRecorder() + reps := []int64{int64(1), int64(2)} + label := "test" + data := &testdata.Test{ + Label: &label, + Reps: reps, + } + + (ProtoBuf{data}).WriteContentType(w) + protoData, err := proto.Marshal(data) + assert.NoError(t, err) + assert.Equal(t, "application/x-protobuf", w.Header().Get("Content-Type")) + + err = (ProtoBuf{data}).Render(w) + + assert.NoError(t, err) + assert.Equal(t, string(protoData[:]), w.Body.String()) + assert.Equal(t, "application/x-protobuf", w.Header().Get("Content-Type")) +} + +func TestRenderProtoBufFail(t *testing.T) { + w := httptest.NewRecorder() + data := &testdata.Test{} + err := (ProtoBuf{data}).Render(w) + assert.Error(t, err) +} + func TestRenderXML(t *testing.T) { w := httptest.NewRecorder() data := xmlmap{ From 6073a79ee08657ea9784c26b8521a8f8ef9e1644 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=94=B0=E6=AC=A7?= Date: Sun, 19 Aug 2018 17:39:58 +0800 Subject: [PATCH 2/5] not use protobuf on context but use it on render (#1496) --- context.go | 11 +++++------ render/json.go | 0 render/protobuf.go | 4 ++-- render/render.go | 0 render/render_test.go | 3 ++- 5 files changed, 9 insertions(+), 9 deletions(-) mode change 100755 => 100644 render/json.go mode change 100755 => 100644 render/render.go mode change 100755 => 100644 render/render_test.go diff --git a/context.go b/context.go index 5cd52837..6d80284e 100644 --- a/context.go +++ b/context.go @@ -20,7 +20,6 @@ import ( "github.com/gin-contrib/sse" "github.com/gin-gonic/gin/binding" "github.com/gin-gonic/gin/render" - "github.com/golang/protobuf/proto" ) // Content-Type MIME of the most common data formats. @@ -784,6 +783,11 @@ func (c *Context) YAML(code int, obj interface{}) { c.Render(code, render.YAML{Data: obj}) } +// ProtoBuf serializes the given struct as ProtoBuf into the response body. +func (c *Context) ProtoBuf(code int, obj interface{}) { + c.Render(code, render.ProtoBuf{Data: obj}) +} + // String writes the given string into the response body. func (c *Context) String(code int, format string, values ...interface{}) { c.Render(code, render.String{Format: format, Data: values}) @@ -846,11 +850,6 @@ func (c *Context) Stream(step func(w io.Writer) bool) { } } -// ProtoBuf serializes the given struct as ProtoBuf into the response body. -func (c *Context) ProtoBuf(code int, obj proto.Message) { - c.Render(code, render.ProtoBuf{Data: obj}) -} - /************************************/ /******** CONTENT NEGOTIATION *******/ /************************************/ diff --git a/render/json.go b/render/json.go old mode 100755 new mode 100644 diff --git a/render/protobuf.go b/render/protobuf.go index fe826884..34f1e9b5 100644 --- a/render/protobuf.go +++ b/render/protobuf.go @@ -11,7 +11,7 @@ import ( ) type ProtoBuf struct { - Data proto.Message + Data interface{} } var protobufContentType = []string{"application/x-protobuf"} @@ -19,7 +19,7 @@ var protobufContentType = []string{"application/x-protobuf"} func (r ProtoBuf) Render(w http.ResponseWriter) error { r.WriteContentType(w) - bytes, err := proto.Marshal(r.Data) + bytes, err := proto.Marshal(r.Data.(proto.Message)) if err != nil { return err } diff --git a/render/render.go b/render/render.go old mode 100755 new mode 100644 diff --git a/render/render_test.go b/render/render_test.go old mode 100755 new mode 100644 index 905b76a1..cd20d01d --- a/render/render_test.go +++ b/render/render_test.go @@ -15,10 +15,11 @@ import ( "strings" "testing" - testdata "github.com/gin-gonic/gin/testdata/protoexample" "github.com/golang/protobuf/proto" "github.com/stretchr/testify/assert" "github.com/ugorji/go/codec" + + testdata "github.com/gin-gonic/gin/testdata/protoexample" ) // TODO unit tests From 32b58e0fd2b101cc0bedc925ac71f1bbca620869 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=94=B0=E6=AC=A7?= Date: Sun, 19 Aug 2018 22:14:02 +0800 Subject: [PATCH 3/5] render: update msgpack usage (#1498) please see msgpack usage: https://github.com/ugorji/go/tree/master/codec#usage --- render/msgpack.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/render/msgpack.go b/render/msgpack.go index e6c13e58..b6b8aa0f 100644 --- a/render/msgpack.go +++ b/render/msgpack.go @@ -26,6 +26,6 @@ func (r MsgPack) Render(w http.ResponseWriter) error { func WriteMsgPack(w http.ResponseWriter, obj interface{}) error { writeContentType(w, msgpackContentType) - var h codec.Handle = new(codec.MsgpackHandle) - return codec.NewEncoder(w, h).Encode(obj) + var mh codec.MsgpackHandle + return codec.NewEncoder(w, &mh).Encode(obj) } From b7bb9baa642d2b9d5918a5bcd3829dae64c65257 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=94=B0=E6=AC=A7?= Date: Sun, 19 Aug 2018 22:52:43 +0800 Subject: [PATCH 4/5] chore: add missing copyright and update if/else (#1497) --- render/reader.go | 4 ++++ render/text.go | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/render/reader.go b/render/reader.go index be2132c8..7a06cce4 100644 --- a/render/reader.go +++ b/render/reader.go @@ -1,3 +1,7 @@ +// Copyright 2018 Gin Core Team. All rights reserved. +// Use of this source code is governed by a MIT style +// license that can be found in the LICENSE file. + package render import ( diff --git a/render/text.go b/render/text.go index 74cd26be..7a6acc44 100644 --- a/render/text.go +++ b/render/text.go @@ -30,7 +30,7 @@ func WriteString(w http.ResponseWriter, format string, data []interface{}) { writeContentType(w, plainContentType) if len(data) > 0 { fmt.Fprintf(w, format, data...) - } else { - io.WriteString(w, format) + return } + io.WriteString(w, format) } From c6110f970ca5af365283bd4083b8a105e974e92c Mon Sep 17 00:00:00 2001 From: Filip Figiel Date: Mon, 20 Aug 2018 09:15:31 +0200 Subject: [PATCH 5/5] Add PureJSON renderer (#694) Closes #693 --- README.md | 28 ++++++++++++++++++++++++++++ context_17.go | 17 +++++++++++++++++ context_17_test.go | 27 +++++++++++++++++++++++++++ context_test.go | 5 +++-- json/json.go | 1 + render/json_17.go | 28 ++++++++++++++++++++++++++++ render/render_17_test.go | 26 ++++++++++++++++++++++++++ render/render_test.go | 5 +++-- 8 files changed, 133 insertions(+), 4 deletions(-) create mode 100644 context_17.go create mode 100644 context_17_test.go create mode 100644 render/json_17.go create mode 100644 render/render_17_test.go diff --git a/README.md b/README.md index ae183f92..053750bb 100644 --- a/README.md +++ b/README.md @@ -991,6 +991,34 @@ func main() { } ``` +#### PureJSON + +Normally, JSON replaces special HTML characters with their unicode entities, e.g. `<` becomes `\u003c`. If you want to encode such characters literally, you can use PureJSON instead. +This feature is unavailable in Go 1.6 and lower. + +```go +func main() { + r := gin.Default() + + // Serves unicode entities + r.GET("/json", func(c *gin.Context) { + c.JSON(200, gin.H{ + "html": "Hello, world!", + }) + }) + + // Serves literal characters + r.GET("/purejson", func(c *gin.Context) { + c.PureJSON(200, gin.H{ + "html": "Hello, world!", + }) + }) + + // listen and serve on 0.0.0.0:8080 + r.Run(":8080) +} +``` + ### Serving static files ```go diff --git a/context_17.go b/context_17.go new file mode 100644 index 00000000..8e9f75ad --- /dev/null +++ b/context_17.go @@ -0,0 +1,17 @@ +// Copyright 2018 Gin Core Team. All rights reserved. +// Use of this source code is governed by a MIT style +// license that can be found in the LICENSE file. + +// +build go1.7 + +package gin + +import ( + "github.com/gin-gonic/gin/render" +) + +// PureJSON serializes the given struct as JSON into the response body. +// PureJSON, unlike JSON, does not replace special html characters with their unicode entities. +func (c *Context) PureJSON(code int, obj interface{}) { + c.Render(code, render.PureJSON{Data: obj}) +} diff --git a/context_17_test.go b/context_17_test.go new file mode 100644 index 00000000..d2251904 --- /dev/null +++ b/context_17_test.go @@ -0,0 +1,27 @@ +// Copyright 2018 Gin Core Team. All rights reserved. +// Use of this source code is governed by a MIT style +// license that can be found in the LICENSE file. + +// +build go1.7 + +package gin + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" +) + +// Tests that the response is serialized as JSON +// and Content-Type is set to application/json +// and special HTML characters are preserved +func TestContextRenderPureJSON(t *testing.T) { + w := httptest.NewRecorder() + c, _ := CreateTestContext(w) + c.PureJSON(http.StatusCreated, H{"foo": "bar", "html": ""}) + assert.Equal(t, http.StatusCreated, w.Code) + assert.Equal(t, "{\"foo\":\"bar\",\"html\":\"\"}\n", w.Body.String()) + assert.Equal(t, "application/json; charset=utf-8", w.HeaderMap.Get("Content-Type")) +} diff --git a/context_test.go b/context_test.go index 675c2a49..782f7bed 100644 --- a/context_test.go +++ b/context_test.go @@ -619,14 +619,15 @@ func TestContextRenderPanicIfErr(t *testing.T) { // Tests that the response is serialized as JSON // and Content-Type is set to application/json +// and special HTML characters are escaped func TestContextRenderJSON(t *testing.T) { w := httptest.NewRecorder() c, _ := CreateTestContext(w) - c.JSON(http.StatusCreated, H{"foo": "bar"}) + c.JSON(http.StatusCreated, H{"foo": "bar", "html": ""}) assert.Equal(t, http.StatusCreated, w.Code) - assert.Equal(t, "{\"foo\":\"bar\"}", w.Body.String()) + assert.Equal(t, "{\"foo\":\"bar\",\"html\":\"\\u003cb\\u003e\"}", w.Body.String()) assert.Equal(t, "application/json; charset=utf-8", w.HeaderMap.Get("Content-Type")) } diff --git a/json/json.go b/json/json.go index aa76aa30..4f643c56 100644 --- a/json/json.go +++ b/json/json.go @@ -12,4 +12,5 @@ var ( Marshal = json.Marshal MarshalIndent = json.MarshalIndent NewDecoder = json.NewDecoder + NewEncoder = json.NewEncoder ) diff --git a/render/json_17.go b/render/json_17.go new file mode 100644 index 00000000..df0dc145 --- /dev/null +++ b/render/json_17.go @@ -0,0 +1,28 @@ +// Copyright 2018 Gin Core Team. All rights reserved. +// Use of this source code is governed by a MIT style +// license that can be found in the LICENSE file. + +// +build go1.7 + +package render + +import ( + "net/http" + + "github.com/gin-gonic/gin/json" +) + +type PureJSON struct { + Data interface{} +} + +func (r PureJSON) Render(w http.ResponseWriter) error { + r.WriteContentType(w) + encoder := json.NewEncoder(w) + encoder.SetEscapeHTML(false) + return encoder.Encode(r.Data) +} + +func (r PureJSON) WriteContentType(w http.ResponseWriter) { + writeContentType(w, jsonContentType) +} diff --git a/render/render_17_test.go b/render/render_17_test.go new file mode 100644 index 00000000..68330090 --- /dev/null +++ b/render/render_17_test.go @@ -0,0 +1,26 @@ +// Copyright 2018 Gin Core Team. All rights reserved. +// Use of this source code is governed by a MIT style +// license that can be found in the LICENSE file. + +// +build go1.7 + +package render + +import ( + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestRenderPureJSON(t *testing.T) { + w := httptest.NewRecorder() + data := map[string]interface{}{ + "foo": "bar", + "html": "", + } + err := (PureJSON{data}).Render(w) + assert.NoError(t, err) + assert.Equal(t, "{\"foo\":\"bar\",\"html\":\"\"}\n", w.Body.String()) + assert.Equal(t, "application/json; charset=utf-8", w.Header().Get("Content-Type")) +} diff --git a/render/render_test.go b/render/render_test.go index cd20d01d..fe9228e9 100644 --- a/render/render_test.go +++ b/render/render_test.go @@ -52,7 +52,8 @@ func TestRenderMsgPack(t *testing.T) { func TestRenderJSON(t *testing.T) { w := httptest.NewRecorder() data := map[string]interface{}{ - "foo": "bar", + "foo": "bar", + "html": "", } (JSON{data}).WriteContentType(w) @@ -61,7 +62,7 @@ func TestRenderJSON(t *testing.T) { err := (JSON{data}).Render(w) assert.NoError(t, err) - assert.Equal(t, "{\"foo\":\"bar\"}", w.Body.String()) + assert.Equal(t, "{\"foo\":\"bar\",\"html\":\"\\u003cb\\u003e\"}", w.Body.String()) assert.Equal(t, "application/json; charset=utf-8", w.Header().Get("Content-Type")) }