gin/binding/binding.go
Chenyang Yan df54454102 Feature: support operate the underlying incoming data for uploading large file
For file uploading, default behaviour will `ReadForm` and write data to new temporary file if oversize `maxMemory`.

If temporary directory size is too small, it will fail for uploading large file and even consume OS memory since copy to buffer.

Inspired by Flask, Flask `request.stream.read` will read underlying socket data directly if payload found. Refer to: https://werkzeug.palletsprojects.com/en/2.0.x/wsgi/#werkzeug.wsgi.LimitedStream

This method makes assumptions because the request payload exists for operating `POST/PATCH/PUT`.

A simple demo for uploading 5G large file in low memory based on docker:
```
[root@control-master tmp]# docker run -ti --rm --memory 1G --tmpfs /tmp:rw,size=1G,mode=1777 -v $(pwd)/go:/go golang:1.16.5 /bin/bash

root@b672a98e5314:/go/testmodules# cat main.go
package main

import (
	"fmt"
	"github.com/gin-gonic/gin"
	"io"
	"os"
	"path/filepath"
)

func main() {
	engine := gin.Default()
	engine.POST("/streamupload", func(c *gin.Context) {
		if c.GetHeader("Content-Type") != "application/octet-stream" {
			err := fmt.Errorf("required octet-stream")
			c.AbortWithStatusJSON(400, map[string]string{"message": err.Error()})
			return
		}
		info, err := os.Create(filepath.Join("/home", "foo"))
		if err != nil {
			c.AbortWithStatusJSON(400, gin.H{"message": "Create: "+err.Error()})
			return
		}
		defer info.Close()
		_, err = io.Copy(info, c.Request.Body)
		if err != nil {
			c.AbortWithStatusJSON(400, gin.H{"message": "Copy: "+err.Error()})
			return
		}
		c.JSON(200, map[string]string{"message": "ok stream"})
	})

	if err := engine.Run("0.0.0.0:19090"); err != nil {
		panic(err)
	}
}

root@b53810b3e294:/go/testmodules# GOTMPDIR=/opt/ go run main.go

[root@control-master ~]# dd if=/dev/zero of=5G bs=1M count=5170 status=progress
[root@control-master ~]# curl -vvv -H "Content-Type:application/octet-stream" -T 5G -X POST 172.17.0.2:19090/streamupload
```

Signed-off-by: Chenyang Yan <memory.yancy@gmail.com>
2022-01-26 21:26:32 +08:00

120 lines
3.9 KiB
Go

// Copyright 2014 Manu Martinez-Almeida. All rights reserved.
// Use of this source code is governed by a MIT style
// license that can be found in the LICENSE file.
//go:build !nomsgpack
// +build !nomsgpack
package binding
import "net/http"
// Content-Type MIME of the most common data formats.
const (
MIMEJSON = "application/json"
MIMEHTML = "text/html"
MIMEXML = "application/xml"
MIMEXML2 = "text/xml"
MIMEPlain = "text/plain"
MIMEPOSTForm = "application/x-www-form-urlencoded"
MIMEMultipartPOSTForm = "multipart/form-data"
MIMEPROTOBUF = "application/x-protobuf"
MIMEMSGPACK = "application/x-msgpack"
MIMEMSGPACK2 = "application/msgpack"
MIMEYAML = "application/x-yaml"
MIMEOctetStream = "application/octet-stream"
)
// Binding describes the interface which needs to be implemented for binding the
// data present in the request such as JSON request body, query parameters or
// the form POST.
type Binding interface {
Name() string
Bind(*http.Request, interface{}) error
}
// BindingBody adds BindBody method to Binding. BindBody is similar with Bind,
// but it reads the body from supplied bytes instead of req.Body.
type BindingBody interface {
Binding
BindBody([]byte, interface{}) error
}
// BindingUri adds BindUri method to Binding. BindUri is similar with Bind,
// but it reads the Params.
type BindingUri interface {
Name() string
BindUri(map[string][]string, interface{}) error
}
// StructValidator is the minimal interface which needs to be implemented in
// order for it to be used as the validator engine for ensuring the correctness
// of the request. Gin provides a default implementation for this using
// https://github.com/go-playground/validator/tree/v10.6.1.
type StructValidator interface {
// ValidateStruct can receive any kind of type and it should never panic, even if the configuration is not right.
// If the received type is a slice|array, the validation should be performed travel on every element.
// If the received type is not a struct or slice|array, any validation should be skipped and nil must be returned.
// If the received type is a struct or pointer to a struct, the validation should be performed.
// If the struct is not valid or the validation itself fails, a descriptive error should be returned.
// Otherwise nil must be returned.
ValidateStruct(interface{}) error
// Engine returns the underlying validator engine which powers the
// StructValidator implementation.
Engine() interface{}
}
// Validator is the default validator which implements the StructValidator
// interface. It uses https://github.com/go-playground/validator/tree/v10.6.1
// under the hood.
var Validator StructValidator = &defaultValidator{}
// These implement the Binding interface and can be used to bind the data
// present in the request to struct instances.
var (
JSON = jsonBinding{}
XML = xmlBinding{}
Form = formBinding{}
Query = queryBinding{}
FormPost = formPostBinding{}
FormMultipart = formMultipartBinding{}
ProtoBuf = protobufBinding{}
MsgPack = msgpackBinding{}
YAML = yamlBinding{}
Uri = uriBinding{}
Header = headerBinding{}
)
// Default returns the appropriate Binding instance based on the HTTP method
// and the content type.
func Default(method, contentType string) Binding {
if method == http.MethodGet {
return Form
}
switch contentType {
case MIMEJSON:
return JSON
case MIMEXML, MIMEXML2:
return XML
case MIMEPROTOBUF:
return ProtoBuf
case MIMEMSGPACK, MIMEMSGPACK2:
return MsgPack
case MIMEYAML:
return YAML
case MIMEMultipartPOSTForm:
return FormMultipart
default: // case MIMEPOSTForm:
return Form
}
}
func validate(obj interface{}) error {
if Validator == nil {
return nil
}
return Validator.ValidateStruct(obj)
}