blackteay/open-im-server
1
0
Fork 0
mirror of https://github.com/openimsdk/open-im-server.git synced 2026-01-07 12:17:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: minio

Browse Source
This commit is contained in:
withchao 2023-08-07 12:02:54 +08:00
parent cf815dcf71
commit 94be8fc0c4
11 changed files with 93 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
config/config.yaml
View File

@ -135,6 +135,7 @@ object:
sessionToken: "" sessionToken: ""
signEndpoint: "http://127.0.0.1:10005" signEndpoint: "http://127.0.0.1:10005"
thumbnailApi: "http://127.0.0.1:10003" thumbnailApi: "http://127.0.0.1:10003"
thumbnailUseSignEndpoint: false
cos: cos:
bucketURL: "https://temp-1252357374.cos.ap-chengdu.myqcloud.com" bucketURL: "https://temp-1252357374.cos.ap-chengdu.myqcloud.com"
secretID: "" secretID: ""

2
go.mod
View File

@ -37,7 +37,7 @@ require (
require github.com/google/uuid v1.3.0 require github.com/google/uuid v1.3.0
require ( require (
github.com/OpenIMSDK/protocol v0.0.3 github.com/OpenIMSDK/protocol v0.0.6
github.com/OpenIMSDK/tools v0.0.13 github.com/OpenIMSDK/tools v0.0.13
github.com/aliyun/aliyun-oss-go-sdk v2.2.7+incompatible github.com/aliyun/aliyun-oss-go-sdk v2.2.7+incompatible
github.com/go-redis/redis v6.15.9+incompatible github.com/go-redis/redis v6.15.9+incompatible

4
go.sum
View File

@ -17,8 +17,8 @@ cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7Biccwk
firebase.google.com/go v3.13.0+incompatible h1:3TdYC3DDi6aHn20qoRkxwGqNgdjtblwVAyRLQwGn/+4= firebase.google.com/go v3.13.0+incompatible h1:3TdYC3DDi6aHn20qoRkxwGqNgdjtblwVAyRLQwGn/+4=
firebase.google.com/go v3.13.0+incompatible/go.mod h1:xlah6XbEyW6tbfSklcfe5FHJIwjt8toICdV5Wh9ptHs= firebase.google.com/go v3.13.0+incompatible/go.mod h1:xlah6XbEyW6tbfSklcfe5FHJIwjt8toICdV5Wh9ptHs=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/OpenIMSDK/protocol v0.0.3 h1:CFQtmnyW+1dYKVFaVaHcJ6oYuMiMdNfU2gC1xz3K/9I= github.com/OpenIMSDK/protocol v0.0.6 h1:KjaItOEww7vjrhwyxHnVzhw80pnjcNukpskadqW6gnA=
github.com/OpenIMSDK/protocol v0.0.3/go.mod h1:F25dFrwrIx3lkNoiuf6FkCfxuwf8L4Z8UIsdTHP/r0Y= github.com/OpenIMSDK/protocol v0.0.6/go.mod h1:F25dFrwrIx3lkNoiuf6FkCfxuwf8L4Z8UIsdTHP/r0Y=
github.com/OpenIMSDK/tools v0.0.13 h1:rcw4HS8S2DPZR9UOBxD8/ol9UBMzXBypzOVEytDRIMo= github.com/OpenIMSDK/tools v0.0.13 h1:rcw4HS8S2DPZR9UOBxD8/ol9UBMzXBypzOVEytDRIMo=
github.com/OpenIMSDK/tools v0.0.13/go.mod h1:eg+q4A34Qmu73xkY0mt37FHGMCMfC6CtmOnm0kFEGFI= github.com/OpenIMSDK/tools v0.0.13/go.mod h1:eg+q4A34Qmu73xkY0mt37FHGMCMfC6CtmOnm0kFEGFI=
github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM= github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM=

9
internal/api/third.go
View File

@ -81,7 +81,14 @@ func (o *ThirdApi) ObjectRedirect(c *gin.Context) {
operationID = strconv.Itoa(rand.Int()) operationID = strconv.Itoa(rand.Int())
} }
ctx := mcontext.SetOperationID(c, operationID) ctx := mcontext.SetOperationID(c, operationID)
resp, err := o.Client.AccessURL(ctx, &third.AccessURLReq{Name: name}) query := make(map[string]string)
for key, values := range c.Request.URL.Query() {
if len(values) == 0 {
continue
}
query[key] = values[0]
}
resp, err := o.Client.AccessURL(ctx, &third.AccessURLReq{Name: name, Query: query})
if err != nil { if err != nil {
if errs.ErrArgs.Is(err) { if errs.ErrArgs.Is(err) {
c.String(http.StatusBadRequest, err.Error()) c.String(http.StatusBadRequest, err.Error())

19
internal/rpc/third/s3.go
View File

@ -16,6 +16,8 @@ package third
import ( import (
"context" "context"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3"
"strconv"
"time" "time"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3/cont" "github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3/cont"
@ -151,7 +153,22 @@ func (t *thirdServer) CompleteMultipartUpload(ctx context.Context, req *third.Co
} }
func (t *thirdServer) AccessURL(ctx context.Context, req *third.AccessURLReq) (*third.AccessURLResp, error) { func (t *thirdServer) AccessURL(ctx context.Context, req *third.AccessURLReq) (*third.AccessURLResp, error) {
expireTime, rawURL, err := t.s3dataBase.AccessURL(ctx, req.Name, t.defaultExpire) opt := &s3.AccessURLOption{}
if len(req.Query) > 0 {
switch req.Query["type"] {
case "image":
opt.Image.Format = req.Query["format"]
opt.Image.Width, _ = strconv.Atoi(req.Query["width"])
opt.Image.Height, _ = strconv.Atoi(req.Query["height"])
case "video":
opt.Video.Format = req.Query["format"]
opt.Video.Width, _ = strconv.Atoi(req.Query["width"])
opt.Video.Height, _ = strconv.Atoi(req.Query["height"])
millisecond, _ := strconv.Atoi(req.Query["time"])
opt.Video.Time = time.Millisecond * time.Duration(millisecond)
}
}
expireTime, rawURL, err := t.s3dataBase.AccessURL(ctx, req.Name, t.defaultExpire, opt)
if err != nil { if err != nil {
return nil, err return nil, err
} }

15
pkg/common/config/config.go
View File

@ -114,13 +114,14 @@ type configStruct struct {
Enable string `yaml:"enable"` Enable string `yaml:"enable"`
ApiURL string `yaml:"apiURL"` ApiURL string `yaml:"apiURL"`
Minio struct { Minio struct {
Bucket string `yaml:"bucket"` Bucket string `yaml:"bucket"`
Endpoint string `yaml:"endpoint"` Endpoint string `yaml:"endpoint"`
AccessKeyID string `yaml:"accessKeyID"` AccessKeyID string `yaml:"accessKeyID"`
SecretAccessKey string `yaml:"secretAccessKey"` SecretAccessKey string `yaml:"secretAccessKey"`
SessionToken string `yaml:"sessionToken"` SessionToken string `yaml:"sessionToken"`
ThumbnailApi string `yaml:"thumbnailApi"` SignEndpoint string `yaml:"signEndpoint"`
SignEndpoint string `yaml:"signEndpoint"` ThumbnailApi string `yaml:"thumbnailApi"`
ThumbnailUseSignEndpoint bool `yaml:"thumbnailUseSignEndpoint"`
} `yaml:"minio"` } `yaml:"minio"`
Cos struct { Cos struct {
BucketURL string `yaml:"bucketURL"` BucketURL string `yaml:"bucketURL"`

15
pkg/common/db/controller/s3.go
View File

@ -30,7 +30,7 @@ type S3Database interface {
AuthSign(ctx context.Context, uploadID string, partNumbers []int) (*s3.AuthSignResult, error) AuthSign(ctx context.Context, uploadID string, partNumbers []int) (*s3.AuthSignResult, error)
InitiateMultipartUpload(ctx context.Context, hash string, size int64, expire time.Duration, maxParts int) (*cont.InitiateUploadResult, error) InitiateMultipartUpload(ctx context.Context, hash string, size int64, expire time.Duration, maxParts int) (*cont.InitiateUploadResult, error)
CompleteMultipartUpload(ctx context.Context, uploadID string, parts []string) (*cont.UploadResult, error) CompleteMultipartUpload(ctx context.Context, uploadID string, parts []string) (*cont.UploadResult, error)
AccessURL(ctx context.Context, name string, expire time.Duration) (time.Time, string, error) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (time.Time, string, error)
SetObject(ctx context.Context, info *relation.ObjectModel) error SetObject(ctx context.Context, info *relation.ObjectModel) error
} }
@ -70,14 +70,19 @@ func (s *s3Database) SetObject(ctx context.Context, info *relation.ObjectModel)
return s.obj.SetObject(ctx, info) return s.obj.SetObject(ctx, info)
} }
func (s *s3Database) AccessURL(ctx context.Context, name string, expire time.Duration) (time.Time, string, error) { func (s *s3Database) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (time.Time, string, error) {
obj, err := s.obj.Take(ctx, name) obj, err := s.obj.Take(ctx, name)
if err != nil { if err != nil {
return time.Time{}, "", err return time.Time{}, "", err
} }
opt := &s3.AccessURLOption{ if opt == nil {
ContentType: obj.ContentType, opt = &s3.AccessURLOption{}
Filename: filepath.Base(obj.Name), }
if opt.ContentType == "" {
opt.ContentType = obj.ContentType
}
if opt.Filename == "" {
opt.Filename = filepath.Base(obj.Name)
} }
expireTime := time.Now().Add(expire) expireTime := time.Now().Add(expire)
rawURL, err := s.s3.AccessURL(ctx, obj.Key, expire, opt) rawURL, err := s.s3.AccessURL(ctx, obj.Key, expire, opt)

14
pkg/common/db/s3/cos/cos.go
View File

@ -308,19 +308,19 @@ func (c *Cos) AccessURL(ctx context.Context, name string, expire time.Duration,
sec = 0 sec = 0
} }
query.Set("time", strconv.FormatFloat(sec, 'f', 3, 64)) query.Set("time", strconv.FormatFloat(sec, 'f', 3, 64))
switch opt.Video.ImageFormat { switch opt.Video.Format {
case case
videoSnapshotImagePng, videoSnapshotImagePng,
videoSnapshotImageJpg: videoSnapshotImageJpg:
default: default:
opt.Video.ImageFormat = videoSnapshotImageJpg opt.Video.Format = videoSnapshotImageJpg
} }
query.Set("format", opt.Video.ImageFormat) query.Set("format", opt.Video.Format)
opt.ContentType = "image/" + opt.Video.ImageFormat opt.ContentType = "image/" + opt.Video.Format
if opt.Filename == "" { if opt.Filename == "" {
opt.Filename = filepath.Base(name) + "." + opt.Video.ImageFormat opt.Filename = filepath.Base(name) + "." + opt.Video.Format
} else if filepath.Ext(opt.Filename) != "."+opt.Video.ImageFormat { } else if filepath.Ext(opt.Filename) != "."+opt.Video.Format {
opt.Filename += "." + opt.Video.ImageFormat opt.Filename += "." + opt.Video.Format
} }
if opt.Video.Width > 0 { if opt.Video.Width > 0 {
query.Set("width", strconv.Itoa(opt.Video.Width)) query.Set("width", strconv.Itoa(opt.Video.Width))

40
pkg/common/db/s3/minio/minio.go
View File

@ -68,12 +68,13 @@ func NewMinio() (s3.Interface, error) {
imageApi += "image?" imageApi += "image?"
} }
m := &Minio{ m := &Minio{
bucket: conf.Bucket, bucket: conf.Bucket,
bucketURL: conf.Endpoint + "/" + conf.Bucket + "/", bucketURL: conf.Endpoint + "/" + conf.Bucket + "/",
imageApi: imageApi, imageApi: imageApi,
core: &minio.Core{Client: client}, imageUseSignAddr: conf.ThumbnailUseSignEndpoint,
lock: &sync.Mutex{}, core: &minio.Core{Client: client},
init: false, lock: &sync.Mutex{},
init: false,
} }
if conf.SignEndpoint == "" { if conf.SignEndpoint == "" {
m.sign = m.core.Client m.sign = m.core.Client
@ -100,15 +101,16 @@ func NewMinio() (s3.Interface, error) {
} }
type Minio struct { type Minio struct {
bucket string bucket string
bucketURL string bucketURL string
imageApi string imageApi string
location string imageUseSignAddr bool
opts *minio.Options location string
core *minio.Core opts *minio.Options
sign *minio.Client core *minio.Core
lock sync.Locker sign *minio.Client
init bool lock sync.Locker
init bool
} }
func (m *Minio) initMinio(ctx context.Context) error { func (m *Minio) initMinio(ctx context.Context) error {
@ -369,7 +371,13 @@ func (m *Minio) AccessURL(ctx context.Context, name string, expire time.Duration
} else if expire < time.Second { } else if expire < time.Second {
expire = time.Second expire = time.Second
} }
u, err := m.sign.PresignedGetObject(ctx, m.bucket, name, expire, reqParams) var client *minio.Client
if m.imageUseSignAddr {
client = m.sign
} else {
client = m.core.Client
}
u, err := client.PresignedGetObject(ctx, m.bucket, name, expire, reqParams)
if err != nil { if err != nil {
return "", err return "", err
} }

20
pkg/common/db/s3/oss/oss.go
View File

@ -291,9 +291,9 @@ func (o *OSS) AccessURL(ctx context.Context, name string, expire time.Duration,
} }
opt.ContentType = "image/" + format opt.ContentType = "image/" + format
if opt.Filename == "" { if opt.Filename == "" {
opt.Filename = filepath.Base(name) + "." + opt.Video.ImageFormat opt.Filename = filepath.Base(name) + "." + opt.Video.Format
} else if filepath.Ext(opt.Filename) != "."+opt.Video.ImageFormat { } else if filepath.Ext(opt.Filename) != "."+opt.Video.Format {
opt.Filename += "." + opt.Video.ImageFormat opt.Filename += "." + opt.Video.Format
} }
// https://oss-console-img-demo-cn-hangzhou.oss-cn-hangzhou.aliyuncs.com/example.jpg?x-oss-process=image/resize,h_100,m_lfit // https://oss-console-img-demo-cn-hangzhou.oss-cn-hangzhou.aliyuncs.com/example.jpg?x-oss-process=image/resize,h_100,m_lfit
process := "image/resize,m_lfit" process := "image/resize,m_lfit"
@ -313,18 +313,18 @@ func (o *OSS) AccessURL(ctx context.Context, name string, expire time.Duration,
if millisecond < 0 { if millisecond < 0 {
millisecond = 0 millisecond = 0
} }
switch opt.Video.ImageFormat { switch opt.Video.Format {
case videoSnapshotImageJpg, videoSnapshotImagePng: case videoSnapshotImageJpg, videoSnapshotImagePng:
default: default:
opt.Video.ImageFormat = videoSnapshotImageJpg opt.Video.Format = videoSnapshotImageJpg
} }
opt.ContentType = "image/" + opt.Video.ImageFormat opt.ContentType = "image/" + opt.Video.Format
if opt.Filename == "" { if opt.Filename == "" {
opt.Filename = filepath.Base(name) + "." + opt.Video.ImageFormat opt.Filename = filepath.Base(name) + "." + opt.Video.Format
} else if filepath.Ext(opt.Filename) != "."+opt.Video.ImageFormat { } else if filepath.Ext(opt.Filename) != "."+opt.Video.Format {
opt.Filename += "." + opt.Video.ImageFormat opt.Filename += "." + opt.Video.Format
} }
process := "video/snapshot,t_" + strconv.Itoa(millisecond) + ",f_" + opt.Video.ImageFormat process := "video/snapshot,t_" + strconv.Itoa(millisecond) + ",f_" + opt.Video.Format
if opt.Video.Width > 0 { if opt.Video.Width > 0 {
process += ",w_" + strconv.Itoa(opt.Video.Width) process += ",w_" + strconv.Itoa(opt.Video.Width)
} }

8
pkg/common/db/s3/s3.go
View File

@ -123,10 +123,10 @@ type Image struct {
} }
type Video struct { type Video struct {
Width int `json:"width"` Width int `json:"width"`
Height int `json:"height"` Height int `json:"height"`
Time time.Duration `json:"time"` Time time.Duration `json:"time"`
ImageFormat string `json:"format"` Format string `json:"format"`
} }
type AccessURLOption struct { type AccessURLOption struct {