blackteay/open-im-server
1
0
Fork 0
mirror of https://github.com/openimsdk/open-im-server.git synced 2026-01-07 12:17:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: minio video snapshot

Browse Source
This commit is contained in:
withchao 2023-08-04 19:34:53 +08:00
parent a643c2f89e
commit cf815dcf71
3 changed files with 80 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
config/config.yaml
View File

@ -24,10 +24,10 @@
# Zookeeper username # Zookeeper username
# Zookeeper password # Zookeeper password
zookeeper: zookeeper:
schema: openim schema: openim
address: [ 127.0.0.1:2181 ] address: [ 127.0.0.1:2181 ]
username: username:
password: password:
###################### Mysql ###################### ###################### Mysql ######################
# MySQL configuration # MySQL configuration
@ -42,12 +42,12 @@ mysql:
address: [ 127.0.0.1:13306 ] address: [ 127.0.0.1:13306 ]
username: root username: root
password: openIM123 password: openIM123
database: openIM_v3 database: openIM_v3
maxOpenConn: 1000 maxOpenConn: 1000
maxIdleConn: 100 maxIdleConn: 100
maxLifeTime: 60 maxLifeTime: 60
logLevel: 4 logLevel: 4
slowThreshold: 500 slowThreshold: 500
###################### Mongo ###################### ###################### Mongo ######################
# MongoDB configuration # MongoDB configuration
@ -62,7 +62,7 @@ mongo:
database: openIM_v3 database: openIM_v3
username: root username: root
password: openIM123 password: openIM123
maxPoolSize: 100 maxPoolSize: 100
###################### Redis ###################### ###################### Redis ######################
# Redis configuration # Redis configuration
@ -70,7 +70,7 @@ mongo:
# Username is required only for Redis version 6.0+ # Username is required only for Redis version 6.0+
redis: redis:
address: [ 127.0.0.1:16379 ] address: [ 127.0.0.1:16379 ]
username: username:
password: openIM123 password: openIM123
###################### Kafka ###################### ###################### Kafka ######################
@ -81,13 +81,13 @@ redis:
# It's not recommended to modify this topic name # It's not recommended to modify this topic name
# Consumer group ID, it's not recommended to modify # Consumer group ID, it's not recommended to modify
kafka: kafka:
username: username:
password: password:
addr: [ 127.0.0.1:9092 ] addr: [ 127.0.0.1:9092 ]
latestMsgToRedis: latestMsgToRedis:
topic: "latestMsgToRedis" topic: "latestMsgToRedis"
offlineMsgToMongo: offlineMsgToMongo:
topic: "offlineMsgToMongoMysql" topic: "offlineMsgToMongoMysql"
msgToPush: msgToPush:
topic: "msgToPush" topic: "msgToPush"
consumerGroupID: consumerGroupID:
@ -111,8 +111,8 @@ rpc:
# API service port # API service port
# Default listen IP is 0.0.0.0 # Default listen IP is 0.0.0.0
api: api:
openImApiPort: [ 10002 ] openImApiPort: [ 10002 ]
listenIP: 0.0.0.0 listenIP: 0.0.0.0
###################### Gateway ###################### ###################### Gateway ######################
# Object storage configuration # Object storage configuration
@ -125,21 +125,22 @@ api:
# Configuration for Tencent COS # Configuration for Tencent COS
# Configuration for Aliyun OSS # Configuration for Aliyun OSS
object: object:
enable: "minio" enable: "minio"
apiURL: http://127.0.0.1:10002 apiURL: http://127.0.0.1:10002
minio: minio:
bucket: "openim" bucket: "openim"
endpoint: http://127.0.0.1:10005 endpoint: http://127.0.0.1:10005
accessKeyID: root accessKeyID: root
secretAccessKey: openIM123 secretAccessKey: openIM123
sessionToken: "" sessionToken: ""
signEndpoint: "http://127.0.0.1:10005"
thumbnailApi: "http://127.0.0.1:10003" thumbnailApi: "http://127.0.0.1:10003"
cos: cos:
bucketURL: "https://temp-1252357374.cos.ap-chengdu.myqcloud.com" bucketURL: "https://temp-1252357374.cos.ap-chengdu.myqcloud.com"
secretID: "" secretID: ""
secretKey: "" secretKey: ""
sessionToken: "" sessionToken: ""
oss: oss:
endpoint: "https://oss-cn-chengdu.aliyuncs.com" endpoint: "https://oss-cn-chengdu.aliyuncs.com"
bucket: "demo-9999999" bucket: "demo-9999999"
bucketURL: "https://demo-9999999.oss-cn-chengdu.aliyuncs.com" bucketURL: "https://demo-9999999.oss-cn-chengdu.aliyuncs.com"
@ -151,7 +152,7 @@ object:
# These ports are passed into the program by the script and are not recommended to modify # These ports are passed into the program by the script and are not recommended to modify
# For launching multiple programs, just fill in multiple ports separated by commas # For launching multiple programs, just fill in multiple ports separated by commas
# For example, [10110, 10111] # For example, [10110, 10111]
rpcPort: rpcPort:
openImUserPort: [ 10110 ] openImUserPort: [ 10110 ]
openImFriendPort: [ 10120 ] openImFriendPort: [ 10120 ]
openImMessagePort: [ 10130 ] openImMessagePort: [ 10130 ]
@ -184,12 +185,12 @@ rpcRegisterName:
# Whether to output in json format # Whether to output in json format
# Whether to include stack trace in logs # Whether to include stack trace in logs
log: log:
storageLocation: ../../../../../logs/ storageLocation: ../../../../../logs/
rotationTime: 24 rotationTime: 24
remainRotationCount: 2 remainRotationCount: 2
remainLogLevel: 6 remainLogLevel: 6
isStdout: false isStdout: false
isJson: false isJson: false
withStack: false withStack: false
# Long connection server configuration # Long connection server configuration
@ -199,10 +200,10 @@ log:
# Maximum length of websocket request package # Maximum length of websocket request package
# Websocket connection handshake timeout # Websocket connection handshake timeout
longConnSvr: longConnSvr:
openImWsPort: [ 10001 ] openImWsPort: [ 10001 ]
websocketMaxConnNum: 100000 websocketMaxConnNum: 100000
websocketMaxMsgLen: 4096 websocketMaxMsgLen: 4096
websocketTimeout: 10 websocketTimeout: 10
# Push notification service configuration # Push notification service configuration
# #

1
pkg/common/config/config.go
View File

@ -120,6 +120,7 @@ type configStruct struct {
SecretAccessKey string `yaml:"secretAccessKey"` SecretAccessKey string `yaml:"secretAccessKey"`
SessionToken string `yaml:"sessionToken"` SessionToken string `yaml:"sessionToken"`
ThumbnailApi string `yaml:"thumbnailApi"` ThumbnailApi string `yaml:"thumbnailApi"`
SignEndpoint string `yaml:"signEndpoint"`
} `yaml:"minio"` } `yaml:"minio"`
Cos struct { Cos struct {
BucketURL string `yaml:"bucketURL"` BucketURL string `yaml:"bucketURL"`

51
pkg/common/db/s3/minio/minio.go
View File

@ -19,15 +19,18 @@ import (
"errors" "errors"
"fmt" "fmt"
"github.com/OpenIMSDK/tools/errs" "github.com/OpenIMSDK/tools/errs"
"github.com/OpenIMSDK/tools/log"
"github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/credentials"
"github.com/minio/minio-go/v7/pkg/signer" "github.com/minio/minio-go/v7/pkg/signer"
"net/http" "net/http"
"net/url" "net/url"
"reflect"
"strconv" "strconv"
"strings" "strings"
"sync" "sync"
"time" "time"
"unsafe"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config" "github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3" "github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3"
@ -68,11 +71,26 @@ func NewMinio() (s3.Interface, error) {
bucket: conf.Bucket, bucket: conf.Bucket,
bucketURL: conf.Endpoint + "/" + conf.Bucket + "/", bucketURL: conf.Endpoint + "/" + conf.Bucket + "/",
imageApi: imageApi, imageApi: imageApi,
opts: opts,
core: &minio.Core{Client: client}, core: &minio.Core{Client: client},
lock: &sync.Mutex{}, lock: &sync.Mutex{},
init: false, init: false,
} }
if conf.SignEndpoint == "" {
m.sign = m.core.Client
} else {
su, err := url.Parse(conf.SignEndpoint)
if err != nil {
return nil, err
}
m.opts = &minio.Options{
Creds: credentials.NewStaticV4(conf.AccessKeyID, conf.SecretAccessKey, conf.SessionToken),
Secure: su.Scheme == "https",
}
m.sign, err = minio.New(su.Host, m.opts)
if err != nil {
return nil, err
}
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel() defer cancel()
if err := m.initMinio(ctx); err != nil { if err := m.initMinio(ctx); err != nil {
@ -85,8 +103,10 @@ type Minio struct {
bucket string bucket string
bucketURL string bucketURL string
imageApi string imageApi string
location string
opts *minio.Options opts *minio.Options
core *minio.Core core *minio.Core
sign *minio.Client
lock sync.Locker lock sync.Locker
init bool init bool
} }
@ -100,15 +120,34 @@ func (m *Minio) initMinio(ctx context.Context) error {
if m.init { if m.init {
return nil return nil
} }
exists, err := m.core.Client.BucketExists(ctx, config.Config.Object.Minio.Bucket) conf := config.Config.Object.Minio
exists, err := m.core.Client.BucketExists(ctx, conf.Bucket)
if err != nil { if err != nil {
return fmt.Errorf("check bucket exists error: %w", err) return fmt.Errorf("check bucket exists error: %w", err)
} }
if !exists { if !exists {
if err := m.core.Client.MakeBucket(ctx, config.Config.Object.Minio.Bucket, minio.MakeBucketOptions{}); err != nil { if err := m.core.Client.MakeBucket(ctx, conf.Bucket, minio.MakeBucketOptions{}); err != nil {
return fmt.Errorf("make bucket error: %w", err) return fmt.Errorf("make bucket error: %w", err)
} }
} }
m.location, err = m.core.Client.GetBucketLocation(ctx, conf.Bucket)
if err != nil {
return err
}
func() {
if conf.SignEndpoint == "" {
return
}
defer func() {
if r := recover(); r != nil {
log.ZWarn(context.Background(), "set sign bucket location cache panic", errors.New("failed to get private field value"), "recover", fmt.Sprintf("%+v", r), "development version", "github.com/minio/minio-go/v7 v7.0.61")
}
}()
filed := reflect.ValueOf(m.sign).Elem().FieldByName("bucketLocCache")
zero := reflect.New(reflect.PtrTo(filed.Type()))
*(*unsafe.Pointer)(zero.UnsafePointer()) = unsafe.Pointer(filed.UnsafeAddr())
zero.Elem().Elem().Interface().(interface{ Set(string, string) }).Set(conf.Bucket, m.location)
}()
m.init = true m.init = true
return nil return nil
} }
@ -200,7 +239,7 @@ func (m *Minio) AuthSign(ctx context.Context, uploadID string, name string, expi
return nil, err return nil, err
} }
request.Header.Set("X-Amz-Content-Sha256", unsignedPayload) request.Header.Set("X-Amz-Content-Sha256", unsignedPayload)
request = signer.SignV4Trailer(*request, creds.AccessKeyID, creds.SecretAccessKey, creds.SessionToken, "us-east-1", nil) request = signer.SignV4Trailer(*request, creds.AccessKeyID, creds.SecretAccessKey, creds.SessionToken, m.location, nil)
result.Parts[i] = s3.SignPart{ result.Parts[i] = s3.SignPart{
PartNumber: partNumber, PartNumber: partNumber,
URL: request.URL.String(), URL: request.URL.String(),
@ -215,7 +254,7 @@ func (m *Minio) PresignedPutObject(ctx context.Context, name string, expire time
if err := m.initMinio(ctx); err != nil { if err := m.initMinio(ctx); err != nil {
return "", err return "", err
} }
rawURL, err := m.core.Client.PresignedPutObject(ctx, m.bucket, name, expire) rawURL, err := m.sign.PresignedPutObject(ctx, m.bucket, name, expire)
if err != nil { if err != nil {
return "", err return "", err
} }
@ -330,7 +369,7 @@ func (m *Minio) AccessURL(ctx context.Context, name string, expire time.Duration
} else if expire < time.Second { } else if expire < time.Second {
expire = time.Second expire = time.Second
} }
u, err := m.core.Client.PresignedGetObject(ctx, m.bucket, name, expire, reqParams) u, err := m.sign.PresignedGetObject(ctx, m.bucket, name, expire, reqParams)
if err != nil { if err != nil {
return "", err return "", err
} }