blackteay/open-im-server
1
0
Fork 0
mirror of https://github.com/openimsdk/open-im-server.git synced 2025-12-10 23:07:30 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: minio cache

Browse Source
This commit is contained in:
withchao 2023-11-02 15:14:42 +08:00
parent c9ab3c63f8
commit fd928b746d
5 changed files with 64 additions and 303 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
internal/rpc/third/third.go
View File

@ -67,7 +67,7 @@ func Start(client discoveryregistry.SvcDiscoveryRegistry, server *grpc.Server) e
var o s3.Interface var o s3.Interface
switch config.Config.Object.Enable { switch config.Config.Object.Enable {
case "minio": case "minio":
o, err = minio.NewMinio() o, err = minio.NewMinio(cache.NewMinioCache(rdb))
case "cos": case "cos":
o, err = cos.NewCos() o, err = cos.NewCos()
case "oss": case "oss":

74
pkg/common/db/cache/s3.go vendored
View File

@ -2,12 +2,11 @@ package cache
import ( import (
"context" "context"
"github.com/OpenIMSDK/tools/errs"
"github.com/dtm-labs/rockscache" "github.com/dtm-labs/rockscache"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3" "github.com/openimsdk/open-im-server/v3/pkg/common/db/s3"
relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/redis/go-redis/v9" "github.com/redis/go-redis/v9"
"image" "strconv"
"time" "time"
) )
@ -117,24 +116,23 @@ func (g *s3CacheRedis) GetKey(ctx context.Context, engine string, name string) (
type MinioCache interface { type MinioCache interface {
metaCache metaCache
GetImageObjectKeyInfo(ctx context.Context, key string, fn func(ctx context.Context, key string) (*MinioImageInfo, image.Image, error)) (*MinioImageInfo, image.Image, error) GetImageObjectKeyInfo(ctx context.Context, key string, fn func(ctx context.Context) (*MinioImageInfo, error)) (*MinioImageInfo, error)
GetThumbnailKey(ctx context.Context, key string, format string, width int, height int, minioCache func(ctx context.Context) (string, error)) (string, error) GetThumbnailKey(ctx context.Context, key string, format string, width int, height int, minioCache func(ctx context.Context) (string, error)) (string, error)
//DelS3Key(engine string, keys ...string) S3Cache DelObjectImageInfoKey(keys ...string) MinioCache
DelImageThumbnailKey(key string, format string, width int, height int) MinioCache
} }
func NewMinioCache(rdb redis.UniversalClient, s3 s3.Interface) MinioCache { func NewMinioCache(rdb redis.UniversalClient) MinioCache {
rcClient := rockscache.NewClient(rdb, rockscache.NewDefaultOptions()) rcClient := rockscache.NewClient(rdb, rockscache.NewDefaultOptions())
return &minioCacheRedis{ return &minioCacheRedis{
rcClient: rcClient, rcClient: rcClient,
expireTime: time.Hour * 12, expireTime: time.Hour * 24 * 7,
s3: s3,
metaCache: NewMetaCacheRedis(rcClient), metaCache: NewMetaCacheRedis(rcClient),
} }
} }
type minioCacheRedis struct { type minioCacheRedis struct {
metaCache metaCache
s3 s3.Interface
rcClient *rockscache.Client rcClient *rockscache.Client
expireTime time.Duration expireTime time.Duration
} }
@ -143,52 +141,44 @@ func (g *minioCacheRedis) NewCache() MinioCache {
return &minioCacheRedis{ return &minioCacheRedis{
rcClient: g.rcClient, rcClient: g.rcClient,
expireTime: g.expireTime, expireTime: g.expireTime,
s3: g.s3,
metaCache: NewMetaCacheRedis(g.rcClient, g.metaCache.GetPreDelKeys()...), metaCache: NewMetaCacheRedis(g.rcClient, g.metaCache.GetPreDelKeys()...),
} }
} }
//func (g *minioCacheRedis) DelS3Key(engine string, keys ...string) MinioCache { func (g *minioCacheRedis) DelObjectImageInfoKey(keys ...string) MinioCache {
// s3cache := g.NewCache() s3cache := g.NewCache()
// ks := make([]string, 0, len(keys)) ks := make([]string, 0, len(keys))
// for _, key := range keys { for _, key := range keys {
// ks = append(ks, g.getS3Key(engine, key)) ks = append(ks, g.getObjectImageInfoKey(key))
// } }
// s3cache.AddKeys(ks...) s3cache.AddKeys(ks...)
// return s3cache return s3cache
//}
func (g *minioCacheRedis) getMinioImageInfoKey(name string) string {
return "MINIO:IMAGE:" + name
} }
func (g *minioCacheRedis) getMinioImageThumbnailKey(name string) string { func (g *minioCacheRedis) DelImageThumbnailKey(key string, format string, width int, height int) MinioCache {
return "MINIO:THUMBNAIL:" + name s3cache := g.NewCache()
s3cache.AddKeys(g.getMinioImageThumbnailKey(key, format, width, height))
return s3cache
} }
func (g *minioCacheRedis) GetImageObjectKeyInfo(ctx context.Context, key string, fn func(ctx context.Context, key string) (*MinioImageInfo, image.Image, error)) (*MinioImageInfo, image.Image, error) { func (g *minioCacheRedis) getObjectImageInfoKey(key string) string {
var img image.Image return "MINIO:IMAGE:" + key
info, err := getCache(ctx, g.rcClient, g.getMinioImageInfoKey(key), g.expireTime, func(ctx context.Context) (info *MinioImageInfo, err error) { }
info, img, err = fn(ctx, key)
return func (g *minioCacheRedis) getMinioImageThumbnailKey(key string, format string, width int, height int) string {
}) return "MINIO:THUMBNAIL:" + format + ":w" + strconv.Itoa(width) + ":h" + strconv.Itoa(height) + ":" + key
}
func (g *minioCacheRedis) GetImageObjectKeyInfo(ctx context.Context, key string, fn func(ctx context.Context) (*MinioImageInfo, error)) (*MinioImageInfo, error) {
info, err := getCache(ctx, g.rcClient, g.getObjectImageInfoKey(key), g.expireTime, fn)
if err != nil { if err != nil {
return nil, nil, err return nil, err
} }
if !info.IsImg { return info, nil
return nil, nil, errs.ErrData.Wrap("object not image")
}
return info, img, nil
} }
func (g *minioCacheRedis) GetThumbnailKey(ctx context.Context, key string, format string, width int, height int, minioCache func(ctx context.Context, key string, format string, width int, height int) (string, error)) (string, error) { func (g *minioCacheRedis) GetThumbnailKey(ctx context.Context, key string, format string, width int, height int, minioCache func(ctx context.Context) (string, error)) (string, error) {
return getCache(ctx, g.rcClient, g.getMinioImageThumbnailKey(key), g.expireTime, func(ctx context.Context) (string, error) { return getCache(ctx, g.rcClient, g.getMinioImageThumbnailKey(key, format, width, height), g.expireTime, minioCache)
info, img, err := g.GetImageObjectKeyInfo(ctx, key, getInfo)
if err != nil {
return "", err
}
return minioCache(ctx, key, format, width, height, info, img)
})
} }
type MinioImageInfo struct { type MinioImageInfo struct {

118
pkg/common/db/s3/minio/minio.go
View File

@ -53,11 +53,10 @@ const (
maxImageWidth = 1024 maxImageWidth = 1024
maxImageHeight = 1024 maxImageHeight = 1024
maxImageSize = 1024 * 1024 * 50 maxImageSize = 1024 * 1024 * 50
pathInfo = "openim/thumbnail" imageThumbnailPath = "openim/thumbnail"
maxImageInfoSize = 1024
) )
func NewMinio() (s3.Interface, error) { func NewMinio(cache cache.MinioCache) (s3.Interface, error) {
u, err := url.Parse(config.Config.Object.Minio.Endpoint) u, err := url.Parse(config.Config.Object.Minio.Endpoint)
if err != nil { if err != nil {
return nil, err return nil, err
@ -223,6 +222,7 @@ func (m *Minio) CompleteMultipartUpload(ctx context.Context, uploadID string, na
if err != nil { if err != nil {
return nil, err return nil, err
} }
m.delObjectImageInfoKey(ctx, name, upload.Size)
return &s3.CompleteMultipartUploadResult{ return &s3.CompleteMultipartUploadResult{
Location: upload.Location, Location: upload.Location,
Bucket: upload.Bucket, Bucket: upload.Bucket,
@ -385,7 +385,7 @@ func (m *Minio) ListUploadedParts(ctx context.Context, uploadID string, name str
return res, nil return res, nil
} }
func (m *Minio) presignedGetObject(ctx context.Context, name string, expire time.Duration, query url.Values) (string, error) { func (m *Minio) PresignedGetObject(ctx context.Context, name string, expire time.Duration, query url.Values) (string, error) {
if expire <= 0 { if expire <= 0 {
expire = time.Hour * 24 * 365 * 99 // 99 years expire = time.Hour * 24 * 365 * 99 // 99 years
} else if expire < time.Second { } else if expire < time.Second {
@ -423,110 +423,9 @@ func (m *Minio) AccessURL(ctx context.Context, name string, expire time.Duration
} }
} }
if opt.Image == nil || (opt.Image.Width < 0 && opt.Image.Height < 0 && opt.Image.Format == "") || (opt.Image.Width > maxImageWidth || opt.Image.Height > maxImageHeight) { if opt.Image == nil || (opt.Image.Width < 0 && opt.Image.Height < 0 && opt.Image.Format == "") || (opt.Image.Width > maxImageWidth || opt.Image.Height > maxImageHeight) {
return m.presignedGetObject(ctx, name, expire, reqParams) return m.PresignedGetObject(ctx, name, expire, reqParams)
} }
return m.GetImageThumbnail(ctx, name, expire, opt.Image) return m.getImageThumbnailURL(ctx, name, expire, opt.Image)
//fileInfo, err := m.StatObject(ctx, name)
//if err != nil {
// return "", err
//}
//if fileInfo.Size > maxImageSize {
// return "", errors.New("file size too large")
//}
//objectInfoPath := path.Join(pathInfo, fileInfo.ETag, "image.json")
//var (
// img image.Image
// info minioImageInfo
//)
//data, err := m.getObjectData(ctx, objectInfoPath, 1024)
//if err == nil {
// if err := json.Unmarshal(data, &info); err != nil {
// return "", fmt.Errorf("unmarshal minio image info.json error: %w", err)
// }
// if info.NotImage {
// return "", errors.New("not image")
// }
//} else if m.IsNotFound(err) {
// reader, err := m.core.Client.GetObject(ctx, m.bucket, name, minio.GetObjectOptions{})
// if err != nil {
// return "", err
// }
// defer reader.Close()
// imageInfo, format, err := ImageStat(reader)
// if err == nil {
// info.NotImage = false
// info.Format = format
// info.Width, info.Height = ImageWidthHeight(imageInfo)
// img = imageInfo
// } else {
// info.NotImage = true
// }
// data, err := json.Marshal(&info)
// if err != nil {
// return "", err
// }
// if _, err := m.core.Client.PutObject(ctx, m.bucket, objectInfoPath, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{}); err != nil {
// return "", err
// }
//} else {
// return "", err
//}
//if opt.Image.Width > info.Width || opt.Image.Width <= 0 {
// opt.Image.Width = info.Width
//}
//if opt.Image.Height > info.Height || opt.Image.Height <= 0 {
// opt.Image.Height = info.Height
//}
//opt.Image.Format = strings.ToLower(opt.Image.Format)
//if opt.Image.Format == formatJpg {
// opt.Image.Format = formatJpeg
//}
//switch opt.Image.Format {
//case formatPng:
//case formatJpeg:
//case formatGif:
//default:
// if info.Format == formatGif {
// opt.Image.Format = formatGif
// } else {
// opt.Image.Format = formatJpeg
// }
//}
//reqParams.Set("response-content-type", "image/"+opt.Image.Format)
//if opt.Image.Width == info.Width && opt.Image.Height == info.Height && opt.Image.Format == info.Format {
// return m.presignedGetObject(ctx, name, expire, reqParams)
//}
//cacheKey := filepath.Join(pathInfo, fileInfo.ETag, fmt.Sprintf("image_w%d_h%d.%s", opt.Image.Width, opt.Image.Height, opt.Image.Format))
//if _, err := m.core.Client.StatObject(ctx, m.bucket, cacheKey, minio.StatObjectOptions{}); err == nil {
// return m.presignedGetObject(ctx, cacheKey, expire, reqParams)
//} else if !m.IsNotFound(err) {
// return "", err
//}
//if img == nil {
// reader, err := m.core.Client.GetObject(ctx, m.bucket, name, minio.GetObjectOptions{})
// if err != nil {
// return "", err
// }
// defer reader.Close()
// img, _, err = ImageStat(reader)
// if err != nil {
// return "", err
// }
//}
//thumbnail := resizeImage(img, opt.Image.Width, opt.Image.Height)
//buf := bytes.NewBuffer(nil)
//switch opt.Image.Format {
//case formatPng:
// err = png.Encode(buf, thumbnail)
//case formatJpeg:
// err = jpeg.Encode(buf, thumbnail, nil)
//case formatGif:
// err = gif.Encode(buf, thumbnail, nil)
//}
//if _, err := m.core.Client.PutObject(ctx, m.bucket, cacheKey, buf, int64(buf.Len()), minio.PutObjectOptions{}); err != nil {
// return "", err
//}
//return m.presignedGetObject(ctx, cacheKey, expire, reqParams)
} }
func (m *Minio) getObjectData(ctx context.Context, name string, limit int64) ([]byte, error) { func (m *Minio) getObjectData(ctx context.Context, name string, limit int64) ([]byte, error) {
@ -540,8 +439,3 @@ func (m *Minio) getObjectData(ctx context.Context, name string, limit int64) ([]
} }
return io.ReadAll(io.LimitReader(object, limit)) return io.ReadAll(io.LimitReader(object, limit))
} }
func (m *Minio) GetThumbnailKey(ctx context.Context, name string) (string, error) {
return "", nil
}

22
pkg/common/db/s3/minio/struct.go
View File

@ -1,22 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package minio
type minioImageInfo struct {
NotImage bool `json:"notImage,omitempty"`
Width int `json:"width,omitempty"`
Height int `json:"height,omitempty"`
Format string `json:"format,omitempty"`
}

145
pkg/common/db/s3/minio/thumbnail.go
View File

@ -3,9 +3,10 @@ package minio
import ( import (
"bytes" "bytes"
"context" "context"
"encoding/json"
"errors" "errors"
"fmt" "fmt"
"github.com/OpenIMSDK/tools/errs"
"github.com/OpenIMSDK/tools/log"
"github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache" "github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3" "github.com/openimsdk/open-im-server/v3/pkg/common/db/s3"
@ -14,27 +15,23 @@ import (
"image/jpeg" "image/jpeg"
"image/png" "image/png"
"net/url" "net/url"
"path"
"path/filepath" "path/filepath"
"strings" "strings"
"time" "time"
) )
//func (m *Minio) getHashImageInfo1(ctx context.Context, key string) (*cache.MinioImageInfo, image.Image, error) { func (m *Minio) getImageThumbnailURL(ctx context.Context, name string, expire time.Duration, opt *s3.Image) (string, error) {
// var img image.Image
// return nil, nil, nil info, err := m.cache.GetImageObjectKeyInfo(ctx, name, func(ctx context.Context) (info *cache.MinioImageInfo, err error) {
//} info, img, err = m.getObjectImageInfo(ctx, name)
return
func (m *Minio) get1(ctx context.Context, key string, format string, width int, height int, info *cache.MinioImageInfo, img image.Image) (string, error) { })
return "", nil
}
func (m *Minio) getHashImageInfo(ctx context.Context, name string, expire time.Duration, opt *s3.Image) (string, error) {
info, img, err := m.cache.GetImageObjectKeyInfo(ctx, name, m.getObjectImageInfo)
if err != nil { if err != nil {
return "", err return "", err
} }
if !info.IsImg {
return "", errs.ErrData.Wrap("object not image")
}
if opt.Width > info.Width || opt.Width <= 0 { if opt.Width > info.Width || opt.Width <= 0 {
opt.Width = info.Width opt.Width = info.Width
} }
@ -49,14 +46,11 @@ func (m *Minio) getHashImageInfo(ctx context.Context, name string, expire time.D
case formatPng, formatJpeg, formatGif: case formatPng, formatJpeg, formatGif:
default: default:
opt.Format = "" opt.Format = ""
//if info.Format == formatGif {
// opt.Format = formatGif
//} else {
// opt.Format = formatJpeg
//}
} }
reqParams := make(url.Values)
if opt.Width == info.Width && opt.Height == info.Height && (opt.Format == info.Format || opt.Format == "") { if opt.Width == info.Width && opt.Height == info.Height && (opt.Format == info.Format || opt.Format == "") {
return "", nil reqParams.Set("response-content-type", "image/"+info.Format)
return m.PresignedGetObject(ctx, name, expire, reqParams)
} }
if opt.Format == "" { if opt.Format == "" {
switch opt.Format { switch opt.Format {
@ -92,7 +86,7 @@ func (m *Minio) getHashImageInfo(ctx context.Context, name string, expire time.D
case formatGif: case formatGif:
err = gif.Encode(buf, thumbnail, nil) err = gif.Encode(buf, thumbnail, nil)
} }
cacheKey := filepath.Join(pathInfo, info.Etag, fmt.Sprintf("image_w%d_h%d.%s", opt.Width, opt.Height, opt.Format)) cacheKey := filepath.Join(imageThumbnailPath, info.Etag, fmt.Sprintf("image_w%d_h%d.%s", opt.Width, opt.Height, opt.Format))
if _, err := m.core.Client.PutObject(ctx, m.bucket, cacheKey, buf, int64(buf.Len()), minio.PutObjectOptions{}); err != nil { if _, err := m.core.Client.PutObject(ctx, m.bucket, cacheKey, buf, int64(buf.Len()), minio.PutObjectOptions{}); err != nil {
return "", err return "", err
} }
@ -101,7 +95,8 @@ func (m *Minio) getHashImageInfo(ctx context.Context, name string, expire time.D
if err != nil { if err != nil {
return "", err return "", err
} }
return m.presignedGetObject(ctx, key, expire, reqParams) reqParams.Set("response-content-type", "image/"+opt.Format)
return m.PresignedGetObject(ctx, key, expire, reqParams)
} }
func (m *Minio) getObjectImageInfo(ctx context.Context, name string) (*cache.MinioImageInfo, image.Image, error) { func (m *Minio) getObjectImageInfo(ctx context.Context, name string) (*cache.MinioImageInfo, image.Image, error) {
@ -129,107 +124,11 @@ func (m *Minio) getObjectImageInfo(ctx context.Context, name string) (*cache.Min
return &info, imageInfo, nil return &info, imageInfo, nil
} }
func (m *Minio) GetImageThumbnail(ctx context.Context, name string, expire time.Duration, opt *s3.Image) (string, error) { func (m *Minio) delObjectImageInfoKey(ctx context.Context, key string, size int64) {
fileInfo, err := m.StatObject(ctx, name) if size > 0 && size > maxImageSize {
if err != nil { return
return "", err
} }
if fileInfo.Size > maxImageSize { if err := m.cache.DelObjectImageInfoKey(key).ExecDel(ctx); err != nil {
return "", errors.New("file size too large") log.ZError(ctx, "DelObjectImageInfoKey failed", err, "key", key)
}
objectInfoPath := path.Join(pathInfo, fileInfo.ETag, "image.json")
var (
img image.Image
info minioImageInfo
)
data, err := m.getObjectData(ctx, objectInfoPath, maxImageInfoSize)
if err == nil {
if err := json.Unmarshal(data, &info); err != nil {
return "", fmt.Errorf("unmarshal minio image info.json error: %w", err)
}
if info.NotImage {
return "", errors.New("not image")
}
} else if m.IsNotFound(err) {
reader, err := m.core.Client.GetObject(ctx, m.bucket, name, minio.GetObjectOptions{})
if err != nil {
return "", err
}
defer reader.Close()
imageInfo, format, err := ImageStat(reader)
if err == nil {
info.NotImage = false
info.Format = format
info.Width, info.Height = ImageWidthHeight(imageInfo)
img = imageInfo
} else {
info.NotImage = true
}
data, err := json.Marshal(&info)
if err != nil {
return "", err
}
if _, err := m.core.Client.PutObject(ctx, m.bucket, objectInfoPath, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{}); err != nil {
return "", err
}
} else {
return "", err
}
if opt.Width > info.Width || opt.Width <= 0 {
opt.Width = info.Width
}
if opt.Height > info.Height || opt.Height <= 0 {
opt.Height = info.Height
}
opt.Format = strings.ToLower(opt.Format)
if opt.Format == formatJpg {
opt.Format = formatJpeg
}
switch opt.Format {
case formatPng:
case formatJpeg:
case formatGif:
default:
if info.Format == formatGif {
opt.Format = formatGif
} else {
opt.Format = formatJpeg
} }
} }
reqParams := make(url.Values)
reqParams.Set("response-content-type", "image/"+opt.Format)
if opt.Width == info.Width && opt.Height == info.Height && opt.Format == info.Format {
return m.presignedGetObject(ctx, name, expire, reqParams)
}
cacheKey := filepath.Join(pathInfo, fileInfo.ETag, fmt.Sprintf("image_w%d_h%d.%s", opt.Width, opt.Height, opt.Format))
if _, err := m.core.Client.StatObject(ctx, m.bucket, cacheKey, minio.StatObjectOptions{}); err == nil {
return m.presignedGetObject(ctx, cacheKey, expire, reqParams)
} else if !m.IsNotFound(err) {
return "", err
}
if img == nil {
reader, err := m.core.Client.GetObject(ctx, m.bucket, name, minio.GetObjectOptions{})
if err != nil {
return "", err
}
defer reader.Close()
img, _, err = ImageStat(reader)
if err != nil {
return "", err
}
}
thumbnail := resizeImage(img, opt.Width, opt.Height)
buf := bytes.NewBuffer(nil)
switch opt.Format {
case formatPng:
err = png.Encode(buf, thumbnail)
case formatJpeg:
err = jpeg.Encode(buf, thumbnail, nil)
case formatGif:
err = gif.Encode(buf, thumbnail, nil)
}
if _, err := m.core.Client.PutObject(ctx, m.bucket, cacheKey, buf, int64(buf.Len()), minio.PutObjectOptions{}); err != nil {
return "", err
}
return m.presignedGetObject(ctx, cacheKey, expire, reqParams)
}