blackteay/open-im-server
1
0
Fork 0
mirror of https://github.com/openimsdk/open-im-server.git synced 2025-12-10 14:54:41 +08:00
Code Issues Packages Projects Releases Wiki Activity

minio cache

Browse Source
This commit is contained in:
withchao 2023-11-02 12:49:01 +08:00
parent ce9883cf26
commit c9ab3c63f8
4 changed files with 425 additions and 158 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
pkg/common/db/cache/meta_cache.go vendored
View File

@ -209,6 +209,9 @@ func batchGetCache2[T any, K comparable](
return fns(ctx, key) return fns(ctx, key)
}) })
if err != nil { if err != nil {
if errs.ErrRecordNotFound.Is(err) {
continue
}
return nil, err return nil, err
} }
res = append(res, val) res = append(res, val)

120
pkg/common/db/cache/s3.go vendored
View File

@ -2,10 +2,12 @@ package cache
import ( import (
"context" "context"
"github.com/OpenIMSDK/tools/errs"
"github.com/dtm-labs/rockscache" "github.com/dtm-labs/rockscache"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3" "github.com/openimsdk/open-im-server/v3/pkg/common/db/s3"
relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/redis/go-redis/v9" "github.com/redis/go-redis/v9"
"image"
"time" "time"
) )
@ -113,39 +115,40 @@ func (g *s3CacheRedis) GetKey(ctx context.Context, engine string, name string) (
}) })
} }
//type MinioCache interface { type MinioCache interface {
// metaCache metaCache
// GetThumbnailKey(ctx context.Context, key string) (string, error) GetImageObjectKeyInfo(ctx context.Context, key string, fn func(ctx context.Context, key string) (*MinioImageInfo, image.Image, error)) (*MinioImageInfo, image.Image, error)
// //DelS3Key(engine string, keys ...string) S3Cache GetThumbnailKey(ctx context.Context, key string, format string, width int, height int, minioCache func(ctx context.Context) (string, error)) (string, error)
//} //DelS3Key(engine string, keys ...string) S3Cache
// }
//func NewMinioCache(rdb redis.UniversalClient, s3 s3.Interface) MinioCache {
// rcClient := rockscache.NewClient(rdb, rockscache.NewDefaultOptions()) func NewMinioCache(rdb redis.UniversalClient, s3 s3.Interface) MinioCache {
// return &minioCacheRedis{ rcClient := rockscache.NewClient(rdb, rockscache.NewDefaultOptions())
// rcClient: rcClient, return &minioCacheRedis{
// expireTime: time.Hour * 12, rcClient: rcClient,
// s3: s3, expireTime: time.Hour * 12,
// metaCache: NewMetaCacheRedis(rcClient), s3: s3,
// } metaCache: NewMetaCacheRedis(rcClient),
//} }
// }
//type minioCacheRedis struct {
// metaCache type minioCacheRedis struct {
// s3 s3.Interface metaCache
// rcClient *rockscache.Client s3 s3.Interface
// expireTime time.Duration rcClient *rockscache.Client
//} expireTime time.Duration
// }
//func (g *minioCacheRedis) NewCache() MinioCache {
// return &minioCacheRedis{ func (g *minioCacheRedis) NewCache() MinioCache {
// rcClient: g.rcClient, return &minioCacheRedis{
// expireTime: g.expireTime, rcClient: g.rcClient,
// s3: g.s3, expireTime: g.expireTime,
// metaCache: NewMetaCacheRedis(g.rcClient, g.metaCache.GetPreDelKeys()...), s3: g.s3,
// } metaCache: NewMetaCacheRedis(g.rcClient, g.metaCache.GetPreDelKeys()...),
//} }
// }
//func (g *minioCacheRedis) DelS3Key(engine string, keys ...string) S3Cache {
//func (g *minioCacheRedis) DelS3Key(engine string, keys ...string) MinioCache {
// s3cache := g.NewCache() // s3cache := g.NewCache()
// ks := make([]string, 0, len(keys)) // ks := make([]string, 0, len(keys))
// for _, key := range keys { // for _, key := range keys {
@ -154,13 +157,44 @@ func (g *s3CacheRedis) GetKey(ctx context.Context, engine string, name string) (
// s3cache.AddKeys(ks...) // s3cache.AddKeys(ks...)
// return s3cache // return s3cache
//} //}
//
//func (g *minioCacheRedis) getMinioImageInfoKey(name string) string { func (g *minioCacheRedis) getMinioImageInfoKey(name string) string {
// return "MINIO:" + ":" + name return "MINIO:IMAGE:" + name
//} }
//
//func (g *minioCacheRedis) GetThumbnailKey(ctx context.Context, name string) (string, error) { func (g *minioCacheRedis) getMinioImageThumbnailKey(name string) string {
// return getCache(ctx, g.rcClient, g.getS3Key(engine, name), g.expireTime, func(ctx context.Context) (*s3.ObjectInfo, error) { return "MINIO:THUMBNAIL:" + name
// return g.s3.StatObject(ctx, name) }
// })
//} func (g *minioCacheRedis) GetImageObjectKeyInfo(ctx context.Context, key string, fn func(ctx context.Context, key string) (*MinioImageInfo, image.Image, error)) (*MinioImageInfo, image.Image, error) {
var img image.Image
info, err := getCache(ctx, g.rcClient, g.getMinioImageInfoKey(key), g.expireTime, func(ctx context.Context) (info *MinioImageInfo, err error) {
info, img, err = fn(ctx, key)
return
})
if err != nil {
return nil, nil, err
}
if !info.IsImg {
return nil, nil, errs.ErrData.Wrap("object not image")
}
return info, img, nil
}
func (g *minioCacheRedis) GetThumbnailKey(ctx context.Context, key string, format string, width int, height int, minioCache func(ctx context.Context, key string, format string, width int, height int) (string, error)) (string, error) {
return getCache(ctx, g.rcClient, g.getMinioImageThumbnailKey(key), g.expireTime, func(ctx context.Context) (string, error) {
info, img, err := g.GetImageObjectKeyInfo(ctx, key, getInfo)
if err != nil {
return "", err
}
return minioCache(ctx, key, format, width, height, info, img)
})
}
type MinioImageInfo struct {
IsImg bool `json:"i,omitempty"`
Width int `json:"w,omitempty"`
Height int `json:"h,omitempty"`
Format string `json:"f,omitempty"`
Etag string `json:"e,omitempty"`
}

225
pkg/common/db/s3/minio/minio.go
View File

@ -15,21 +15,14 @@
package minio package minio
import ( import (
"bytes"
"context" "context"
"encoding/json"
"errors" "errors"
"fmt" "fmt"
"github.com/dtm-labs/rockscache" "github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
"image"
"image/gif"
"image/jpeg"
"image/png"
"io" "io"
"net/http" "net/http"
"net/url" "net/url"
"path" "path"
"path/filepath"
"reflect" "reflect"
"strconv" "strconv"
"strings" "strings"
@ -57,10 +50,11 @@ const (
) )
const ( const (
maxImageWidth = 1024 maxImageWidth = 1024
maxImageHeight = 1024 maxImageHeight = 1024
maxImageSize = 1024 * 1024 * 50 maxImageSize = 1024 * 1024 * 50
pathInfo = "openim/thumbnail" pathInfo = "openim/thumbnail"
maxImageInfoSize = 1024
) )
func NewMinio() (s3.Interface, error) { func NewMinio() (s3.Interface, error) {
@ -125,7 +119,7 @@ type Minio struct {
lock sync.Locker lock sync.Locker
init bool init bool
prefix string prefix string
rcClient *rockscache.Client cache cache.MinioCache
} }
func (m *Minio) initMinio(ctx context.Context) error { func (m *Minio) initMinio(ctx context.Context) error {
@ -431,107 +425,108 @@ func (m *Minio) AccessURL(ctx context.Context, name string, expire time.Duration
if opt.Image == nil || (opt.Image.Width < 0 && opt.Image.Height < 0 && opt.Image.Format == "") || (opt.Image.Width > maxImageWidth || opt.Image.Height > maxImageHeight) { if opt.Image == nil || (opt.Image.Width < 0 && opt.Image.Height < 0 && opt.Image.Format == "") || (opt.Image.Width > maxImageWidth || opt.Image.Height > maxImageHeight) {
return m.presignedGetObject(ctx, name, expire, reqParams) return m.presignedGetObject(ctx, name, expire, reqParams)
} }
fileInfo, err := m.StatObject(ctx, name) return m.GetImageThumbnail(ctx, name, expire, opt.Image)
if err != nil { //fileInfo, err := m.StatObject(ctx, name)
return "", err //if err != nil {
} // return "", err
if fileInfo.Size > maxImageSize { //}
return "", errors.New("file size too large") //if fileInfo.Size > maxImageSize {
} // return "", errors.New("file size too large")
objectInfoPath := path.Join(pathInfo, fileInfo.ETag, "image.json") //}
var ( //objectInfoPath := path.Join(pathInfo, fileInfo.ETag, "image.json")
img image.Image //var (
info minioImageInfo // img image.Image
) // info minioImageInfo
data, err := m.getObjectData(ctx, objectInfoPath, 1024) //)
if err == nil { //data, err := m.getObjectData(ctx, objectInfoPath, 1024)
if err := json.Unmarshal(data, &info); err != nil { //if err == nil {
return "", fmt.Errorf("unmarshal minio image info.json error: %w", err) // if err := json.Unmarshal(data, &info); err != nil {
} // return "", fmt.Errorf("unmarshal minio image info.json error: %w", err)
if info.NotImage { // }
return "", errors.New("not image") // if info.NotImage {
} // return "", errors.New("not image")
} else if m.IsNotFound(err) { // }
reader, err := m.core.Client.GetObject(ctx, m.bucket, name, minio.GetObjectOptions{}) //} else if m.IsNotFound(err) {
if err != nil { // reader, err := m.core.Client.GetObject(ctx, m.bucket, name, minio.GetObjectOptions{})
return "", err // if err != nil {
} // return "", err
defer reader.Close() // }
imageInfo, format, err := ImageStat(reader) // defer reader.Close()
if err == nil { // imageInfo, format, err := ImageStat(reader)
info.NotImage = false // if err == nil {
info.Format = format // info.NotImage = false
info.Width, info.Height = ImageWidthHeight(imageInfo) // info.Format = format
img = imageInfo // info.Width, info.Height = ImageWidthHeight(imageInfo)
} else { // img = imageInfo
info.NotImage = true // } else {
} // info.NotImage = true
data, err := json.Marshal(&info) // }
if err != nil { // data, err := json.Marshal(&info)
return "", err // if err != nil {
} // return "", err
if _, err := m.core.Client.PutObject(ctx, m.bucket, objectInfoPath, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{}); err != nil { // }
return "", err // if _, err := m.core.Client.PutObject(ctx, m.bucket, objectInfoPath, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{}); err != nil {
} // return "", err
} else { // }
return "", err //} else {
} // return "", err
if opt.Image.Width > info.Width || opt.Image.Width <= 0 { //}
opt.Image.Width = info.Width //if opt.Image.Width > info.Width || opt.Image.Width <= 0 {
} // opt.Image.Width = info.Width
if opt.Image.Height > info.Height || opt.Image.Height <= 0 { //}
opt.Image.Height = info.Height //if opt.Image.Height > info.Height || opt.Image.Height <= 0 {
} // opt.Image.Height = info.Height
opt.Image.Format = strings.ToLower(opt.Image.Format) //}
if opt.Image.Format == formatJpg { //opt.Image.Format = strings.ToLower(opt.Image.Format)
opt.Image.Format = formatJpeg //if opt.Image.Format == formatJpg {
} // opt.Image.Format = formatJpeg
switch opt.Image.Format { //}
case formatPng: //switch opt.Image.Format {
case formatJpeg: //case formatPng:
case formatGif: //case formatJpeg:
default: //case formatGif:
if info.Format == formatGif { //default:
opt.Image.Format = formatGif // if info.Format == formatGif {
} else { // opt.Image.Format = formatGif
opt.Image.Format = formatJpeg // } else {
} // opt.Image.Format = formatJpeg
} // }
reqParams.Set("response-content-type", "image/"+opt.Image.Format) //}
if opt.Image.Width == info.Width && opt.Image.Height == info.Height && opt.Image.Format == info.Format { //reqParams.Set("response-content-type", "image/"+opt.Image.Format)
return m.presignedGetObject(ctx, name, expire, reqParams) //if opt.Image.Width == info.Width && opt.Image.Height == info.Height && opt.Image.Format == info.Format {
} // return m.presignedGetObject(ctx, name, expire, reqParams)
cacheKey := filepath.Join(pathInfo, fileInfo.ETag, fmt.Sprintf("image_w%d_h%d.%s", opt.Image.Width, opt.Image.Height, opt.Image.Format)) //}
if _, err := m.core.Client.StatObject(ctx, m.bucket, cacheKey, minio.StatObjectOptions{}); err == nil { //cacheKey := filepath.Join(pathInfo, fileInfo.ETag, fmt.Sprintf("image_w%d_h%d.%s", opt.Image.Width, opt.Image.Height, opt.Image.Format))
return m.presignedGetObject(ctx, cacheKey, expire, reqParams) //if _, err := m.core.Client.StatObject(ctx, m.bucket, cacheKey, minio.StatObjectOptions{}); err == nil {
} else if !m.IsNotFound(err) { // return m.presignedGetObject(ctx, cacheKey, expire, reqParams)
return "", err //} else if !m.IsNotFound(err) {
} // return "", err
if img == nil { //}
reader, err := m.core.Client.GetObject(ctx, m.bucket, name, minio.GetObjectOptions{}) //if img == nil {
if err != nil { // reader, err := m.core.Client.GetObject(ctx, m.bucket, name, minio.GetObjectOptions{})
return "", err // if err != nil {
} // return "", err
defer reader.Close() // }
img, _, err = ImageStat(reader) // defer reader.Close()
if err != nil { // img, _, err = ImageStat(reader)
return "", err // if err != nil {
} // return "", err
} // }
thumbnail := resizeImage(img, opt.Image.Width, opt.Image.Height) //}
buf := bytes.NewBuffer(nil) //thumbnail := resizeImage(img, opt.Image.Width, opt.Image.Height)
switch opt.Image.Format { //buf := bytes.NewBuffer(nil)
case formatPng: //switch opt.Image.Format {
err = png.Encode(buf, thumbnail) //case formatPng:
case formatJpeg: // err = png.Encode(buf, thumbnail)
err = jpeg.Encode(buf, thumbnail, nil) //case formatJpeg:
case formatGif: // err = jpeg.Encode(buf, thumbnail, nil)
err = gif.Encode(buf, thumbnail, nil) //case formatGif:
} // err = gif.Encode(buf, thumbnail, nil)
if _, err := m.core.Client.PutObject(ctx, m.bucket, cacheKey, buf, int64(buf.Len()), minio.PutObjectOptions{}); err != nil { //}
return "", err //if _, err := m.core.Client.PutObject(ctx, m.bucket, cacheKey, buf, int64(buf.Len()), minio.PutObjectOptions{}); err != nil {
} // return "", err
return m.presignedGetObject(ctx, cacheKey, expire, reqParams) //}
//return m.presignedGetObject(ctx, cacheKey, expire, reqParams)
} }
func (m *Minio) getObjectData(ctx context.Context, name string, limit int64) ([]byte, error) { func (m *Minio) getObjectData(ctx context.Context, name string, limit int64) ([]byte, error) {
@ -543,7 +538,7 @@ func (m *Minio) getObjectData(ctx context.Context, name string, limit int64) ([]
if limit < 0 { if limit < 0 {
return io.ReadAll(object) return io.ReadAll(object)
} }
return io.ReadAll(io.LimitReader(object, 1024)) return io.ReadAll(io.LimitReader(object, limit))
} }
func (m *Minio) GetThumbnailKey(ctx context.Context, name string) (string, error) { func (m *Minio) GetThumbnailKey(ctx context.Context, name string) (string, error) {

235
pkg/common/db/s3/minio/thumbnail.go Normal file
View File

@ -0,0 +1,235 @@
package minio
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"github.com/minio/minio-go/v7"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3"
"image"
"image/gif"
"image/jpeg"
"image/png"
"net/url"
"path"
"path/filepath"
"strings"
"time"
)
//func (m *Minio) getHashImageInfo1(ctx context.Context, key string) (*cache.MinioImageInfo, image.Image, error) {
//
// return nil, nil, nil
//}
func (m *Minio) get1(ctx context.Context, key string, format string, width int, height int, info *cache.MinioImageInfo, img image.Image) (string, error) {
return "", nil
}
func (m *Minio) getHashImageInfo(ctx context.Context, name string, expire time.Duration, opt *s3.Image) (string, error) {
info, img, err := m.cache.GetImageObjectKeyInfo(ctx, name, m.getObjectImageInfo)
if err != nil {
return "", err
}
if opt.Width > info.Width || opt.Width <= 0 {
opt.Width = info.Width
}
if opt.Height > info.Height || opt.Height <= 0 {
opt.Height = info.Height
}
opt.Format = strings.ToLower(opt.Format)
if opt.Format == formatJpg {
opt.Format = formatJpeg
}
switch opt.Format {
case formatPng, formatJpeg, formatGif:
default:
opt.Format = ""
//if info.Format == formatGif {
// opt.Format = formatGif
//} else {
// opt.Format = formatJpeg
//}
}
if opt.Width == info.Width && opt.Height == info.Height && (opt.Format == info.Format || opt.Format == "") {
return "", nil
}
if opt.Format == "" {
switch opt.Format {
case formatGif:
opt.Format = formatGif
case formatJpeg:
opt.Format = formatJpeg
case formatPng:
opt.Format = formatPng
default:
opt.Format = formatPng
}
}
key, err := m.cache.GetThumbnailKey(ctx, name, opt.Format, opt.Width, opt.Height, func(ctx context.Context) (string, error) {
if img == nil {
reader, err := m.core.Client.GetObject(ctx, m.bucket, name, minio.GetObjectOptions{})
if err != nil {
return "", err
}
defer reader.Close()
img, _, err = ImageStat(reader)
if err != nil {
return "", err
}
}
thumbnail := resizeImage(img, opt.Width, opt.Height)
buf := bytes.NewBuffer(nil)
switch opt.Format {
case formatPng:
err = png.Encode(buf, thumbnail)
case formatJpeg:
err = jpeg.Encode(buf, thumbnail, nil)
case formatGif:
err = gif.Encode(buf, thumbnail, nil)
}
cacheKey := filepath.Join(pathInfo, info.Etag, fmt.Sprintf("image_w%d_h%d.%s", opt.Width, opt.Height, opt.Format))
if _, err := m.core.Client.PutObject(ctx, m.bucket, cacheKey, buf, int64(buf.Len()), minio.PutObjectOptions{}); err != nil {
return "", err
}
return cacheKey, nil
})
if err != nil {
return "", err
}
return m.presignedGetObject(ctx, key, expire, reqParams)
}
func (m *Minio) getObjectImageInfo(ctx context.Context, name string) (*cache.MinioImageInfo, image.Image, error) {
fileInfo, err := m.StatObject(ctx, name)
if err != nil {
return nil, nil, err
}
if fileInfo.Size > maxImageSize {
return nil, nil, errors.New("file size too large")
}
imageData, err := m.getObjectData(ctx, name, fileInfo.Size)
if err != nil {
return nil, nil, err
}
var info cache.MinioImageInfo
imageInfo, format, err := ImageStat(bytes.NewReader(imageData))
if err == nil {
info.IsImg = true
info.Format = format
info.Width, info.Height = ImageWidthHeight(imageInfo)
} else {
info.IsImg = false
}
info.Etag = fileInfo.ETag
return &info, imageInfo, nil
}
func (m *Minio) GetImageThumbnail(ctx context.Context, name string, expire time.Duration, opt *s3.Image) (string, error) {
fileInfo, err := m.StatObject(ctx, name)
if err != nil {
return "", err
}
if fileInfo.Size > maxImageSize {
return "", errors.New("file size too large")
}
objectInfoPath := path.Join(pathInfo, fileInfo.ETag, "image.json")
var (
img image.Image
info minioImageInfo
)
data, err := m.getObjectData(ctx, objectInfoPath, maxImageInfoSize)
if err == nil {
if err := json.Unmarshal(data, &info); err != nil {
return "", fmt.Errorf("unmarshal minio image info.json error: %w", err)
}
if info.NotImage {
return "", errors.New("not image")
}
} else if m.IsNotFound(err) {
reader, err := m.core.Client.GetObject(ctx, m.bucket, name, minio.GetObjectOptions{})
if err != nil {
return "", err
}
defer reader.Close()
imageInfo, format, err := ImageStat(reader)
if err == nil {
info.NotImage = false
info.Format = format
info.Width, info.Height = ImageWidthHeight(imageInfo)
img = imageInfo
} else {
info.NotImage = true
}
data, err := json.Marshal(&info)
if err != nil {
return "", err
}
if _, err := m.core.Client.PutObject(ctx, m.bucket, objectInfoPath, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{}); err != nil {
return "", err
}
} else {
return "", err
}
if opt.Width > info.Width || opt.Width <= 0 {
opt.Width = info.Width
}
if opt.Height > info.Height || opt.Height <= 0 {
opt.Height = info.Height
}
opt.Format = strings.ToLower(opt.Format)
if opt.Format == formatJpg {
opt.Format = formatJpeg
}
switch opt.Format {
case formatPng:
case formatJpeg:
case formatGif:
default:
if info.Format == formatGif {
opt.Format = formatGif
} else {
opt.Format = formatJpeg
}
}
reqParams := make(url.Values)
reqParams.Set("response-content-type", "image/"+opt.Format)
if opt.Width == info.Width && opt.Height == info.Height && opt.Format == info.Format {
return m.presignedGetObject(ctx, name, expire, reqParams)
}
cacheKey := filepath.Join(pathInfo, fileInfo.ETag, fmt.Sprintf("image_w%d_h%d.%s", opt.Width, opt.Height, opt.Format))
if _, err := m.core.Client.StatObject(ctx, m.bucket, cacheKey, minio.StatObjectOptions{}); err == nil {
return m.presignedGetObject(ctx, cacheKey, expire, reqParams)
} else if !m.IsNotFound(err) {
return "", err
}
if img == nil {
reader, err := m.core.Client.GetObject(ctx, m.bucket, name, minio.GetObjectOptions{})
if err != nil {
return "", err
}
defer reader.Close()
img, _, err = ImageStat(reader)
if err != nil {
return "", err
}
}
thumbnail := resizeImage(img, opt.Width, opt.Height)
buf := bytes.NewBuffer(nil)
switch opt.Format {
case formatPng:
err = png.Encode(buf, thumbnail)
case formatJpeg:
err = jpeg.Encode(buf, thumbnail, nil)
case formatGif:
err = gif.Encode(buf, thumbnail, nil)
}
if _, err := m.core.Client.PutObject(ctx, m.bucket, cacheKey, buf, int64(buf.Len()), minio.PutObjectOptions{}); err != nil {
return "", err
}
return m.presignedGetObject(ctx, cacheKey, expire, reqParams)
}