diff --git a/config/config.yaml b/config/config.yaml index 9d6b3c335..0d2493604 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -140,11 +140,13 @@ object: secretAccessKey: "openIM123" sessionToken: '' signEndpoint: "http://127.0.0.1:10005" + publicRead: false cos: bucketURL: https://temp-1252357374.cos.ap-chengdu.myqcloud.com secretID: '' secretKey: '' sessionToken: '' + publicRead: false oss: endpoint: "https://oss-cn-chengdu.aliyuncs.com" bucket: "demo-9999999" @@ -152,6 +154,7 @@ object: accessKeyID: '' accessKeySecret: '' sessionToken: '' + publicRead: false ###################### RPC Port Configuration ###################### diff --git a/deployments/templates/openim.yaml b/deployments/templates/openim.yaml index 9465ea872..5c6be94f5 100644 --- a/deployments/templates/openim.yaml +++ b/deployments/templates/openim.yaml @@ -140,11 +140,13 @@ object: secretAccessKey: "${MINIO_SECRET_KEY}" sessionToken: ${MINIO_SESSION_TOKEN} signEndpoint: "${MINIO_SIGN_ENDPOINT}" + publicRead: ${MINIO_PUBLIC_READ} cos: bucketURL: ${COS_BUCKET_URL} secretID: ${COS_SECRET_ID} secretKey: ${COS_SECRET_KEY} sessionToken: ${COS_SESSION_TOKEN} + publicRead: ${COS_PUBLIC_READ} oss: endpoint: "${OSS_ENDPOINT}" bucket: "${OSS_BUCKET}" @@ -152,7 +154,7 @@ object: accessKeyID: ${OSS_ACCESS_KEY_ID} accessKeySecret: ${OSS_ACCESS_KEY_SECRET} sessionToken: ${OSS_SESSION_TOKEN} - + publicRead: ${OSS_PUBLIC_READ} ###################### RPC Port Configuration ###################### # RPC service ports diff --git a/internal/api/msg.go b/internal/api/msg.go index 2777d8855..83949903e 100644 --- a/internal/api/msg.go +++ b/internal/api/msg.go @@ -58,7 +58,7 @@ func (m MessageApi) newUserSendMsgReq(c *gin.Context, params *apistruct.SendMsg) options := make(map[string]bool, 5) switch params.ContentType { case constant.Text: - newContent = params.Content["text"].(string) + fallthrough case constant.Picture: fallthrough case constant.Custom: diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index c6dd41419..c31814743 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -128,12 +128,14 @@ type configStruct struct { SecretAccessKey string `yaml:"secretAccessKey"` SessionToken string `yaml:"sessionToken"` SignEndpoint string `yaml:"signEndpoint"` + PublicRead bool `yaml:"publicRead"` } `yaml:"minio"` Cos struct { BucketURL string `yaml:"bucketURL"` SecretID string `yaml:"secretID"` SecretKey string `yaml:"secretKey"` SessionToken string `yaml:"sessionToken"` + PublicRead bool `yaml:"publicRead"` } `yaml:"cos"` Oss struct { Endpoint string `yaml:"endpoint"` @@ -142,6 +144,7 @@ type configStruct struct { AccessKeyID string `yaml:"accessKeyID"` AccessKeySecret string `yaml:"accessKeySecret"` SessionToken string `yaml:"sessionToken"` + PublicRead bool `yaml:"publicRead"` } `yaml:"oss"` } `yaml:"object"` diff --git a/pkg/common/db/s3/cos/cos.go b/pkg/common/db/s3/cos/cos.go index 57a205cbe..7add88487 100644 --- a/pkg/common/db/s3/cos/cos.go +++ b/pkg/common/db/s3/cos/cos.go @@ -288,7 +288,7 @@ func (c *Cos) AccessURL(ctx context.Context, name string, expire time.Duration, style = append(style, "format/"+opt.Image.Format) } if len(style) > 0 { - imageMogr = "&imageMogr2/thumbnail/" + strings.Join(style, "/") + "/ignore-error/1" + imageMogr = "imageMogr2/thumbnail/" + strings.Join(style, "/") + "/ignore-error/1" } } if opt.ContentType != "" { @@ -306,13 +306,23 @@ func (c *Cos) AccessURL(ctx context.Context, name string, expire time.Duration, } else if expire < time.Second { expire = time.Second } - rawURL, err := c.client.Object.GetPresignedURL(ctx, http.MethodGet, name, c.credential.SecretID, c.credential.SecretKey, expire, &option) + rawURL, err := c.getPresignedURL(ctx, name, expire, &option) if err != nil { return "", err } - urlStr := rawURL.String() if imageMogr != "" { - urlStr += imageMogr + if rawURL.RawQuery == "" { + rawURL.RawQuery = imageMogr + } else { + rawURL.RawQuery = rawURL.RawQuery + "&" + imageMogr + } } - return urlStr, nil + return rawURL.String(), nil +} + +func (c *Cos) getPresignedURL(ctx context.Context, name string, expire time.Duration, opt *cos.PresignedURLOptions) (*url.URL, error) { + if !config.Config.Object.Cos.PublicRead { + return c.client.Object.GetPresignedURL(ctx, http.MethodGet, name, c.credential.SecretID, c.credential.SecretKey, expire, opt) + } + return c.client.Object.GetObjectURL(name), nil } diff --git a/pkg/common/db/s3/cos/doc.go b/pkg/common/db/s3/cos/doc.go deleted file mode 100644 index 592f90ee2..000000000 --- a/pkg/common/db/s3/cos/doc.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cos // import "github.com/openimsdk/open-im-server/v3/pkg/common/db/s3/cos" diff --git a/pkg/common/db/s3/cos/internal.go b/pkg/common/db/s3/cos/internal.go new file mode 100644 index 000000000..0e58a851c --- /dev/null +++ b/pkg/common/db/s3/cos/internal.go @@ -0,0 +1,13 @@ +package cos + +import ( + "context" + "net/http" + "net/url" + _ "unsafe" + + "github.com/tencentyun/cos-go-sdk-v5" +) + +//go:linkname newRequest github.com/tencentyun/cos-go-sdk-v5.(*Client).newRequest +func newRequest(c *cos.Client, ctx context.Context, baseURL *url.URL, uri, method string, body interface{}, optQuery interface{}, optHeader interface{}) (req *http.Request, err error) diff --git a/pkg/common/db/s3/minio/doc.go b/pkg/common/db/s3/minio/doc.go deleted file mode 100644 index 42e898c15..000000000 --- a/pkg/common/db/s3/minio/doc.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package minio // import "github.com/openimsdk/open-im-server/v3/pkg/common/db/s3/minio" diff --git a/pkg/common/db/s3/minio/internal.go b/pkg/common/db/s3/minio/internal.go new file mode 100644 index 000000000..41129ce31 --- /dev/null +++ b/pkg/common/db/s3/minio/internal.go @@ -0,0 +1,11 @@ +package minio + +import ( + "net/url" + _ "unsafe" + + "github.com/minio/minio-go/v7" +) + +//go:linkname makeTargetURL github.com/minio/minio-go/v7.(*Client).makeTargetURL +func makeTargetURL(client *minio.Client, bucketName, objectName, bucketLocation string, isVirtualHostStyle bool, queryValues url.Values) (*url.URL, error) diff --git a/pkg/common/db/s3/minio/minio.go b/pkg/common/db/s3/minio/minio.go index 937b9f78a..037c933bb 100644 --- a/pkg/common/db/s3/minio/minio.go +++ b/pkg/common/db/s3/minio/minio.go @@ -139,6 +139,15 @@ func (m *Minio) initMinio(ctx context.Context) error { return fmt.Errorf("make bucket error: %w", err) } } + if conf.PublicRead { + policy := fmt.Sprintf( + `{"Version": "2012-10-17","Statement": [{"Action": ["s3:GetObject","s3:PutObject"],"Effect": "Allow","Principal": {"AWS": ["*"]},"Resource": ["arn:aws:s3:::%s/*"],"Sid": ""}]}`, + conf.Bucket, + ) + if err := m.core.Client.SetBucketPolicy(ctx, conf.Bucket, policy); err != nil { + return err + } + } m.location, err = m.core.Client.GetBucketLocation(ctx, conf.Bucket) if err != nil { return err @@ -375,7 +384,15 @@ func (m *Minio) presignedGetObject(ctx context.Context, name string, expire time } else if expire < time.Second { expire = time.Second } - rawURL, err := m.sign.PresignedGetObject(ctx, m.bucket, name, expire, query) + var ( + rawURL *url.URL + err error + ) + if config.Config.Object.Minio.PublicRead { + rawURL, err = makeTargetURL(m.sign, m.bucket, name, m.location, false, query) + } else { + rawURL, err = m.sign.PresignedGetObject(ctx, m.bucket, name, expire, query) + } if err != nil { return "", err } diff --git a/pkg/common/db/s3/oss/doc.go b/pkg/common/db/s3/oss/doc.go deleted file mode 100644 index d2e2c91af..000000000 --- a/pkg/common/db/s3/oss/doc.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package oss // import "github.com/openimsdk/open-im-server/v3/pkg/common/db/s3/oss" diff --git a/pkg/common/db/s3/oss/sign.go b/pkg/common/db/s3/oss/internal.go similarity index 56% rename from pkg/common/db/s3/oss/sign.go rename to pkg/common/db/s3/oss/internal.go index 1bff18f4d..4ca1acc47 100644 --- a/pkg/common/db/s3/oss/sign.go +++ b/pkg/common/db/s3/oss/internal.go @@ -16,10 +16,24 @@ package oss import ( "net/http" + "net/url" _ "unsafe" "github.com/aliyun/aliyun-oss-go-sdk/oss" ) -//go:linkname ossSignHeader github.com/aliyun/aliyun-oss-go-sdk/oss.(*Conn).signHeader -func ossSignHeader(c *oss.Conn, req *http.Request, canonicalizedResource string) +//go:linkname signHeader github.com/aliyun/aliyun-oss-go-sdk/oss.Conn.signHeader +func signHeader(c oss.Conn, req *http.Request, canonicalizedResource string) + +//go:linkname getURLParams github.com/aliyun/aliyun-oss-go-sdk/oss.Conn.getURLParams +func getURLParams(c oss.Conn, params map[string]interface{}) string + +//go:linkname getURL github.com/aliyun/aliyun-oss-go-sdk/oss.urlMaker.getURL +func getURL(um urlMaker, bucket, object, params string) *url.URL + +type urlMaker struct { + Scheme string + NetLoc string + Type int + IsProxy bool +} diff --git a/pkg/common/db/s3/oss/oss.go b/pkg/common/db/s3/oss/oss.go index 384ce8093..6a728127b 100644 --- a/pkg/common/db/s3/oss/oss.go +++ b/pkg/common/db/s3/oss/oss.go @@ -20,6 +20,7 @@ import ( "fmt" "net/http" "net/url" + "reflect" "strconv" "strings" "time" @@ -69,6 +70,7 @@ func NewOSS() (s3.Interface, error) { bucketURL: conf.BucketURL, bucket: bucket, credentials: client.Config.GetCredentials(), + um: *(*urlMaker)(reflect.ValueOf(bucket.Client.Conn).Elem().FieldByName("url").UnsafePointer()), }, nil } @@ -76,6 +78,7 @@ type OSS struct { bucketURL string bucket *oss.Bucket credentials oss.Credentials + um urlMaker } func (o *OSS) Engine() string { @@ -163,7 +166,7 @@ func (o *OSS) AuthSign(ctx context.Context, uploadID string, name string, expire request.Header.Set(oss.HTTPHeaderHost, request.Host) request.Header.Set(oss.HTTPHeaderDate, now) request.Header.Set(oss.HttpHeaderOssDate, now) - ossSignHeader(o.bucket.Client.Conn, request, fmt.Sprintf(`/%s/%s?partNumber=%d&uploadId=%s`, o.bucket.BucketName, name, partNumber, uploadID)) + signHeader(*o.bucket.Client.Conn, request, fmt.Sprintf(`/%s/%s?partNumber=%d&uploadId=%s`, o.bucket.BucketName, name, partNumber, uploadID)) delete(request.Header, oss.HTTPHeaderDate) result.Parts[i] = s3.SignPart{ PartNumber: partNumber, @@ -272,6 +275,7 @@ func (o *OSS) ListUploadedParts(ctx context.Context, uploadID string, name strin } func (o *OSS) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) { + publicRead := config.Config.Object.Oss.PublicRead var opts []oss.Option if opt != nil { if opt.Image != nil { @@ -299,11 +303,13 @@ func (o *OSS) AccessURL(ctx context.Context, name string, expire time.Duration, process += ",format," + format opts = append(opts, oss.Process(process)) } - if opt.ContentType != "" { - opts = append(opts, oss.ResponseContentType(opt.ContentType)) - } - if opt.Filename != "" { - opts = append(opts, oss.ResponseContentDisposition(`attachment; filename=`+strconv.Quote(opt.Filename))) + if !publicRead { + if opt.ContentType != "" { + opts = append(opts, oss.ResponseContentType(opt.ContentType)) + } + if opt.Filename != "" { + opts = append(opts, oss.ResponseContentDisposition(`attachment; filename=`+strconv.Quote(opt.Filename))) + } } } if expire <= 0 { @@ -311,5 +317,13 @@ func (o *OSS) AccessURL(ctx context.Context, name string, expire time.Duration, } else if expire < time.Second { expire = time.Second } - return o.bucket.SignURL(name, http.MethodGet, int64(expire/time.Second), opts...) + if !publicRead { + return o.bucket.SignURL(name, http.MethodGet, int64(expire/time.Second), opts...) + } + rawParams, err := oss.GetRawParams(opts) + if err != nil { + return "", err + } + params := getURLParams(*o.bucket.Client.Conn, rawParams) + return getURL(o.um, o.bucket.BucketName, name, params).String(), nil } diff --git a/scripts/install/environment.sh b/scripts/install/environment.sh index b17570be5..7984e23bd 100755 --- a/scripts/install/environment.sh +++ b/scripts/install/environment.sh @@ -186,17 +186,21 @@ def "MINIO_ACCESS_KEY" "${USER}" def "MINIO_SECRET_KEY" "${PASSWORD}" # MinIO的密钥 def "MINIO_SESSION_TOKEN" # MinIO的会话令牌 readonly MINIO_SIGN_ENDPOINT=${MINIO_SIGN_ENDPOINT:-"http://${IP}:${MINIO_PORT}"} # signEndpoint为minio公网地址 +def "MINIO_PUBLIC_READ" "false" # 公有读 + # 腾讯云COS的存储桶URL def "COS_BUCKET_URL" "https://temp-1252357374.cos.ap-chengdu.myqcloud.com" def "COS_SECRET_ID" # 腾讯云COS的密钥ID def "COS_SECRET_KEY" # 腾讯云COS的密钥 def "COS_SESSION_TOKEN" # 腾讯云COS的会话令牌 +def "COS_PUBLIC_READ" "false" # 公有读 def "OSS_ENDPOINT" "https://oss-cn-chengdu.aliyuncs.com" # 阿里云OSS的端点URL def "OSS_BUCKET" "demo-9999999" # 阿里云OSS的存储桶名称 def "OSS_BUCKET_URL" "https://demo-9999999.oss-cn-chengdu.aliyuncs.com" # 阿里云OSS的存储桶URL def "OSS_ACCESS_KEY_ID" # 阿里云OSS的访问密钥ID def "OSS_ACCESS_KEY_SECRET" # 阿里云OSS的密钥 def "OSS_SESSION_TOKEN" # 阿里云OSS的会话令牌 +def "OSS_PUBLIC_READ" "false" # 公有读 ###################### Redis 配置信息 ###################### def "REDIS_PORT" "16379" # Redis的端口