blackteay/open-im-server
1
0
Fork 0
mirror of https://github.com/openimsdk/open-im-server.git synced 2025-04-05 20:11:14 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: s3 public read (#1080)

Browse Source 
* fix: repeated modification session notification

* fix: repeated modification session notification

* fix: jpush return a nil pointer panic

* fix: push redis pkg

* fix: OANotification

* feat: add rpc GetConversationNeedOfflinePushUserIDs

* update pkg

* cicd: robot automated Change

* offlinePushMsg

* conversation

* conversation

* cicd: robot automated Change

* conversation

* cicd: robot automated Change

* conversation

* url 2 im s3

* url 2 im s3

* cicd: robot automated Change

* url 2 im s3

* s3 public read

* cicd: robot automated Change

* s3 public read

* cicd: robot automated Change

* s3 public read

* s3 public read

* s3 public read

* s3 public read

* s3 public read

* cicd: robot automated Change

* s3 public read

* s3 public read

* fix: SendMsg api

* config scripts

* config scripts

---------

Co-authored-by: withchao <withchao@users.noreply.github.com>
This commit is contained in:
withchao 2023-09-19 15:23:46 +08:00 committed by GitHub
parent bba662f404
commit 5c31d12253
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 108 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
config/config.yaml
View File

@ -140,11 +140,13 @@ object:
secretAccessKey: "openIM123"
sessionToken: ''
signEndpoint: "http://127.0.0.1:10005"
publicRead: false
cos:
bucketURL: https://temp-1252357374.cos.ap-chengdu.myqcloud.com
secretID: ''
secretKey: ''
sessionToken: ''
publicRead: false
oss:
endpoint: "https://oss-cn-chengdu.aliyuncs.com"
bucket: "demo-9999999"
@ -152,6 +154,7 @@ object:
accessKeyID: ''
accessKeySecret: ''
sessionToken: ''
publicRead: false
###################### RPC Port Configuration ######################

4
deployments/templates/openim.yaml
View File

@ -140,11 +140,13 @@ object:
secretAccessKey: "${MINIO_SECRET_KEY}"
sessionToken: ${MINIO_SESSION_TOKEN}
signEndpoint: "${MINIO_SIGN_ENDPOINT}"
publicRead: ${MINIO_PUBLIC_READ}
cos:
bucketURL: ${COS_BUCKET_URL}
secretID: ${COS_SECRET_ID}
secretKey: ${COS_SECRET_KEY}
sessionToken: ${COS_SESSION_TOKEN}
publicRead: ${COS_PUBLIC_READ}
oss:
endpoint: "${OSS_ENDPOINT}"
bucket: "${OSS_BUCKET}"
@ -152,7 +154,7 @@ object:
accessKeyID: ${OSS_ACCESS_KEY_ID}
accessKeySecret: ${OSS_ACCESS_KEY_SECRET}
sessionToken: ${OSS_SESSION_TOKEN}
publicRead: ${OSS_PUBLIC_READ}
###################### RPC Port Configuration ######################
# RPC service ports

2
internal/api/msg.go
View File

@ -58,7 +58,7 @@ func (m MessageApi) newUserSendMsgReq(c *gin.Context, params *apistruct.SendMsg)
options := make(map[string]bool, 5)
switch params.ContentType {
case constant.Text:
newContent = params.Content["text"].(string)
fallthrough
case constant.Picture:
fallthrough
case constant.Custom:

3
pkg/common/config/config.go
View File

@ -128,12 +128,14 @@ type configStruct struct {
SecretAccessKey string `yaml:"secretAccessKey"`
SessionToken string `yaml:"sessionToken"`
SignEndpoint string `yaml:"signEndpoint"`
PublicRead bool `yaml:"publicRead"`
} `yaml:"minio"`
Cos struct {
BucketURL string `yaml:"bucketURL"`
SecretID string `yaml:"secretID"`
SecretKey string `yaml:"secretKey"`
SessionToken string `yaml:"sessionToken"`
PublicRead bool `yaml:"publicRead"`
} `yaml:"cos"`
Oss struct {
Endpoint string `yaml:"endpoint"`
@ -142,6 +144,7 @@ type configStruct struct {
AccessKeyID string `yaml:"accessKeyID"`
AccessKeySecret string `yaml:"accessKeySecret"`
SessionToken string `yaml:"sessionToken"`
PublicRead bool `yaml:"publicRead"`
} `yaml:"oss"`
} `yaml:"object"`

20
pkg/common/db/s3/cos/cos.go
View File

@ -288,7 +288,7 @@ func (c *Cos) AccessURL(ctx context.Context, name string, expire time.Duration,
style = append(style, "format/"+opt.Image.Format)
}
if len(style) > 0 {
imageMogr = "&imageMogr2/thumbnail/" + strings.Join(style, "/") + "/ignore-error/1"
imageMogr = "imageMogr2/thumbnail/" + strings.Join(style, "/") + "/ignore-error/1"
}
}
if opt.ContentType != "" {
@ -306,13 +306,23 @@ func (c *Cos) AccessURL(ctx context.Context, name string, expire time.Duration,
} else if expire < time.Second {
expire = time.Second
}
rawURL, err := c.client.Object.GetPresignedURL(ctx, http.MethodGet, name, c.credential.SecretID, c.credential.SecretKey, expire, &option)
rawURL, err := c.getPresignedURL(ctx, name, expire, &option)
if err != nil {
return "", err
}
urlStr := rawURL.String()
if imageMogr != "" {
urlStr += imageMogr
if rawURL.RawQuery == "" {
rawURL.RawQuery = imageMogr
} else {
rawURL.RawQuery = rawURL.RawQuery + "&" + imageMogr
}
}
return urlStr, nil
return rawURL.String(), nil
}
func (c *Cos) getPresignedURL(ctx context.Context, name string, expire time.Duration, opt *cos.PresignedURLOptions) (*url.URL, error) {
if !config.Config.Object.Cos.PublicRead {
return c.client.Object.GetPresignedURL(ctx, http.MethodGet, name, c.credential.SecretID, c.credential.SecretKey, expire, opt)
}
return c.client.Object.GetObjectURL(name), nil
}

15
pkg/common/db/s3/cos/doc.go
View File

@ -1,15 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cos // import "github.com/openimsdk/open-im-server/v3/pkg/common/db/s3/cos"

13
pkg/common/db/s3/cos/internal.go Normal file
View File

@ -0,0 +1,13 @@
package cos
import (
"context"
"net/http"
"net/url"
_ "unsafe"
"github.com/tencentyun/cos-go-sdk-v5"
)
//go:linkname newRequest github.com/tencentyun/cos-go-sdk-v5.(*Client).newRequest
func newRequest(c *cos.Client, ctx context.Context, baseURL *url.URL, uri, method string, body interface{}, optQuery interface{}, optHeader interface{}) (req *http.Request, err error)

15
pkg/common/db/s3/minio/doc.go
View File

@ -1,15 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package minio // import "github.com/openimsdk/open-im-server/v3/pkg/common/db/s3/minio"

11
pkg/common/db/s3/minio/internal.go Normal file
View File

@ -0,0 +1,11 @@
package minio
import (
"net/url"
_ "unsafe"
"github.com/minio/minio-go/v7"
)
//go:linkname makeTargetURL github.com/minio/minio-go/v7.(*Client).makeTargetURL
func makeTargetURL(client *minio.Client, bucketName, objectName, bucketLocation string, isVirtualHostStyle bool, queryValues url.Values) (*url.URL, error)

19
pkg/common/db/s3/minio/minio.go
View File

@ -139,6 +139,15 @@ func (m *Minio) initMinio(ctx context.Context) error {
return fmt.Errorf("make bucket error: %w", err)
}
}
if conf.PublicRead {
policy := fmt.Sprintf(
`{"Version": "2012-10-17","Statement": [{"Action": ["s3:GetObject","s3:PutObject"],"Effect": "Allow","Principal": {"AWS": ["*"]},"Resource": ["arn:aws:s3:::%s/*"],"Sid": ""}]}`,
conf.Bucket,
)
if err := m.core.Client.SetBucketPolicy(ctx, conf.Bucket, policy); err != nil {
return err
}
}
m.location, err = m.core.Client.GetBucketLocation(ctx, conf.Bucket)
if err != nil {
return err
@ -375,7 +384,15 @@ func (m *Minio) presignedGetObject(ctx context.Context, name string, expire time
} else if expire < time.Second {
expire = time.Second
}
rawURL, err := m.sign.PresignedGetObject(ctx, m.bucket, name, expire, query)
var (
rawURL *url.URL
err error
)
if config.Config.Object.Minio.PublicRead {
rawURL, err = makeTargetURL(m.sign, m.bucket, name, m.location, false, query)
} else {
rawURL, err = m.sign.PresignedGetObject(ctx, m.bucket, name, expire, query)
}
if err != nil {
return "", err
}

15
pkg/common/db/s3/oss/doc.go
View File

@ -1,15 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package oss // import "github.com/openimsdk/open-im-server/v3/pkg/common/db/s3/oss"

18
pkg/common/db/s3/oss/sign.go → pkg/common/db/s3/oss/internal.go
View File

@ -16,10 +16,24 @@ package oss
import (
"net/http"
"net/url"
_ "unsafe"
"github.com/aliyun/aliyun-oss-go-sdk/oss"
)
//go:linkname ossSignHeader github.com/aliyun/aliyun-oss-go-sdk/oss.(*Conn).signHeader
func ossSignHeader(c *oss.Conn, req *http.Request, canonicalizedResource string)
//go:linkname signHeader github.com/aliyun/aliyun-oss-go-sdk/oss.Conn.signHeader
func signHeader(c oss.Conn, req *http.Request, canonicalizedResource string)
//go:linkname getURLParams github.com/aliyun/aliyun-oss-go-sdk/oss.Conn.getURLParams
func getURLParams(c oss.Conn, params map[string]interface{}) string
//go:linkname getURL github.com/aliyun/aliyun-oss-go-sdk/oss.urlMaker.getURL
func getURL(um urlMaker, bucket, object, params string) *url.URL
type urlMaker struct {
Scheme string
NetLoc string
Type int
IsProxy bool
}

28
pkg/common/db/s3/oss/oss.go
View File

@ -20,6 +20,7 @@ import (
"fmt"
"net/http"
"net/url"
"reflect"
"strconv"
"strings"
"time"
@ -69,6 +70,7 @@ func NewOSS() (s3.Interface, error) {
bucketURL: conf.BucketURL,
bucket: bucket,
credentials: client.Config.GetCredentials(),
um: *(*urlMaker)(reflect.ValueOf(bucket.Client.Conn).Elem().FieldByName("url").UnsafePointer()),
}, nil
}
@ -76,6 +78,7 @@ type OSS struct {
bucketURL string
bucket *oss.Bucket
credentials oss.Credentials
um urlMaker
}
func (o *OSS) Engine() string {
@ -163,7 +166,7 @@ func (o *OSS) AuthSign(ctx context.Context, uploadID string, name string, expire
request.Header.Set(oss.HTTPHeaderHost, request.Host)
request.Header.Set(oss.HTTPHeaderDate, now)
request.Header.Set(oss.HttpHeaderOssDate, now)
ossSignHeader(o.bucket.Client.Conn, request, fmt.Sprintf(`/%s/%s?partNumber=%d&uploadId=%s`, o.bucket.BucketName, name, partNumber, uploadID))
signHeader(*o.bucket.Client.Conn, request, fmt.Sprintf(`/%s/%s?partNumber=%d&uploadId=%s`, o.bucket.BucketName, name, partNumber, uploadID))
delete(request.Header, oss.HTTPHeaderDate)
result.Parts[i] = s3.SignPart{
PartNumber: partNumber,
@ -272,6 +275,7 @@ func (o *OSS) ListUploadedParts(ctx context.Context, uploadID string, name strin
}
func (o *OSS) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) {
publicRead := config.Config.Object.Oss.PublicRead
var opts []oss.Option
if opt != nil {
if opt.Image != nil {
@ -299,11 +303,13 @@ func (o *OSS) AccessURL(ctx context.Context, name string, expire time.Duration,
process += ",format," + format
opts = append(opts, oss.Process(process))
}
if opt.ContentType != "" {
opts = append(opts, oss.ResponseContentType(opt.ContentType))
}
if opt.Filename != "" {
opts = append(opts, oss.ResponseContentDisposition(`attachment; filename=`+strconv.Quote(opt.Filename)))
if !publicRead {
if opt.ContentType != "" {
opts = append(opts, oss.ResponseContentType(opt.ContentType))
}
if opt.Filename != "" {
opts = append(opts, oss.ResponseContentDisposition(`attachment; filename=`+strconv.Quote(opt.Filename)))
}
}
}
if expire <= 0 {
@ -311,5 +317,13 @@ func (o *OSS) AccessURL(ctx context.Context, name string, expire time.Duration,
} else if expire < time.Second {
expire = time.Second
}
return o.bucket.SignURL(name, http.MethodGet, int64(expire/time.Second), opts...)
if !publicRead {
return o.bucket.SignURL(name, http.MethodGet, int64(expire/time.Second), opts...)
}
rawParams, err := oss.GetRawParams(opts)
if err != nil {
return "", err
}
params := getURLParams(*o.bucket.Client.Conn, rawParams)
return getURL(o.um, o.bucket.BucketName, name, params).String(), nil
}

4
scripts/install/environment.sh
View File

@ -186,17 +186,21 @@ def "MINIO_ACCESS_KEY" "${USER}"
def "MINIO_SECRET_KEY" "${PASSWORD}" # MinIO的密钥
def "MINIO_SESSION_TOKEN" # MinIO的会话令牌
readonly MINIO_SIGN_ENDPOINT=${MINIO_SIGN_ENDPOINT:-"http://${IP}:${MINIO_PORT}"} # signEndpoint为minio公网地址
def "MINIO_PUBLIC_READ" "false" # 公有读
# 腾讯云COS的存储桶URL
def "COS_BUCKET_URL" "https://temp-1252357374.cos.ap-chengdu.myqcloud.com"
def "COS_SECRET_ID" # 腾讯云COS的密钥ID
def "COS_SECRET_KEY" # 腾讯云COS的密钥
def "COS_SESSION_TOKEN" # 腾讯云COS的会话令牌
def "COS_PUBLIC_READ" "false" # 公有读
def "OSS_ENDPOINT" "https://oss-cn-chengdu.aliyuncs.com" # 阿里云OSS的端点URL
def "OSS_BUCKET" "demo-9999999" # 阿里云OSS的存储桶名称
def "OSS_BUCKET_URL" "https://demo-9999999.oss-cn-chengdu.aliyuncs.com" # 阿里云OSS的存储桶URL
def "OSS_ACCESS_KEY_ID" # 阿里云OSS的访问密钥ID
def "OSS_ACCESS_KEY_SECRET" # 阿里云OSS的密钥
def "OSS_SESSION_TOKEN" # 阿里云OSS的会话令牌
def "OSS_PUBLIC_READ" "false" # 公有读
###################### Redis 配置信息 ######################
def "REDIS_PORT" "16379" # Redis的端口