From ddf8b736c3eae885d9ce058b60589d718ea64b4c Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 12 Dec 2024 17:03:34 +0800 Subject: [PATCH 01/42] pb --- internal/rpc/msg/delete.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/internal/rpc/msg/delete.go b/internal/rpc/msg/delete.go index e19bba867..371c50e2e 100644 --- a/internal/rpc/msg/delete.go +++ b/internal/rpc/msg/delete.go @@ -138,9 +138,16 @@ func (m *msgServer) clearConversation(ctx context.Context, conversationIDs []str } isSyncSelf, isSyncOther := m.validateDeleteSyncOpt(deleteSyncOpt) if !isSyncOther { - if err := m.MsgDatabase.SetUserConversationsMinSeqs(ctx, userID, m.getMinSeqs(maxSeqs)); err != nil { + setSeqs := m.getMinSeqs(maxSeqs) + if err := m.MsgDatabase.SetUserConversationsMinSeqs(ctx, userID, setSeqs); err != nil { return err } + ownerUserIDs := []string{userID} + for conversationID, seq := range setSeqs { + if err := m.Conversation.SetConversationMinSeq(ctx, ownerUserIDs, conversationID, seq); err != nil { + return err + } + } // notification 2 self if isSyncSelf { tips := &sdkws.ClearConversationTips{UserID: userID, ConversationIDs: existConversationIDs} From 6f169ec9690f62e8ea8edb697c87bb74cd3c17c3 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 17 Dec 2024 11:58:06 +0800 Subject: [PATCH 02/42] fix: Modifying other fields while setting IsPrivateChat does not take effect --- internal/rpc/conversation/conversation.go | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/internal/rpc/conversation/conversation.go b/internal/rpc/conversation/conversation.go index 7345c965d..8ed1323d5 100644 --- a/internal/rpc/conversation/conversation.go +++ b/internal/rpc/conversation/conversation.go @@ -354,7 +354,15 @@ func (c *conversationServer) SetConversations(ctx context.Context, req *pbconver needUpdateUsersList = append(needUpdateUsersList, userID) } } + if len(m) != 0 && len(needUpdateUsersList) != 0 { + if err := c.conversationDatabase.SetUsersConversationFieldTx(ctx, needUpdateUsersList, &conversation, m); err != nil { + return nil, err + } + for _, v := range needUpdateUsersList { + c.conversationNotificationSender.ConversationChangeNotification(ctx, v, []string{req.Conversation.ConversationID}) + } + } if req.Conversation.IsPrivateChat != nil && req.Conversation.ConversationType != constant.ReadGroupChatType { var conversations []*dbModel.Conversation for _, ownerUserID := range req.UserIDs { @@ -372,16 +380,6 @@ func (c *conversationServer) SetConversations(ctx context.Context, req *pbconver c.conversationNotificationSender.ConversationSetPrivateNotification(ctx, userID, req.Conversation.UserID, req.Conversation.IsPrivateChat.Value, req.Conversation.ConversationID) } - } else { - if len(m) != 0 && len(needUpdateUsersList) != 0 { - if err := c.conversationDatabase.SetUsersConversationFieldTx(ctx, needUpdateUsersList, &conversation, m); err != nil { - return nil, err - } - - for _, v := range needUpdateUsersList { - c.conversationNotificationSender.ConversationChangeNotification(ctx, v, []string{req.Conversation.ConversationID}) - } - } } return &pbconversation.SetConversationsResp{}, nil From d15414d8851c37b938e5d795fd40e91a836ff5f7 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 17 Dec 2024 18:28:16 +0800 Subject: [PATCH 03/42] fix: quote message error revoke --- pkg/common/storage/controller/msg.go | 52 +++++++++++++++++++++++----- 1 file changed, 44 insertions(+), 8 deletions(-) diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index 8d82d8543..3c3cd9671 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -269,7 +269,6 @@ func (db *commonMsgDatabase) getMsgBySeqs(ctx context.Context, userID, conversat } return totalMsgs, nil } - func (db *commonMsgDatabase) handlerDBMsg(ctx context.Context, cache map[int64][]*model.MsgInfoModel, userID, conversationID string, msg *model.MsgInfoModel) { if msg.IsRead { msg.Msg.IsRead = true @@ -280,16 +279,53 @@ func (db *commonMsgDatabase) handlerDBMsg(ctx context.Context, cache map[int64][ if msg.Msg.Content == "" { return } + type MsgData struct { + SendID string `json:"sendID"` + RecvID string `json:"recvID"` + GroupID string `json:"groupID"` + ClientMsgID string `json:"clientMsgID"` + ServerMsgID string `json:"serverMsgID"` + SenderPlatformID int32 `json:"senderPlatformID"` + SenderNickname string `json:"senderNickname"` + SenderFaceURL string `json:"senderFaceURL"` + SessionType int32 `json:"sessionType"` + MsgFrom int32 `json:"msgFrom"` + ContentType int32 `json:"contentType"` + Content string `json:"content"` + Seq int64 `json:"seq"` + SendTime int64 `json:"sendTime"` + CreateTime int64 `json:"createTime"` + Status int32 `json:"status"` + IsRead bool `json:"isRead"` + Options map[string]bool `json:"options,omitempty"` + OfflinePushInfo *sdkws.OfflinePushInfo `json:"offlinePushInfo"` + AtUserIDList []string `json:"atUserIDList"` + AttachedInfo string `json:"attachedInfo"` + Ex string `json:"ex"` + KeyVersion int32 `json:"keyVersion"` + DstUserIDs []string `json:"dstUserIDs"` + } var quoteMsg struct { Text string `json:"text,omitempty"` - QuoteMessage *sdkws.MsgData `json:"quoteMessage,omitempty"` + QuoteMessage *MsgData `json:"quoteMessage,omitempty"` MessageEntityList json.RawMessage `json:"messageEntityList,omitempty"` } if err := json.Unmarshal([]byte(msg.Msg.Content), "eMsg); err != nil { log.ZError(ctx, "json.Unmarshal", err) return } - if quoteMsg.QuoteMessage == nil || quoteMsg.QuoteMessage.ContentType == constant.MsgRevokeNotification { + if quoteMsg.QuoteMessage == nil || quoteMsg.QuoteMessage.Content == "" { + return + } + if quoteMsg.QuoteMessage.Content == "e30=" { + quoteMsg.QuoteMessage.Content = "{}" + data, err := json.Marshal("eMsg) + if err != nil { + return + } + msg.Msg.Content = string(data) + } + if quoteMsg.QuoteMessage.Seq <= 0 && quoteMsg.QuoteMessage.ContentType == constant.MsgRevokeNotification { return } var msgs []*model.MsgInfoModel @@ -311,9 +347,9 @@ func (db *commonMsgDatabase) handlerDBMsg(ctx context.Context, cache map[int64][ } quoteMsg.QuoteMessage.ContentType = constant.MsgRevokeNotification if len(msgs) > 0 { - quoteMsg.QuoteMessage.Content = []byte(msgs[0].Msg.Content) + quoteMsg.QuoteMessage.Content = msgs[0].Msg.Content } else { - quoteMsg.QuoteMessage.Content = []byte("{}") + quoteMsg.QuoteMessage.Content = "{}" } data, err := json.Marshal("eMsg) if err != nil { @@ -321,9 +357,9 @@ func (db *commonMsgDatabase) handlerDBMsg(ctx context.Context, cache map[int64][ return } msg.Msg.Content = string(data) - if _, err := db.msgDocDatabase.UpdateMsg(ctx, db.msgTable.GetDocID(conversationID, msg.Msg.Seq), db.msgTable.GetMsgIndex(msg.Msg.Seq), "msg", msg.Msg); err != nil { - log.ZError(ctx, "UpdateMsgContent", err) - } + //if _, err := db.msgDocDatabase.UpdateMsg(ctx, db.msgTable.GetDocID(conversationID, msg.Msg.Seq), db.msgTable.GetMsgIndex(msg.Msg.Seq), "msg", msg.Msg); err != nil { + // log.ZError(ctx, "UpdateMsgContent", err) + //} } func (db *commonMsgDatabase) findMsgInfoBySeq(ctx context.Context, userID, docID string, conversationID string, seqs []int64) (totalMsgs []*model.MsgInfoModel, err error) { From f03cbec6532cfadface6a55b365d36df879e8069 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 18 Dec 2024 17:41:49 +0800 Subject: [PATCH 04/42] refactoring scheduled tasks --- internal/rpc/third/s3.go | 93 +++------------- internal/rpc/third/third.go | 14 +-- internal/tools/cron_task.go | 125 ++++++++-------------- internal/tools/msg.go | 23 ++++ internal/tools/s3.go | 34 ++++++ internal/tools/user_msg.go | 31 ++++++ pkg/common/storage/controller/s3.go | 20 +--- pkg/common/storage/database/mgo/object.go | 29 +++-- pkg/common/storage/database/object.go | 6 +- 9 files changed, 175 insertions(+), 200 deletions(-) create mode 100644 internal/tools/msg.go create mode 100644 internal/tools/s3.go create mode 100644 internal/tools/user_msg.go diff --git a/internal/rpc/third/s3.go b/internal/rpc/third/s3.go index b4b064d7f..d4146fdf4 100644 --- a/internal/rpc/third/s3.go +++ b/internal/rpc/third/s3.go @@ -23,13 +23,9 @@ import ( "strconv" "time" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "go.mongodb.org/mongo-driver/mongo" - "github.com/google/uuid" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" - "github.com/openimsdk/protocol/sdkws" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/protocol/third" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" @@ -288,87 +284,30 @@ func (t *thirdServer) apiAddress(prefix, name string) string { } func (t *thirdServer) DeleteOutdatedData(ctx context.Context, req *third.DeleteOutdatedDataReq) (*third.DeleteOutdatedDataResp, error) { - var conf config.Third + engine := t.config.RpcConfig.Object.Enable expireTime := time.UnixMilli(req.ExpireTime) - - findPagination := &sdkws.RequestPagination{ - PageNumber: 1, - ShowNumber: 500, - } - // Find all expired data in S3 database - total, models, err := t.s3dataBase.FindNeedDeleteObjectByDB(ctx, expireTime, req.ObjectGroup, findPagination) - if err != nil && errs.Unwrap(err) != mongo.ErrNoDocuments { - return nil, errs.Wrap(err) + models, err := t.s3dataBase.FindExpirationObject(ctx, engine, expireTime, req.ObjectGroup, int64(req.Count)) + if err != nil { + return nil, err } - - if total == 0 { - log.ZDebug(ctx, "Not have OutdatedData", "delete Total", total) - return &third.DeleteOutdatedDataResp{Count: int32(total)}, nil - } - - needDelObjectKeys := make([]string, len(models)) - for _, model := range models { - needDelObjectKeys = append(needDelObjectKeys, model.Key) - } - - // Remove duplicate keys, have the same key use in different models - needDelObjectKeys = datautil.Distinct(needDelObjectKeys) - - for _, key := range needDelObjectKeys { - // Find all models by key - keyModels, err := t.s3dataBase.FindModelsByKey(ctx, key) - if err != nil && errs.Unwrap(err) != mongo.ErrNoDocuments { + if len(models) > 0 { + names := datautil.Batch(func(o *model.Object) string { + return o.Name + }, models) + if err := t.s3dataBase.DeleteSpecifiedData(ctx, engine, names); err != nil { return nil, errs.Wrap(err) } - - // check keyModels, if all keyModels. - needDelKey := true // Default can delete - for _, keymodel := range keyModels { - // If group is empty or CreateTime is after expireTime, can't delete this key - if keymodel.Group == "" || keymodel.CreateTime.After(expireTime) { - needDelKey = false - break - } + if err := t.s3dataBase.DelS3Key(ctx, engine, names...); err != nil { + return nil, err } - - // If this object is not referenced by not expire data, delete it - if needDelKey && t.minio != nil { - // If have a thumbnail, delete it - thumbnailKey, _ := t.getMinioImageThumbnailKey(ctx, key) - if thumbnailKey != "" { - err := t.s3dataBase.DeleteObject(ctx, thumbnailKey) - if err != nil { - log.ZWarn(ctx, "Delete thumbnail object is error:", errs.Wrap(err), "thumbnailKey", thumbnailKey) - } - } - - // Delete object - err = t.s3dataBase.DeleteObject(ctx, key) - if err != nil { - log.ZWarn(ctx, "Delete object is error", errs.Wrap(err), "object key", key) - } - - // Delete cache key - err = t.s3dataBase.DelS3Key(ctx, conf.Object.Enable, key) - if err != nil { - log.ZWarn(ctx, "Delete cache key is error:", errs.Wrap(err), "cache S3 key:", key) + for _, object := range models { + if err := t.s3.DeleteObject(ctx, object.Key); err != nil { + return nil, err } } } - - // handle delete data in S3 database - for _, model := range models { - // Delete all expired data row in S3 database - err := t.s3dataBase.DeleteSpecifiedData(ctx, model.Engine, model.Name) - if err != nil { - return nil, errs.Wrap(err) - } - } - - log.ZDebug(ctx, "DeleteOutdatedData", "delete Total", total) - - return &third.DeleteOutdatedDataResp{Count: int32(total)}, nil + return &third.DeleteOutdatedDataResp{Count: int32(len(models))}, nil } type FormDataMate struct { diff --git a/internal/rpc/third/third.go b/internal/rpc/third/third.go index 77e6d459f..dc964fdb1 100644 --- a/internal/rpc/third/third.go +++ b/internal/rpc/third/third.go @@ -44,7 +44,7 @@ type thirdServer struct { s3dataBase controller.S3Database defaultExpire time.Duration config *Config - minio *minio.Minio + s3 s3.Interface } type Config struct { @@ -79,13 +79,11 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg // Select the oss method according to the profile policy enable := config.RpcConfig.Object.Enable var ( - o s3.Interface - minioCli *minio.Minio + o s3.Interface ) switch enable { case "minio": - minioCli, err = minio.NewMinio(ctx, redis.NewMinioCache(rdb), *config.MinioConfig.Build()) - o = minioCli + o, err = minio.NewMinio(ctx, redis.NewMinioCache(rdb), *config.MinioConfig.Build()) case "cos": o, err = cos.NewCos(*config.RpcConfig.Object.Cos.Build()) case "oss": @@ -106,15 +104,11 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg s3dataBase: controller.NewS3Database(rdb, o, s3db), defaultExpire: time.Hour * 24 * 7, config: config, - minio: minioCli, + s3: o, }) return nil } -func (t *thirdServer) getMinioImageThumbnailKey(ctx context.Context, name string) (string, error) { - return t.minio.GetImageThumbnailKey(ctx, name) -} - func (t *thirdServer) FcmUpdateToken(ctx context.Context, req *third.FcmUpdateTokenReq) (resp *third.FcmUpdateTokenResp, err error) { err = t.thirdDatabase.FcmUpdateToken(ctx, req.Account, int(req.PlatformID), req.FcmToken, req.ExpireTime) if err != nil { diff --git a/internal/tools/cron_task.go b/internal/tools/cron_task.go index 2fe7d0e39..049a6199c 100644 --- a/internal/tools/cron_task.go +++ b/internal/tools/cron_task.go @@ -2,10 +2,6 @@ package tools import ( "context" - "fmt" - "os" - "time" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" pbconversation "github.com/openimsdk/protocol/conversation" @@ -60,87 +56,58 @@ func Start(ctx context.Context, config *CronTaskConfig) error { return err } - msgClient := msg.NewMsgClient(msgConn) - conversationClient := pbconversation.NewConversationClient(conversationConn) - thirdClient := third.NewThirdClient(thirdConn) - - crontab := cron.New() - - // scheduled hard delete outdated Msgs in specific time. - destructMsgsFunc := func() { - now := time.Now() - deltime := now.Add(-time.Hour * 24 * time.Duration(config.CronTask.RetainChatRecords)) - ctx := mcontext.SetOperationID(ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), deltime.UnixMilli())) - log.ZDebug(ctx, "Destruct chat records", "deltime", deltime, "timestamp", deltime.UnixMilli()) - - if _, err := msgClient.DestructMsgs(ctx, &msg.DestructMsgsReq{Timestamp: deltime.UnixMilli()}); err != nil { - log.ZError(ctx, "cron destruct chat records failed", err, "deltime", deltime, "cont", time.Since(now)) - return - } - log.ZDebug(ctx, "cron destruct chat records success", "deltime", deltime, "cont", time.Since(now)) - } - if _, err := crontab.AddFunc(config.CronTask.CronExecuteTime, destructMsgsFunc); err != nil { - return errs.Wrap(err) + srv := &cronServer{ + ctx: ctx, + config: config, + cron: cron.New(), + msgClient: msg.NewMsgClient(msgConn), + conversationClient: pbconversation.NewConversationClient(conversationConn), + thirdClient: third.NewThirdClient(thirdConn), } - // scheduled soft delete outdated Msgs in specific time when user set `is_msg_destruct` feature. - clearMsgFunc := func() { - now := time.Now() - ctx := mcontext.SetOperationID(ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), now.UnixMilli())) - log.ZDebug(ctx, "clear msg cron start", "now", now) - - conversations, err := conversationClient.GetConversationsNeedClearMsg(ctx, &pbconversation.GetConversationsNeedClearMsgReq{}) - if err != nil { - log.ZError(ctx, "Get conversation need Destruct msgs failed.", err) - return - } - - _, err = msgClient.ClearMsg(ctx, &msg.ClearMsgReq{Conversations: conversations.Conversations}) - if err != nil { - log.ZError(ctx, "Clear Msg failed.", err) - return - } - - log.ZDebug(ctx, "clear msg cron task completed", "cont", time.Since(now)) + if err := srv.registerClearS3(); err != nil { + return err } - if _, err := crontab.AddFunc(config.CronTask.CronExecuteTime, clearMsgFunc); err != nil { - return errs.Wrap(err) + if err := srv.registerDeleteMsg(); err != nil { + return err } - - // scheduled delete outdated file Objects and their datas in specific time. - deleteObjectFunc := func() { - now := time.Now() - executeNum := 5 - // number of pagination. if need modify, need update value in third.DeleteOutdatedData - pageShowNumber := 500 - deleteTime := now.Add(-time.Hour * 24 * time.Duration(config.CronTask.FileExpireTime)) - ctx := mcontext.SetOperationID(ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), deleteTime.UnixMilli())) - log.ZDebug(ctx, "deleteoutDatedData", "deletetime", deleteTime, "timestamp", deleteTime.UnixMilli()) - - if len(config.CronTask.DeleteObjectType) == 0 { - log.ZDebug(ctx, "cron deleteoutDatedData not type need delete", "deletetime", deleteTime, "DeleteObjectType", config.CronTask.DeleteObjectType, "cont", time.Since(now)) - return - } - - for i := 0; i < executeNum; i++ { - resp, err := thirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: deleteTime.UnixMilli(), ObjectGroup: config.CronTask.DeleteObjectType}) - if err != nil { - log.ZError(ctx, "cron deleteoutDatedData failed", err, "deleteTime", deleteTime, "cont", time.Since(now)) - return - } - if resp.Count == 0 || resp.Count < int32(pageShowNumber) { - break - } - } - - log.ZDebug(ctx, "cron deleteoutDatedData success", "deltime", deleteTime, "cont", time.Since(now)) + if err := srv.registerClearUserMsg(); err != nil { + return err } - if _, err := crontab.AddFunc(config.CronTask.CronExecuteTime, deleteObjectFunc); err != nil { - return errs.Wrap(err) - } - log.ZDebug(ctx, "start cron task", "CronExecuteTime", config.CronTask.CronExecuteTime) - crontab.Start() + srv.cron.Start() <-ctx.Done() return nil } + +type cronServer struct { + ctx context.Context + config *CronTaskConfig + cron *cron.Cron + msgClient msg.MsgClient + conversationClient pbconversation.ConversationClient + thirdClient third.ThirdClient +} + +func (c *cronServer) registerClearS3() error { + if c.config.CronTask.FileExpireTime <= 0 || len(c.config.CronTask.DeleteObjectType) == 0 { + log.ZInfo(c.ctx, "disable scheduled cleanup of s3", "fileExpireTime", c.config.CronTask.FileExpireTime, "deleteObjectType", c.config.CronTask.DeleteObjectType) + return nil + } + _, err := c.cron.AddFunc(c.config.CronTask.CronExecuteTime, c.clearS3) + return errs.WrapMsg(err, "failed to register clear s3 cron task") +} + +func (c *cronServer) registerDeleteMsg() error { + if c.config.CronTask.RetainChatRecords <= 0 { + log.ZInfo(c.ctx, "disable scheduled cleanup of chat records", "retainChatRecords", c.config.CronTask.RetainChatRecords) + return nil + } + _, err := c.cron.AddFunc(c.config.CronTask.CronExecuteTime, c.deleteMsg) + return errs.WrapMsg(err, "failed to register delete msg cron task") +} + +func (c *cronServer) registerClearUserMsg() error { + _, err := c.cron.AddFunc(c.config.CronTask.CronExecuteTime, c.clearUserMsg) + return errs.WrapMsg(err, "failed to register clear user msg cron task") +} diff --git a/internal/tools/msg.go b/internal/tools/msg.go new file mode 100644 index 000000000..1149b1f29 --- /dev/null +++ b/internal/tools/msg.go @@ -0,0 +1,23 @@ +package tools + +import ( + "fmt" + "github.com/openimsdk/protocol/msg" + "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/mcontext" + "os" + "time" +) + +func (c *cronServer) deleteMsg() { + now := time.Now() + deltime := now.Add(-time.Hour * 24 * time.Duration(c.config.CronTask.RetainChatRecords)) + ctx := mcontext.SetOperationID(c.ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), deltime.UnixMilli())) + log.ZDebug(ctx, "Destruct chat records", "deltime", deltime, "timestamp", deltime.UnixMilli()) + + if _, err := c.msgClient.DestructMsgs(ctx, &msg.DestructMsgsReq{Timestamp: deltime.UnixMilli()}); err != nil { + log.ZError(ctx, "cron destruct chat records failed", err, "deltime", deltime, "cont", time.Since(now)) + return + } + log.ZDebug(ctx, "cron destruct chat records success", "deltime", deltime, "cont", time.Since(now)) +} diff --git a/internal/tools/s3.go b/internal/tools/s3.go new file mode 100644 index 000000000..38d8badc6 --- /dev/null +++ b/internal/tools/s3.go @@ -0,0 +1,34 @@ +package tools + +import ( + "fmt" + "github.com/openimsdk/protocol/third" + "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/mcontext" + "os" + "time" +) + +func (c *cronServer) clearS3() { + start := time.Now() + executeNum := 10 + // number of pagination. if need modify, need update value in third.DeleteOutdatedData + pageShowNumber := 500 + deleteTime := start.Add(-time.Hour * 24 * time.Duration(c.config.CronTask.FileExpireTime)) + operationID := fmt.Sprintf("cron_%d_%d", os.Getpid(), deleteTime.UnixMilli()) + ctx := mcontext.SetOperationID(c.ctx, operationID) + log.ZDebug(ctx, "deleteoutDatedData", "deletetime", deleteTime, "timestamp", deleteTime.UnixMilli()) + for i := 1; i <= executeNum; i++ { + ctx := mcontext.SetOperationID(c.ctx, fmt.Sprintf("%s_%d", operationID, i)) + resp, err := c.thirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: deleteTime.UnixMilli(), ObjectGroup: c.config.CronTask.DeleteObjectType, Count: int32(pageShowNumber)}) + if err != nil { + log.ZError(ctx, "cron deleteoutDatedData failed", err, "deleteTime", deleteTime, "cont", time.Since(start)) + return + } + if resp.Count < int32(pageShowNumber) { + break + } + } + + log.ZDebug(ctx, "cron deleteoutDatedData success", "deltime", deleteTime, "cont", time.Since(start)) +} diff --git a/internal/tools/user_msg.go b/internal/tools/user_msg.go new file mode 100644 index 000000000..9dfaf58c3 --- /dev/null +++ b/internal/tools/user_msg.go @@ -0,0 +1,31 @@ +package tools + +import ( + "fmt" + pbconversation "github.com/openimsdk/protocol/conversation" + "github.com/openimsdk/protocol/msg" + "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/mcontext" + "os" + "time" +) + +func (c *cronServer) clearUserMsg() { + now := time.Now() + ctx := mcontext.SetOperationID(c.ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), now.UnixMilli())) + log.ZDebug(ctx, "clear msg cron start", "now", now) + + conversations, err := c.conversationClient.GetConversationsNeedClearMsg(ctx, &pbconversation.GetConversationsNeedClearMsgReq{}) + if err != nil { + log.ZError(ctx, "Get conversation need Destruct msgs failed.", err) + return + } + + _, err = c.msgClient.ClearMsg(ctx, &msg.ClearMsgReq{Conversations: conversations.Conversations}) + if err != nil { + log.ZError(ctx, "Clear Msg failed.", err) + return + } + + log.ZDebug(ctx, "clear msg cron task completed", "cont", time.Since(now)) +} diff --git a/pkg/common/storage/controller/s3.go b/pkg/common/storage/controller/s3.go index 4e5ad18b6..6a0f11b1c 100644 --- a/pkg/common/storage/controller/s3.go +++ b/pkg/common/storage/controller/s3.go @@ -24,7 +24,6 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" - "github.com/openimsdk/tools/db/pagination" "github.com/openimsdk/tools/s3" "github.com/openimsdk/tools/s3/cont" "github.com/redis/go-redis/v9" @@ -40,10 +39,8 @@ type S3Database interface { SetObject(ctx context.Context, info *model.Object) error StatObject(ctx context.Context, name string) (*s3.ObjectInfo, error) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) - FindNeedDeleteObjectByDB(ctx context.Context, duration time.Time, needDelType []string, pagination pagination.Pagination) (total int64, objects []*model.Object, err error) - DeleteObject(ctx context.Context, name string) error - DeleteSpecifiedData(ctx context.Context, engine string, name string) error - FindModelsByKey(ctx context.Context, key string) (objects []*model.Object, err error) + FindExpirationObject(ctx context.Context, engine string, expiration time.Time, needDelType []string, count int64) ([]*model.Object, error) + DeleteSpecifiedData(ctx context.Context, engine string, name []string) error DelS3Key(ctx context.Context, engine string, keys ...string) error } @@ -120,21 +117,14 @@ func (s *s3Database) StatObject(ctx context.Context, name string) (*s3.ObjectInf func (s *s3Database) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) { return s.s3.FormData(ctx, name, size, contentType, duration) } -func (s *s3Database) FindNeedDeleteObjectByDB(ctx context.Context, duration time.Time, needDelType []string, pagination pagination.Pagination) (total int64, objects []*model.Object, err error) { - return s.db.FindNeedDeleteObjectByDB(ctx, duration, needDelType, pagination) +func (s *s3Database) FindExpirationObject(ctx context.Context, engine string, expiration time.Time, needDelType []string, count int64) ([]*model.Object, error) { + return s.db.FindExpirationObject(ctx, engine, expiration, needDelType, count) } -func (s *s3Database) DeleteObject(ctx context.Context, name string) error { - return s.s3.DeleteObject(ctx, name) -} -func (s *s3Database) DeleteSpecifiedData(ctx context.Context, engine string, name string) error { +func (s *s3Database) DeleteSpecifiedData(ctx context.Context, engine string, name []string) error { return s.db.Delete(ctx, engine, name) } -func (s *s3Database) FindModelsByKey(ctx context.Context, key string) (objects []*model.Object, err error) { - return s.db.FindModelsByKey(ctx, key) -} - func (s *s3Database) DelS3Key(ctx context.Context, engine string, keys ...string) error { return s.s3cache.DelS3Key(ctx, engine, keys...) } diff --git a/pkg/common/storage/database/mgo/object.go b/pkg/common/storage/database/mgo/object.go index 5bc329d33..53f2a6ba4 100644 --- a/pkg/common/storage/database/mgo/object.go +++ b/pkg/common/storage/database/mgo/object.go @@ -22,7 +22,6 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/tools/db/mongoutil" - "github.com/openimsdk/tools/db/pagination" "github.com/openimsdk/tools/errs" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" @@ -91,21 +90,21 @@ func (o *S3Mongo) Take(ctx context.Context, engine string, name string) (*model. return mongoutil.FindOne[*model.Object](ctx, o.coll, bson.M{"name": name, "engine": engine}) } -func (o *S3Mongo) Delete(ctx context.Context, engine string, name string) error { - return mongoutil.DeleteOne(ctx, o.coll, bson.M{"name": name, "engine": engine}) +func (o *S3Mongo) Delete(ctx context.Context, engine string, name []string) error { + if len(name) == 0 { + return nil + } + return mongoutil.DeleteOne(ctx, o.coll, bson.M{"engine": engine, "name": bson.M{"$in": name}}) } -// Find Expires object -func (o *S3Mongo) FindNeedDeleteObjectByDB(ctx context.Context, duration time.Time, needDelType []string, pagination pagination.Pagination) (total int64, objects []*model.Object, err error) { - return mongoutil.FindPage[*model.Object](ctx, o.coll, bson.M{ - "create_time": bson.M{"$lt": duration}, - "group": bson.M{"$in": needDelType}, - }, pagination) -} - -// Find object by key -func (o *S3Mongo) FindModelsByKey(ctx context.Context, key string) (objects []*model.Object, err error) { +func (o *S3Mongo) FindExpirationObject(ctx context.Context, engine string, expiration time.Time, needDelType []string, count int64) ([]*model.Object, error) { + opt := options.Find() + if count > 0 { + opt.SetLimit(count) + } return mongoutil.Find[*model.Object](ctx, o.coll, bson.M{ - "key": key, - }) + "engine": engine, + "create_time": bson.M{"$lt": expiration}, + "group": bson.M{"$in": needDelType}, + }, opt) } diff --git a/pkg/common/storage/database/object.go b/pkg/common/storage/database/object.go index c741e39a6..86d47a2a6 100644 --- a/pkg/common/storage/database/object.go +++ b/pkg/common/storage/database/object.go @@ -19,13 +19,11 @@ import ( "time" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "github.com/openimsdk/tools/db/pagination" ) type ObjectInfo interface { SetObject(ctx context.Context, obj *model.Object) error Take(ctx context.Context, engine string, name string) (*model.Object, error) - Delete(ctx context.Context, engine string, name string) error - FindNeedDeleteObjectByDB(ctx context.Context, duration time.Time, needDelType []string, pagination pagination.Pagination) (total int64, objects []*model.Object, err error) - FindModelsByKey(ctx context.Context, key string) (objects []*model.Object, err error) + Delete(ctx context.Context, engine string, name []string) error + FindExpirationObject(ctx context.Context, engine string, expiration time.Time, needDelType []string, count int64) ([]*model.Object, error) } From 705bc37f9942b3059e2ec1bf82c5fb227329b271 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 19 Dec 2024 11:28:49 +0800 Subject: [PATCH 05/42] refactoring scheduled tasks --- internal/rpc/msg/clear.go | 75 +++++++------- internal/rpc/third/s3.go | 6 +- internal/tools/msg.go | 25 +++-- internal/tools/s3.go | 11 ++- internal/tools/user_msg.go | 6 +- pkg/common/storage/controller/msg.go | 102 +++++++++----------- pkg/common/storage/database/mgo/msg.go | 20 ++-- pkg/common/storage/database/mgo/msg_test.go | 78 ++++++++++----- pkg/common/storage/database/msg.go | 4 +- 9 files changed, 180 insertions(+), 147 deletions(-) diff --git a/internal/rpc/msg/clear.go b/internal/rpc/msg/clear.go index 7d62e7c8f..8d4bd0c8e 100644 --- a/internal/rpc/msg/clear.go +++ b/internal/rpc/msg/clear.go @@ -2,6 +2,7 @@ package msg import ( "context" + "strings" "time" "github.com/openimsdk/open-im-server/v3/pkg/authverify" @@ -26,52 +27,42 @@ func (m *msgServer) DestructMsgs(ctx context.Context, req *msg.DestructMsgsReq) if req.Timestamp > time.Now().UnixMilli() { return nil, errs.ErrArgs.WrapMsg("request millisecond timestamp error") } - var ( - docNum int - msgNum int - start = time.Now() - getLimit = 5000 - ) - - destructMsg := func(ctx context.Context) (bool, error) { - docIDs, err := m.MsgDatabase.GetDocIDs(ctx) - if err != nil { - return false, err - } - - msgs, err := m.MsgDatabase.GetBeforeMsg(ctx, req.Timestamp, docIDs, getLimit) - if err != nil { - return false, err - } - if len(msgs) == 0 { - return false, nil - } - - for _, msg := range msgs { - index, err := m.MsgDatabase.DeleteDocMsgBefore(ctx, req.Timestamp, msg) - if err != nil { - return false, err - } - if len(index) == 0 { - return false, errs.ErrInternalServer.WrapMsg("delete doc msg failed") - } - - docNum++ - msgNum += len(index) - } - - return true, nil + if req.Limit <= 0 { + return nil, errs.ErrArgs.WrapMsg("request limit error") } - - _, err = destructMsg(ctx) + docs, err := m.MsgDatabase.GetRandBeforeMsg(ctx, req.Timestamp, int(req.Limit)) if err != nil { - log.ZError(ctx, "clear msg failed", err, "docNum", docNum, "msgNum", msgNum, "cost", time.Since(start)) return nil, err } - - log.ZDebug(ctx, "clearing message", "docNum", docNum, "msgNum", msgNum, "cost", time.Since(start)) - - return &msg.DestructMsgsResp{}, nil + for _, doc := range docs { + if err := m.MsgDatabase.DeleteDoc(ctx, doc.DocID); err != nil { + return nil, err + } + index := strings.LastIndex(doc.DocID, ":") + if index < 0 { + continue + } + var minSeq int64 + for _, model := range doc.Msg { + if model.Msg == nil { + continue + } + if model.Msg.Seq > minSeq { + minSeq = model.Msg.Seq + } + } + if minSeq <= 0 { + continue + } + conversationID := doc.DocID[:index] + if conversationID == "" { + continue + } + if err := m.MsgDatabase.SetMinSeq(ctx, conversationID, minSeq); err != nil { + return nil, err + } + } + return &msg.DestructMsgsResp{Count: int32(len(docs))}, nil } // soft delete for user self diff --git a/internal/rpc/third/s3.go b/internal/rpc/third/s3.go index d4146fdf4..a541462a2 100644 --- a/internal/rpc/third/s3.go +++ b/internal/rpc/third/s3.go @@ -19,6 +19,7 @@ import ( "encoding/base64" "encoding/hex" "encoding/json" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" "path" "strconv" "time" @@ -284,10 +285,13 @@ func (t *thirdServer) apiAddress(prefix, name string) string { } func (t *thirdServer) DeleteOutdatedData(ctx context.Context, req *third.DeleteOutdatedDataReq) (*third.DeleteOutdatedDataResp, error) { + if err := authverify.CheckAdmin(ctx, t.config.Share.IMAdminUserID); err != nil { + return nil, err + } engine := t.config.RpcConfig.Object.Enable expireTime := time.UnixMilli(req.ExpireTime) // Find all expired data in S3 database - models, err := t.s3dataBase.FindExpirationObject(ctx, engine, expireTime, req.ObjectGroup, int64(req.Count)) + models, err := t.s3dataBase.FindExpirationObject(ctx, engine, expireTime, req.ObjectGroup, int64(req.Limit)) if err != nil { return nil, err } diff --git a/internal/tools/msg.go b/internal/tools/msg.go index 1149b1f29..46f1573f2 100644 --- a/internal/tools/msg.go +++ b/internal/tools/msg.go @@ -12,12 +12,25 @@ import ( func (c *cronServer) deleteMsg() { now := time.Now() deltime := now.Add(-time.Hour * 24 * time.Duration(c.config.CronTask.RetainChatRecords)) - ctx := mcontext.SetOperationID(c.ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), deltime.UnixMilli())) + operationID := fmt.Sprintf("cron_msg_%d_%d", os.Getpid(), deltime.UnixMilli()) + ctx := mcontext.SetOperationID(c.ctx, operationID) log.ZDebug(ctx, "Destruct chat records", "deltime", deltime, "timestamp", deltime.UnixMilli()) - - if _, err := c.msgClient.DestructMsgs(ctx, &msg.DestructMsgsReq{Timestamp: deltime.UnixMilli()}); err != nil { - log.ZError(ctx, "cron destruct chat records failed", err, "deltime", deltime, "cont", time.Since(now)) - return + const ( + deleteCount = 20 + deleteLimit = 50 + ) + var count int + for i := 1; i <= deleteCount; i++ { + ctx := mcontext.SetOperationID(c.ctx, fmt.Sprintf("%s_%d", operationID, i)) + resp, err := c.msgClient.DestructMsgs(ctx, &msg.DestructMsgsReq{Timestamp: deltime.UnixMilli(), Limit: deleteLimit}) + if err != nil { + log.ZError(ctx, "cron destruct chat records failed", err) + break + } + count += int(resp.Count) + if resp.Count <= deleteLimit { + break + } } - log.ZDebug(ctx, "cron destruct chat records success", "deltime", deltime, "cont", time.Since(now)) + log.ZDebug(ctx, "cron destruct chat records end", "deltime", deltime, "cont", time.Since(now), "deleteDocs", count) } diff --git a/internal/tools/s3.go b/internal/tools/s3.go index 38d8badc6..3b0122c84 100644 --- a/internal/tools/s3.go +++ b/internal/tools/s3.go @@ -15,20 +15,21 @@ func (c *cronServer) clearS3() { // number of pagination. if need modify, need update value in third.DeleteOutdatedData pageShowNumber := 500 deleteTime := start.Add(-time.Hour * 24 * time.Duration(c.config.CronTask.FileExpireTime)) - operationID := fmt.Sprintf("cron_%d_%d", os.Getpid(), deleteTime.UnixMilli()) + operationID := fmt.Sprintf("cron_s3_%d_%d", os.Getpid(), deleteTime.UnixMilli()) ctx := mcontext.SetOperationID(c.ctx, operationID) log.ZDebug(ctx, "deleteoutDatedData", "deletetime", deleteTime, "timestamp", deleteTime.UnixMilli()) + var count int for i := 1; i <= executeNum; i++ { ctx := mcontext.SetOperationID(c.ctx, fmt.Sprintf("%s_%d", operationID, i)) - resp, err := c.thirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: deleteTime.UnixMilli(), ObjectGroup: c.config.CronTask.DeleteObjectType, Count: int32(pageShowNumber)}) + resp, err := c.thirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: deleteTime.UnixMilli(), ObjectGroup: c.config.CronTask.DeleteObjectType, Limit: int32(pageShowNumber)}) if err != nil { - log.ZError(ctx, "cron deleteoutDatedData failed", err, "deleteTime", deleteTime, "cont", time.Since(start)) + log.ZError(ctx, "cron deleteoutDatedData failed", err) return } + count += int(resp.Count) if resp.Count < int32(pageShowNumber) { break } } - - log.ZDebug(ctx, "cron deleteoutDatedData success", "deltime", deleteTime, "cont", time.Since(start)) + log.ZDebug(ctx, "cron deleteoutDatedData success", "deltime", deleteTime, "cont", time.Since(start), "count", count) } diff --git a/internal/tools/user_msg.go b/internal/tools/user_msg.go index 9dfaf58c3..52a2b6bdc 100644 --- a/internal/tools/user_msg.go +++ b/internal/tools/user_msg.go @@ -12,9 +12,9 @@ import ( func (c *cronServer) clearUserMsg() { now := time.Now() - ctx := mcontext.SetOperationID(c.ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), now.UnixMilli())) - log.ZDebug(ctx, "clear msg cron start", "now", now) - + operationID := fmt.Sprintf("cron_user_msg_%d_%d", os.Getpid(), now.UnixMilli()) + ctx := mcontext.SetOperationID(c.ctx, operationID) + log.ZDebug(ctx, "clear msg cron start") conversations, err := c.conversationClient.GetConversationsNeedClearMsg(ctx, &pbconversation.GetConversationsNeedClearMsgReq{}) if err != nil { log.ZError(ctx, "Get conversation need Destruct msgs failed.", err) diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index 3c3cd9671..88089f1a0 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -18,7 +18,6 @@ import ( "context" "encoding/json" "errors" - "strings" "time" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" @@ -69,6 +68,7 @@ type CommonMsgDatabase interface { GetMaxSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) SetMinSeqs(ctx context.Context, seqs map[string]int64) error + SetMinSeq(ctx context.Context, conversationID string, seq int64) error SetUserConversationsMinSeqs(ctx context.Context, userID string, seqs map[string]int64) (err error) SetHasReadSeq(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error @@ -95,13 +95,14 @@ type CommonMsgDatabase interface { ConvertMsgsDocLen(ctx context.Context, conversationIDs []string) // get Msg when destruct msg before - GetBeforeMsg(ctx context.Context, ts int64, docIds []string, limit int) ([]*model.MsgDocModel, error) - DeleteDocMsgBefore(ctx context.Context, ts int64, doc *model.MsgDocModel) ([]int, error) + //DeleteDocMsgBefore(ctx context.Context, ts int64, doc *model.MsgDocModel) ([]int, error) - GetDocIDs(ctx context.Context) ([]string, error) + GetRandBeforeMsg(ctx context.Context, ts int64, limit int) ([]*model.MsgDocModel, error) SetUserConversationsMaxSeq(ctx context.Context, conversationID string, userID string, seq int64) error SetUserConversationsMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error + + DeleteDoc(ctx context.Context, docID string) error } func NewCommonMsgDatabase(msgDocModel database.Msg, msg cache.MsgCache, seqUser cache.SeqUser, seqConversation cache.SeqConversationCache, kafkaConf *config.Kafka) (CommonMsgDatabase, error) { @@ -806,9 +807,10 @@ func (db *commonMsgDatabase) GetMaxSeq(ctx context.Context, conversationID strin return db.seqConversation.GetMaxSeq(ctx, conversationID) } -func (db *commonMsgDatabase) SetMinSeq(ctx context.Context, conversationID string, minSeq int64) error { - return db.seqConversation.SetMinSeq(ctx, conversationID, minSeq) -} +// +//func (db *commonMsgDatabase) SetMinSeq(ctx context.Context, conversationID string, minSeq int64) error { +// return db.seqConversation.SetMinSeq(ctx, conversationID, minSeq) +//} func (db *commonMsgDatabase) SetMinSeqs(ctx context.Context, seqs map[string]int64) error { return db.seqConversation.SetMinSeqs(ctx, seqs) @@ -947,56 +949,40 @@ func (db *commonMsgDatabase) ConvertMsgsDocLen(ctx context.Context, conversation db.msgDocDatabase.ConvertMsgsDocLen(ctx, conversationIDs) } -func (db *commonMsgDatabase) GetBeforeMsg(ctx context.Context, ts int64, docIDs []string, limit int) ([]*model.MsgDocModel, error) { - var msgs []*model.MsgDocModel - for i := 0; i < len(docIDs); i += 1000 { - end := i + 1000 - if end > len(docIDs) { - end = len(docIDs) - } - - res, err := db.msgDocDatabase.GetBeforeMsg(ctx, ts, docIDs[i:end], limit) - if err != nil { - return nil, err - } - msgs = append(msgs, res...) - - if len(msgs) >= limit { - return msgs[:limit], nil - } - } - return msgs, nil +func (db *commonMsgDatabase) GetRandBeforeMsg(ctx context.Context, ts int64, limit int) ([]*model.MsgDocModel, error) { + return db.msgDocDatabase.GetRandBeforeMsg(ctx, ts, limit) } -func (db *commonMsgDatabase) DeleteDocMsgBefore(ctx context.Context, ts int64, doc *model.MsgDocModel) ([]int, error) { - var notNull int - index := make([]int, 0, len(doc.Msg)) - for i, message := range doc.Msg { - if message.Msg != nil { - notNull++ - if message.Msg.SendTime < ts { - index = append(index, i) - } - } - } - if len(index) == 0 { - return index, nil - } - maxSeq := doc.Msg[index[len(index)-1]].Msg.Seq - conversationID := doc.DocID[:strings.LastIndex(doc.DocID, ":")] - if err := db.setMinSeq(ctx, conversationID, maxSeq+1); err != nil { - return index, err - } - if len(index) == notNull { - log.ZDebug(ctx, "Delete db in Doc", "DocID", doc.DocID, "index", index, "maxSeq", maxSeq) - return index, db.msgDocDatabase.DeleteDoc(ctx, doc.DocID) - } else { - log.ZDebug(ctx, "delete db in index", "DocID", doc.DocID, "index", index, "maxSeq", maxSeq) - return index, db.msgDocDatabase.DeleteMsgByIndex(ctx, doc.DocID, index) - } -} +// +//func (db *commonMsgDatabase) DeleteDocMsgBefore(ctx context.Context, ts int64, doc *model.MsgDocModel) ([]int, error) { +// var notNull int +// index := make([]int, 0, len(doc.Msg)) +// for i, message := range doc.Msg { +// if message.Msg != nil { +// notNull++ +// if message.Msg.SendTime < ts { +// index = append(index, i) +// } +// } +// } +// if len(index) == 0 { +// return index, nil +// } +// maxSeq := doc.Msg[index[len(index)-1]].Msg.Seq +// conversationID := doc.DocID[:strings.LastIndex(doc.DocID, ":")] +// if err := db.SetMinSeq(ctx, conversationID, maxSeq+1); err != nil { +// return index, err +// } +// if len(index) == notNull { +// log.ZDebug(ctx, "Delete db in Doc", "DocID", doc.DocID, "index", index, "maxSeq", maxSeq) +// return index, db.msgDocDatabase.DeleteDoc(ctx, doc.DocID) +// } else { +// log.ZDebug(ctx, "delete db in index", "DocID", doc.DocID, "index", index, "maxSeq", maxSeq) +// return index, db.msgDocDatabase.DeleteMsgByIndex(ctx, doc.DocID, index) +// } +//} -func (db *commonMsgDatabase) setMinSeq(ctx context.Context, conversationID string, seq int64) error { +func (db *commonMsgDatabase) SetMinSeq(ctx context.Context, conversationID string, seq int64) error { dbSeq, err := db.seqConversation.GetMinSeq(ctx, conversationID) if err != nil { if errors.Is(errs.Unwrap(err), redis.Nil) { @@ -1010,8 +996,8 @@ func (db *commonMsgDatabase) setMinSeq(ctx context.Context, conversationID strin return db.seqConversation.SetMinSeq(ctx, conversationID, seq) } -func (db *commonMsgDatabase) GetDocIDs(ctx context.Context) ([]string, error) { - return db.msgDocDatabase.GetDocIDs(ctx) +func (db *commonMsgDatabase) GetRandDocIDs(ctx context.Context, limit int) ([]string, error) { + return db.msgDocDatabase.GetRandDocIDs(ctx, limit) } func (db *commonMsgDatabase) GetCacheMaxSeqWithTime(ctx context.Context, conversationIDs []string) (map[string]database.SeqTime, error) { @@ -1026,3 +1012,7 @@ func (db *commonMsgDatabase) GetMaxSeqsWithTime(ctx context.Context, conversatio // todo: only the time in the redis cache will be taken, not the message time return db.seqConversation.GetMaxSeqsWithTime(ctx, conversationIDs) } + +func (db *commonMsgDatabase) DeleteDoc(ctx context.Context, docID string) error { + return db.msgDocDatabase.DeleteDoc(ctx, docID) +} diff --git a/pkg/common/storage/database/mgo/msg.go b/pkg/common/storage/database/mgo/msg.go index fc1fe47ea..937dbae1d 100644 --- a/pkg/common/storage/database/mgo/msg.go +++ b/pkg/common/storage/database/mgo/msg.go @@ -1227,8 +1227,7 @@ func (m *MsgMgo) ConvertMsgsDocLen(ctx context.Context, conversationIDs []string } } -func (m *MsgMgo) GetDocIDs(ctx context.Context) ([]string, error) { - limit := 5000 +func (m *MsgMgo) GetRandDocIDs(ctx context.Context, limit int) ([]string, error) { var skip int var docIDs []string var offset int @@ -1267,15 +1266,18 @@ func (m *MsgMgo) GetDocIDs(ctx context.Context) ([]string, error) { return docIDs, errs.Wrap(err) } -func (m *MsgMgo) GetBeforeMsg(ctx context.Context, ts int64, docIDs []string, limit int) ([]*model.MsgDocModel, error) { +func (m *MsgMgo) GetRandBeforeMsg(ctx context.Context, ts int64, limit int) ([]*model.MsgDocModel, error) { return mongoutil.Aggregate[*model.MsgDocModel](ctx, m.coll, []bson.M{ { "$match": bson.M{ - "doc_id": bson.M{ - "$in": docIDs, - }, - "msgs.msg.send_time": bson.M{ - "$lt": ts, + "msgs": bson.M{ + "$not": bson.M{ + "$elemMatch": bson.M{ + "msg.send_time": bson.M{ + "$gt": ts, + }, + }, + }, }, }, }, @@ -1288,7 +1290,7 @@ func (m *MsgMgo) GetBeforeMsg(ctx context.Context, ts int64, docIDs []string, li }, }, { - "$limit": limit, + "$sample": limit, }, }) } diff --git a/pkg/common/storage/database/mgo/msg_test.go b/pkg/common/storage/database/mgo/msg_test.go index 5aed4dc51..eff847d26 100644 --- a/pkg/common/storage/database/mgo/msg_test.go +++ b/pkg/common/storage/database/mgo/msg_test.go @@ -3,12 +3,11 @@ package mgo import ( "context" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "github.com/openimsdk/protocol/msg" - "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/db/mongoutil" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" + "math" "math/rand" "strconv" "testing" @@ -18,33 +17,43 @@ import ( func TestName1(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*300) defer cancel() - cli := Result(mongo.Connect(ctx, options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) + cli := Result(mongo.Connect(ctx, options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.66:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) - v := &MsgMgo{ - coll: cli.Database("openim_v3").Collection("msg3"), - } + //v := &MsgMgo{ + // coll: cli.Database("openim_v3").Collection("msg3"), + //} + // + //req := &msg.SearchMessageReq{ + // //RecvID: "3187706596", + // //SendID: "7009965934", + // ContentType: 101, + // //SendTime: "2024-05-06", + // //SessionType: 3, + // Pagination: &sdkws.RequestPagination{ + // PageNumber: 1, + // ShowNumber: 10, + // }, + //} + //total, res, err := v.SearchMessage(ctx, req) + //if err != nil { + // panic(err) + //} + // + //for i, re := range res { + // t.Logf("%d => %d | %+v", i+1, re.Msg.Seq, re.Msg.Content) + //} + // + //t.Log(total) - req := &msg.SearchMessageReq{ - //RecvID: "3187706596", - //SendID: "7009965934", - ContentType: 101, - //SendTime: "2024-05-06", - //SessionType: 3, - Pagination: &sdkws.RequestPagination{ - PageNumber: 1, - ShowNumber: 10, - }, - } - total, res, err := v.SearchMessage(ctx, req) + msg, err := NewMsgMongo(cli.Database("openim_v3")) if err != nil { panic(err) } - - for i, re := range res { - t.Logf("%d => %d | %+v", i+1, re.Msg.Seq, re.Msg.Content) + res, err := msg.GetBeforeMsg(ctx, time.Now().UnixMilli(), []string{"1:0"}, 1000) + if err != nil { + panic(err) } - - t.Log(total) + t.Log(len(res)) } func TestName10(t *testing.T) { @@ -73,3 +82,26 @@ func TestName10(t *testing.T) { } } + +func TestName3(t *testing.T) { + t.Log(uint64(math.MaxUint64)) + t.Log(int64(math.MaxInt64)) + + t.Log(int64(math.MinInt64)) +} + +func TestName4(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*300) + defer cancel() + cli := Result(mongo.Connect(ctx, options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.66:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) + + msg, err := NewMsgMongo(cli.Database("openim_v3")) + if err != nil { + panic(err) + } + res, err := msg.GetBeforeMsg(ctx, time.Now().UnixMilli(), []string{"1:0"}, 1000) + if err != nil { + panic(err) + } + t.Log(len(res)) +} diff --git a/pkg/common/storage/database/msg.go b/pkg/common/storage/database/msg.go index 23a99f5b9..17aaf2342 100644 --- a/pkg/common/storage/database/msg.go +++ b/pkg/common/storage/database/msg.go @@ -45,7 +45,7 @@ type Msg interface { DeleteDoc(ctx context.Context, docID string) error DeleteMsgByIndex(ctx context.Context, docID string, index []int) error - GetBeforeMsg(ctx context.Context, ts int64, docIDs []string, limit int) ([]*model.MsgDocModel, error) + GetRandBeforeMsg(ctx context.Context, ts int64, limit int) ([]*model.MsgDocModel, error) - GetDocIDs(ctx context.Context) ([]string, error) + GetRandDocIDs(ctx context.Context, limit int) ([]string, error) } From 8807597ffbba18544bab40021bbc90e87a6caadd Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 19 Dec 2024 17:15:15 +0800 Subject: [PATCH 06/42] refactoring scheduled tasks --- internal/api/router.go | 16 +++ internal/rpc/conversation/conversation.go | 29 ++++++ internal/rpc/msg/clear.go | 19 ++-- internal/tools/msg.go | 2 +- internal/tools/s3.go | 58 +++++++++-- internal/tools/user_msg.go | 31 +++--- pkg/common/storage/controller/conversation.go | 6 ++ pkg/common/storage/controller/msg.go | 6 ++ pkg/common/storage/database/conversation.go | 1 + .../storage/database/mgo/conversation.go | 32 ++++++ pkg/common/storage/database/mgo/msg.go | 99 ++++++++++--------- pkg/common/storage/database/mgo/msg_test.go | 41 ++++---- pkg/common/storage/database/msg.go | 2 + 13 files changed, 247 insertions(+), 95 deletions(-) diff --git a/internal/api/router.go b/internal/api/router.go index d6bf4e130..33f16ade3 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -1,7 +1,9 @@ package api import ( + "context" "fmt" + "github.com/openimsdk/protocol/user" "net/http" "strings" @@ -164,6 +166,7 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En authRouterGroup.POST("/get_user_token", a.GetUserToken) authRouterGroup.POST("/parse_token", a.ParseToken) authRouterGroup.POST("/force_logout", a.ForceLogout) + } // Third service thirdGroup := r.Group("/third") @@ -297,3 +300,16 @@ var Whitelist = []string{ "/auth/get_admin_token", "/auth/parse_token", } + +func init() { + + var uc user.UserClient + var g *gin.Engine + g.POST("/get_admin_token", New(uc, uc.AccountCheck)) + +} + +func New[A, B, C any](c C, fn func(c C, ctx context.Context, req *A, opts ...grpc.CallOption) (*B, error)) func(c *gin.Context) { + + return nil +} diff --git a/internal/rpc/conversation/conversation.go b/internal/rpc/conversation/conversation.go index f534ca6de..9fcd5abb1 100644 --- a/internal/rpc/conversation/conversation.go +++ b/internal/rpc/conversation/conversation.go @@ -763,3 +763,32 @@ func (c *conversationServer) GetPinnedConversationIDs(ctx context.Context, req * } return &pbconversation.GetPinnedConversationIDsResp{ConversationIDs: conversationIDs}, nil } + +func (c *conversationServer) ClearUserConversationMsg(ctx context.Context, req *pbconversation.ClearUserConversationMsgReq) (*pbconversation.ClearUserConversationMsgResp, error) { + conversations, err := c.conversationDatabase.FindRandConversation(ctx, req.Timestamp, int(req.Limit)) + if err != nil { + return nil, err + } + for _, conversation := range conversations { + if conversation.IsMsgDestruct == false || conversation.MsgDestructTime == 0 { + continue + } + rcpReq := &pbmsg.GetLastMessageSeqByTimeReq{ConversationID: conversation.ConversationID, Time: req.Timestamp - conversation.MsgDestructTime} + resp, err := pbmsg.GetLastMessageSeqByTime.Invoke(ctx, rcpReq) + if err != nil { + return nil, err + } + if resp.Seq == 0 { + continue + } + _, err = c.SetConversationMinSeq(ctx, &pbconversation.SetConversationMinSeqReq{ + ConversationID: conversation.ConversationID, + OwnerUserID: []string{conversation.OwnerUserID}, + MinSeq: resp.Seq + 1, + }) + if err != nil { + return nil, err + } + } + return &pbconversation.ClearUserConversationMsgResp{}, nil +} diff --git a/internal/rpc/msg/clear.go b/internal/rpc/msg/clear.go index 8d4bd0c8e..eb7a4b7a6 100644 --- a/internal/rpc/msg/clear.go +++ b/internal/rpc/msg/clear.go @@ -10,7 +10,6 @@ import ( pbconv "github.com/openimsdk/protocol/conversation" "github.com/openimsdk/protocol/msg" "github.com/openimsdk/protocol/wrapperspb" - "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mcontext" "github.com/openimsdk/tools/utils/datautil" @@ -20,16 +19,10 @@ import ( ) // hard delete in Database. -func (m *msgServer) DestructMsgs(ctx context.Context, req *msg.DestructMsgsReq) (_ *msg.DestructMsgsResp, err error) { +func (m *msgServer) DestructMsgs(ctx context.Context, req *msg.DestructMsgsReq) (*msg.DestructMsgsResp, error) { if err := authverify.CheckAdmin(ctx, m.config.Share.IMAdminUserID); err != nil { return nil, err } - if req.Timestamp > time.Now().UnixMilli() { - return nil, errs.ErrArgs.WrapMsg("request millisecond timestamp error") - } - if req.Limit <= 0 { - return nil, errs.ErrArgs.WrapMsg("request limit error") - } docs, err := m.MsgDatabase.GetRandBeforeMsg(ctx, req.Timestamp, int(req.Limit)) if err != nil { return nil, err @@ -66,7 +59,7 @@ func (m *msgServer) DestructMsgs(ctx context.Context, req *msg.DestructMsgsReq) } // soft delete for user self -func (m *msgServer) ClearMsg(ctx context.Context, req *msg.ClearMsgReq) (_ *msg.ClearMsgResp, err error) { +func (m *msgServer) ClearMsg(ctx context.Context, req *msg.ClearMsgReq) (*msg.ClearMsgResp, error) { temp := convert.ConversationsPb2DB(req.Conversations) batchNum := 100 @@ -128,3 +121,11 @@ func (m *msgServer) ClearMsg(ctx context.Context, req *msg.ClearMsgReq) (_ *msg. return nil, nil } + +func (m *msgServer) GetLastMessageSeqByTime(ctx context.Context, req *msg.GetLastMessageSeqByTimeReq) (*msg.GetLastMessageSeqByTimeResp, error) { + seq, err := m.MsgDatabase.GetLastMessageSeqByTime(ctx, req.ConversationID, req.Time) + if err != nil { + return nil, err + } + return &msg.GetLastMessageSeqByTimeResp{Seq: seq}, nil +} diff --git a/internal/tools/msg.go b/internal/tools/msg.go index 46f1573f2..522649645 100644 --- a/internal/tools/msg.go +++ b/internal/tools/msg.go @@ -16,7 +16,7 @@ func (c *cronServer) deleteMsg() { ctx := mcontext.SetOperationID(c.ctx, operationID) log.ZDebug(ctx, "Destruct chat records", "deltime", deltime, "timestamp", deltime.UnixMilli()) const ( - deleteCount = 20 + deleteCount = 200 deleteLimit = 50 ) var count int diff --git a/internal/tools/s3.go b/internal/tools/s3.go index 3b0122c84..469b0d0cb 100644 --- a/internal/tools/s3.go +++ b/internal/tools/s3.go @@ -11,25 +11,69 @@ import ( func (c *cronServer) clearS3() { start := time.Now() - executeNum := 10 - // number of pagination. if need modify, need update value in third.DeleteOutdatedData - pageShowNumber := 500 deleteTime := start.Add(-time.Hour * 24 * time.Duration(c.config.CronTask.FileExpireTime)) operationID := fmt.Sprintf("cron_s3_%d_%d", os.Getpid(), deleteTime.UnixMilli()) ctx := mcontext.SetOperationID(c.ctx, operationID) log.ZDebug(ctx, "deleteoutDatedData", "deletetime", deleteTime, "timestamp", deleteTime.UnixMilli()) + const ( + deleteCount = 200 + deleteLimit = 100 + ) + var count int - for i := 1; i <= executeNum; i++ { - ctx := mcontext.SetOperationID(c.ctx, fmt.Sprintf("%s_%d", operationID, i)) - resp, err := c.thirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: deleteTime.UnixMilli(), ObjectGroup: c.config.CronTask.DeleteObjectType, Limit: int32(pageShowNumber)}) + for i := 1; i <= deleteCount; i++ { + resp, err := c.thirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: deleteTime.UnixMilli(), ObjectGroup: c.config.CronTask.DeleteObjectType, Limit: deleteLimit}) if err != nil { log.ZError(ctx, "cron deleteoutDatedData failed", err) return } count += int(resp.Count) - if resp.Count < int32(pageShowNumber) { + if resp.Count < deleteLimit { break } } log.ZDebug(ctx, "cron deleteoutDatedData success", "deltime", deleteTime, "cont", time.Since(start), "count", count) } + +// var req *third.DeleteOutdatedDataReq +// count1, err := ExtractField(ctx, c.thirdClient.DeleteOutdatedData, req, (*third.DeleteOutdatedDataResp).GetCount) +// +// c.thirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{}) +// msggateway.GetUsersOnlineStatusCaller.Invoke(ctx, &msggateway.GetUsersOnlineStatusReq{}) +// +// var cli ThirdClient +// +// c111, err := cli.DeleteOutdatedData(ctx, 100) +// +// cli.ThirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{}) +// +// cli.AuthSign(ctx, &third.AuthSignReq{}) +// +// cli.SetAppBadge() +// +//} +// +//func extractField[A, B, C any](ctx context.Context, fn func(ctx context.Context, req *A, opts ...grpc.CallOption) (*B, error), req *A, get func(*B) C) (C, error) { +// resp, err := fn(ctx, req) +// if err != nil { +// var c C +// return c, err +// } +// return get(resp), nil +//} +// +//func ignore(_ any, err error) error { +// return err +//} +// +//type ThirdClient struct { +// third.ThirdClient +//} +// +//func (c *ThirdClient) DeleteOutdatedData(ctx context.Context, expireTime int64) (int32, error) { +// return extractField(ctx, c.ThirdClient.DeleteOutdatedData, &third.DeleteOutdatedDataReq{ExpireTime: expireTime}, (*third.DeleteOutdatedDataResp).GetCount) +//} +// +//func (c *ThirdClient) DeleteOutdatedData1(ctx context.Context, expireTime int64) error { +// return ignore(c.ThirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: expireTime})) +//} diff --git a/internal/tools/user_msg.go b/internal/tools/user_msg.go index 52a2b6bdc..3d4ca40d3 100644 --- a/internal/tools/user_msg.go +++ b/internal/tools/user_msg.go @@ -3,7 +3,6 @@ package tools import ( "fmt" pbconversation "github.com/openimsdk/protocol/conversation" - "github.com/openimsdk/protocol/msg" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mcontext" "os" @@ -14,18 +13,22 @@ func (c *cronServer) clearUserMsg() { now := time.Now() operationID := fmt.Sprintf("cron_user_msg_%d_%d", os.Getpid(), now.UnixMilli()) ctx := mcontext.SetOperationID(c.ctx, operationID) - log.ZDebug(ctx, "clear msg cron start") - conversations, err := c.conversationClient.GetConversationsNeedClearMsg(ctx, &pbconversation.GetConversationsNeedClearMsgReq{}) - if err != nil { - log.ZError(ctx, "Get conversation need Destruct msgs failed.", err) - return + log.ZDebug(ctx, "clear user msg cron start") + const ( + deleteCount = 200 + deleteLimit = 100 + ) + var count int + for i := 1; i <= deleteCount; i++ { + resp, err := c.conversationClient.ClearUserConversationMsg(ctx, &pbconversation.ClearUserConversationMsgReq{Timestamp: now.UnixMilli(), Limit: deleteLimit}) + if err != nil { + log.ZError(ctx, "ClearUserConversationMsg failed.", err) + return + } + count += int(resp.Count) + if resp.Count < deleteLimit { + break + } } - - _, err = c.msgClient.ClearMsg(ctx, &msg.ClearMsgReq{Conversations: conversations.Conversations}) - if err != nil { - log.ZError(ctx, "Clear Msg failed.", err) - return - } - - log.ZDebug(ctx, "clear msg cron task completed", "cont", time.Since(now)) + log.ZDebug(ctx, "clear user msg cron task completed", "cont", time.Since(now), "count", count) } diff --git a/pkg/common/storage/controller/conversation.go b/pkg/common/storage/controller/conversation.go index bf41cce95..d4088e0c0 100644 --- a/pkg/common/storage/controller/conversation.go +++ b/pkg/common/storage/controller/conversation.go @@ -74,6 +74,8 @@ type ConversationDatabase interface { GetNotNotifyConversationIDs(ctx context.Context, userID string) ([]string, error) // GetPinnedConversationIDs gets pinned conversationIDs by userID GetPinnedConversationIDs(ctx context.Context, userID string) ([]string, error) + // FindRandConversation finds random conversations based on the specified timestamp and limit. + FindRandConversation(ctx context.Context, ts int64, limit int) ([]*relationtb.Conversation, error) } func NewConversationDatabase(conversation database.Conversation, cache cache.ConversationCache, tx tx.Tx) ConversationDatabase { @@ -401,3 +403,7 @@ func (c *conversationDatabase) GetPinnedConversationIDs(ctx context.Context, use } return conversationIDs, nil } + +func (c *conversationDatabase) FindRandConversation(ctx context.Context, ts int64, limit int) ([]*relationtb.Conversation, error) { + return c.conversationDB.FindRandConversation(ctx, ts, limit) +} diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index 88089f1a0..c29544c33 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -103,6 +103,8 @@ type CommonMsgDatabase interface { SetUserConversationsMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error DeleteDoc(ctx context.Context, docID string) error + + GetLastMessageSeqByTime(ctx context.Context, conversationID string, time int64) (int64, error) } func NewCommonMsgDatabase(msgDocModel database.Msg, msg cache.MsgCache, seqUser cache.SeqUser, seqConversation cache.SeqConversationCache, kafkaConf *config.Kafka) (CommonMsgDatabase, error) { @@ -1016,3 +1018,7 @@ func (db *commonMsgDatabase) GetMaxSeqsWithTime(ctx context.Context, conversatio func (db *commonMsgDatabase) DeleteDoc(ctx context.Context, docID string) error { return db.msgDocDatabase.DeleteDoc(ctx, docID) } + +func (db *commonMsgDatabase) GetLastMessageSeqByTime(ctx context.Context, conversationID string, time int64) (int64, error) { + return db.msgDocDatabase.GetLastMessageSeqByTime(ctx, conversationID, time) +} diff --git a/pkg/common/storage/database/conversation.go b/pkg/common/storage/database/conversation.go index 30ca01ee7..1fb53cfed 100644 --- a/pkg/common/storage/database/conversation.go +++ b/pkg/common/storage/database/conversation.go @@ -42,4 +42,5 @@ type Conversation interface { GetConversationIDsNeedDestruct(ctx context.Context) ([]*model.Conversation, error) GetConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) ([]string, error) FindConversationUserVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) + FindRandConversation(ctx context.Context, ts int64, limit int) ([]*model.Conversation, error) } diff --git a/pkg/common/storage/database/mgo/conversation.go b/pkg/common/storage/database/mgo/conversation.go index 10e223c89..851ec99c4 100644 --- a/pkg/common/storage/database/mgo/conversation.go +++ b/pkg/common/storage/database/mgo/conversation.go @@ -228,3 +228,35 @@ func (c *ConversationMgo) GetConversationNotReceiveMessageUserIDs(ctx context.Co func (c *ConversationMgo) FindConversationUserVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) { return c.version.FindChangeLog(ctx, userID, version, limit) } + +func (c *ConversationMgo) FindRandConversation(ctx context.Context, ts int64, limit int) ([]*model.Conversation, error) { + pipeline := []bson.M{ + { + "$match": bson.M{ + "is_msg_destruct": true, + "msg_destruct_time": bson.M{"$ne": 0}, + }, + }, + { + "$addFields": bson.M{ + "next_msg_destruct_timestamp": bson.M{ + "$add": []any{ + bson.M{ + "$toLong": "$latest_msg_destruct_time", + }, "$msg_destruct_time"}, + }, + }, + }, + { + "$match": bson.M{ + "next_msg_destruct_timestamp": bson.M{"$lt": ts}, + }, + }, + { + "$sample": bson.M{ + "size": limit, + }, + }, + } + return mongoutil.Aggregate[*model.Conversation](ctx, c.coll, pipeline) +} diff --git a/pkg/common/storage/database/mgo/msg.go b/pkg/common/storage/database/mgo/msg.go index 937dbae1d..e01465466 100644 --- a/pkg/common/storage/database/mgo/msg.go +++ b/pkg/common/storage/database/mgo/msg.go @@ -1290,7 +1290,9 @@ func (m *MsgMgo) GetRandBeforeMsg(ctx context.Context, ts int64, limit int) ([]* }, }, { - "$sample": limit, + "$sample": bson.M{ + "size": limit, + }, }, }) } @@ -1307,53 +1309,56 @@ func (m *MsgMgo) DeleteMsgByIndex(ctx context.Context, docID string, index []int return mongoutil.UpdateOne(ctx, m.coll, bson.M{"doc_id": docID}, bson.M{"$set": set}, true) } -//func (m *MsgMgo) ClearMsg(ctx context.Context, t time.Time) (int64, error) { -// ts := t.UnixMilli() -// var count int64 -// for { -// msgs, err := m.GetBeforeMsg(ctx, ts, 100) -// if err != nil { -// return count, err -// } -// if len(msgs) == 0 { -// return count, nil -// } -// for _, msg := range msgs { -// num, err := m.deleteOneMsg(ctx, ts, msg) -// count += num -// if err != nil { -// return count, err -// } -// } -// } -//} - func (m *MsgMgo) DeleteDoc(ctx context.Context, docID string) error { return mongoutil.DeleteOne(ctx, m.coll, bson.M{"doc_id": docID}) } -//func (m *MsgMgo) DeleteDocMsg(ctx context.Context, ts int64, doc *relation.MsgDocModel) (int64, error) { -// var notNull int -// index := make([]int, 0, len(doc.Msg)) -// for i, message := range doc.Msg { -// if message.Msg != nil { -// notNull++ -// if message.Msg.SendTime < ts { -// index = append(index, i) -// } -// } -// } -// if len(index) == 0 { -// return 0, errs.New("no msg to delete").WrapMsg("deleteOneMsg", "docID", doc.DocID) -// } -// if len(index) == notNull { -// if err := m.DeleteDoc(ctx, doc.DocID); err != nil { -// return 0, err -// } -// } else { -// if err := m.setNullMsg(ctx, doc.DocID, index); err != nil { -// return 0, err -// } -// } -// return int64(len(index)), nil -//} +func (m *MsgMgo) GetLastMessageSeqByTime(ctx context.Context, conversationID string, time int64) (int64, error) { + pipeline := []bson.M{ + { + "$match": bson.M{ + "doc_id": bson.M{ + "$regex": fmt.Sprintf("^%s:", conversationID), + }, + }, + }, + { + "$match": bson.M{ + "msgs.msg.send_time": bson.M{ + "msgs.msg.send_time": bson.M{ + "$lte": time, + }, + }, + }, + }, + { + "$sort": bson.M{ + "_id": -1, + }, + }, + { + "$limit": 1, + }, + { + "$project": bson.M{ + "_id": 0, + "msgs.msg.send_time": 1, + "msgs.msg.seq": 1, + }, + }, + } + res, err := mongoutil.Aggregate[*model.MsgDocModel](ctx, m.coll, pipeline) + if err != nil { + return 0, err + } + if len(res) == 0 { + return 0, nil + } + var seq int64 + for _, v := range res[0].Msg { + if v.Msg.SendTime <= time { + seq = v.Msg.Seq + } + } + return seq, nil +} diff --git a/pkg/common/storage/database/mgo/msg_test.go b/pkg/common/storage/database/mgo/msg_test.go index eff847d26..7218ef1ad 100644 --- a/pkg/common/storage/database/mgo/msg_test.go +++ b/pkg/common/storage/database/mgo/msg_test.go @@ -15,10 +15,10 @@ import ( ) func TestName1(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*300) - defer cancel() - cli := Result(mongo.Connect(ctx, options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.66:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) - + //ctx, cancel := context.WithTimeout(context.Background(), time.Second*300) + //defer cancel() + //cli := Result(mongo.Connect(ctx, options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.66:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) + // //v := &MsgMgo{ // coll: cli.Database("openim_v3").Collection("msg3"), //} @@ -44,16 +44,16 @@ func TestName1(t *testing.T) { //} // //t.Log(total) - - msg, err := NewMsgMongo(cli.Database("openim_v3")) - if err != nil { - panic(err) - } - res, err := msg.GetBeforeMsg(ctx, time.Now().UnixMilli(), []string{"1:0"}, 1000) - if err != nil { - panic(err) - } - t.Log(len(res)) + // + //msg, err := NewMsgMongo(cli.Database("openim_v3")) + //if err != nil { + // panic(err) + //} + //res, err := msg.GetBeforeMsg(ctx, time.Now().UnixMilli(), []string{"1:0"}, 1000) + //if err != nil { + // panic(err) + //} + //t.Log(len(res)) } func TestName10(t *testing.T) { @@ -95,13 +95,20 @@ func TestName4(t *testing.T) { defer cancel() cli := Result(mongo.Connect(ctx, options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.66:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) - msg, err := NewMsgMongo(cli.Database("openim_v3")) + msg, err := NewConversationMongo(cli.Database("openim_v3")) if err != nil { panic(err) } - res, err := msg.GetBeforeMsg(ctx, time.Now().UnixMilli(), []string{"1:0"}, 1000) + ts := time.Now().UnixMilli() + t.Log(ts) + res, err := msg.FindRandConversation(ctx, ts, 10) if err != nil { panic(err) } - t.Log(len(res)) + t.Log(res) +} + +func TestName5(t *testing.T) { + var v time.Time + t.Log(v.UnixMilli()) } diff --git a/pkg/common/storage/database/msg.go b/pkg/common/storage/database/msg.go index 17aaf2342..abb2a44c2 100644 --- a/pkg/common/storage/database/msg.go +++ b/pkg/common/storage/database/msg.go @@ -48,4 +48,6 @@ type Msg interface { GetRandBeforeMsg(ctx context.Context, ts int64, limit int) ([]*model.MsgDocModel, error) GetRandDocIDs(ctx context.Context, limit int) ([]string, error) + + GetLastMessageSeqByTime(ctx context.Context, conversationID string, time int64) (int64, error) } From 1949d6c75684e7fd0bbd349472fac581c8111725 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 19 Dec 2024 17:34:34 +0800 Subject: [PATCH 07/42] refactoring scheduled tasks --- internal/api/router.go | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/internal/api/router.go b/internal/api/router.go index 33f16ade3..5e8038068 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -1,9 +1,7 @@ package api import ( - "context" "fmt" - "github.com/openimsdk/protocol/user" "net/http" "strings" @@ -300,16 +298,3 @@ var Whitelist = []string{ "/auth/get_admin_token", "/auth/parse_token", } - -func init() { - - var uc user.UserClient - var g *gin.Engine - g.POST("/get_admin_token", New(uc, uc.AccountCheck)) - -} - -func New[A, B, C any](c C, fn func(c C, ctx context.Context, req *A, opts ...grpc.CallOption) (*B, error)) func(c *gin.Context) { - - return nil -} From 86d58253c96e0e7ac7a1285df46c816a803c366f Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 19 Dec 2024 18:53:49 +0800 Subject: [PATCH 08/42] refactoring scheduled tasks --- internal/rpc/third/s3.go | 25 +++++---- internal/tools/cron_test.go | 63 +++++++++++++++++++++++ internal/tools/msg.go | 6 +-- internal/tools/s3.go | 2 +- internal/tools/user_msg.go | 2 +- pkg/common/storage/controller/s3.go | 6 +++ pkg/common/storage/database/mgo/object.go | 4 ++ pkg/common/storage/database/object.go | 1 + 8 files changed, 94 insertions(+), 15 deletions(-) create mode 100644 internal/tools/cron_test.go diff --git a/internal/rpc/third/s3.go b/internal/rpc/third/s3.go index a541462a2..e1b42ca33 100644 --- a/internal/rpc/third/s3.go +++ b/internal/rpc/third/s3.go @@ -295,18 +295,23 @@ func (t *thirdServer) DeleteOutdatedData(ctx context.Context, req *third.DeleteO if err != nil { return nil, err } - if len(models) > 0 { - names := datautil.Batch(func(o *model.Object) string { - return o.Name - }, models) - if err := t.s3dataBase.DeleteSpecifiedData(ctx, engine, names); err != nil { - return nil, errs.Wrap(err) - } - if err := t.s3dataBase.DelS3Key(ctx, engine, names...); err != nil { + keyCount := make(map[string]int) + for _, obj := range models { + keyCount[obj.Key]++ + } + for _, obj := range models { + count, err := t.s3dataBase.GetKeyCount(ctx, engine, obj.Key) + if err != nil { return nil, err } - for _, object := range models { - if err := t.s3.DeleteObject(ctx, object.Key); err != nil { + if err := t.s3dataBase.DeleteSpecifiedData(ctx, engine, []string{obj.Name}); err != nil { + return nil, errs.Wrap(err) + } + if err := t.s3dataBase.DelS3Key(ctx, engine, obj.Name); err != nil { + return nil, err + } + if int(count) <= keyCount[obj.Key] { + if err := t.s3.DeleteObject(ctx, obj.Key); err != nil { return nil, err } } diff --git a/internal/tools/cron_test.go b/internal/tools/cron_test.go new file mode 100644 index 000000000..890349069 --- /dev/null +++ b/internal/tools/cron_test.go @@ -0,0 +1,63 @@ +package tools + +import ( + "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" + kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" + pbconversation "github.com/openimsdk/protocol/conversation" + "github.com/openimsdk/protocol/msg" + "github.com/openimsdk/protocol/third" + "github.com/openimsdk/tools/mcontext" + "github.com/openimsdk/tools/mw" + "github.com/robfig/cron/v3" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "testing" +) + +func TestName(t *testing.T) { + conf := &config.Discovery{ + Enable: config.ETCD, + Etcd: config.Etcd{ + RootDirectory: "openim", + Address: []string{"localhost:12379"}, + }, + } + client, err := kdisc.NewDiscoveryRegister(conf, "source") + if err != nil { + panic(err) + } + client.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials())) + ctx := mcontext.SetOpUserID(context.Background(), "imAdmin") + msgConn, err := client.GetConn(ctx, "msg-rpc-service") + if err != nil { + panic(err) + } + thirdConn, err := client.GetConn(ctx, "third-rpc-service") + if err != nil { + panic(err) + } + + conversationConn, err := client.GetConn(ctx, "conversation-rpc-service") + if err != nil { + panic(err) + } + + srv := &cronServer{ + ctx: ctx, + config: &CronTaskConfig{ + CronTask: config.CronTask{ + RetainChatRecords: 1, + FileExpireTime: 1, + DeleteObjectType: []string{"msg-picture", "msg-file", "msg-voice", "msg-video", "msg-video-snapshot", "sdklog", ""}, + }, + }, + cron: cron.New(), + msgClient: msg.NewMsgClient(msgConn), + conversationClient: pbconversation.NewConversationClient(conversationConn), + thirdClient: third.NewThirdClient(thirdConn), + } + srv.deleteMsg() + //srv.clearS3() + //srv.clearUserMsg() +} diff --git a/internal/tools/msg.go b/internal/tools/msg.go index 522649645..cc00cc5b8 100644 --- a/internal/tools/msg.go +++ b/internal/tools/msg.go @@ -16,7 +16,7 @@ func (c *cronServer) deleteMsg() { ctx := mcontext.SetOperationID(c.ctx, operationID) log.ZDebug(ctx, "Destruct chat records", "deltime", deltime, "timestamp", deltime.UnixMilli()) const ( - deleteCount = 200 + deleteCount = 10000 deleteLimit = 50 ) var count int @@ -28,9 +28,9 @@ func (c *cronServer) deleteMsg() { break } count += int(resp.Count) - if resp.Count <= deleteLimit { + if resp.Count < deleteLimit { break } } - log.ZDebug(ctx, "cron destruct chat records end", "deltime", deltime, "cont", time.Since(now), "deleteDocs", count) + log.ZDebug(ctx, "cron destruct chat records end", "deltime", deltime, "cont", time.Since(now), "count", count) } diff --git a/internal/tools/s3.go b/internal/tools/s3.go index 469b0d0cb..9b6b9c408 100644 --- a/internal/tools/s3.go +++ b/internal/tools/s3.go @@ -16,7 +16,7 @@ func (c *cronServer) clearS3() { ctx := mcontext.SetOperationID(c.ctx, operationID) log.ZDebug(ctx, "deleteoutDatedData", "deletetime", deleteTime, "timestamp", deleteTime.UnixMilli()) const ( - deleteCount = 200 + deleteCount = 10000 deleteLimit = 100 ) diff --git a/internal/tools/user_msg.go b/internal/tools/user_msg.go index 3d4ca40d3..a4afa769e 100644 --- a/internal/tools/user_msg.go +++ b/internal/tools/user_msg.go @@ -15,7 +15,7 @@ func (c *cronServer) clearUserMsg() { ctx := mcontext.SetOperationID(c.ctx, operationID) log.ZDebug(ctx, "clear user msg cron start") const ( - deleteCount = 200 + deleteCount = 10000 deleteLimit = 100 ) var count int diff --git a/pkg/common/storage/controller/s3.go b/pkg/common/storage/controller/s3.go index 6a0f11b1c..6693d2dde 100644 --- a/pkg/common/storage/controller/s3.go +++ b/pkg/common/storage/controller/s3.go @@ -42,6 +42,7 @@ type S3Database interface { FindExpirationObject(ctx context.Context, engine string, expiration time.Time, needDelType []string, count int64) ([]*model.Object, error) DeleteSpecifiedData(ctx context.Context, engine string, name []string) error DelS3Key(ctx context.Context, engine string, keys ...string) error + GetKeyCount(ctx context.Context, engine string, key string) (int64, error) } func NewS3Database(rdb redis.UniversalClient, s3 s3.Interface, obj database.ObjectInfo) S3Database { @@ -117,10 +118,15 @@ func (s *s3Database) StatObject(ctx context.Context, name string) (*s3.ObjectInf func (s *s3Database) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) { return s.s3.FormData(ctx, name, size, contentType, duration) } + func (s *s3Database) FindExpirationObject(ctx context.Context, engine string, expiration time.Time, needDelType []string, count int64) ([]*model.Object, error) { return s.db.FindExpirationObject(ctx, engine, expiration, needDelType, count) } +func (s *s3Database) GetKeyCount(ctx context.Context, engine string, key string) (int64, error) { + return s.db.GetKeyCount(ctx, engine, key) +} + func (s *s3Database) DeleteSpecifiedData(ctx context.Context, engine string, name []string) error { return s.db.Delete(ctx, engine, name) } diff --git a/pkg/common/storage/database/mgo/object.go b/pkg/common/storage/database/mgo/object.go index 53f2a6ba4..624eb84a0 100644 --- a/pkg/common/storage/database/mgo/object.go +++ b/pkg/common/storage/database/mgo/object.go @@ -108,3 +108,7 @@ func (o *S3Mongo) FindExpirationObject(ctx context.Context, engine string, expir "group": bson.M{"$in": needDelType}, }, opt) } + +func (o *S3Mongo) GetKeyCount(ctx context.Context, engine string, key string) (int64, error) { + return mongoutil.Count(ctx, o.coll, bson.M{"engine": engine, "key": key}) +} diff --git a/pkg/common/storage/database/object.go b/pkg/common/storage/database/object.go index 86d47a2a6..5541a159b 100644 --- a/pkg/common/storage/database/object.go +++ b/pkg/common/storage/database/object.go @@ -26,4 +26,5 @@ type ObjectInfo interface { Take(ctx context.Context, engine string, name string) (*model.Object, error) Delete(ctx context.Context, engine string, name []string) error FindExpirationObject(ctx context.Context, engine string, expiration time.Time, needDelType []string, count int64) ([]*model.Object, error) + GetKeyCount(ctx context.Context, engine string, key string) (int64, error) } From 46a8d171049faefba46f0c3cc19a9c4099b35cc7 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 20 Dec 2024 15:26:32 +0800 Subject: [PATCH 09/42] refactoring scheduled tasks --- go.mod | 4 +-- go.sum | 8 ++--- internal/rpc/conversation/conversation.go | 39 ++++++++++++++++----- internal/rpc/msg/clear.go | 5 ++- internal/rpc/third/s3.go | 17 ++++----- pkg/common/storage/database/mgo/msg.go | 10 +++--- pkg/common/storage/database/mgo/msg_test.go | 6 ++-- 7 files changed, 56 insertions(+), 33 deletions(-) diff --git a/go.mod b/go.mod index 4eaa18ccc..6c1d421c8 100644 --- a/go.mod +++ b/go.mod @@ -14,8 +14,8 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.72-alpha.66 - github.com/openimsdk/tools v0.0.50-alpha.57 + github.com/openimsdk/protocol v0.0.72-alpha.67 + github.com/openimsdk/tools v0.0.50-alpha.58 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index 6e283eb66..86ed9ed6e 100644 --- a/go.sum +++ b/go.sum @@ -347,10 +347,10 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.15-alpha.2 h1:5Q8yl8ezy2yx+q8/ucU/t4kJnDfCzNOrkXcDACCqtyM= github.com/openimsdk/gomake v0.0.15-alpha.2/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.72-alpha.66 h1:5KoDY6M4T+pXg449ScF6hqeQ+WenBwNyUJn/t8W0oBQ= -github.com/openimsdk/protocol v0.0.72-alpha.66/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= -github.com/openimsdk/tools v0.0.50-alpha.57 h1:oIKV6vYhqp7TRmZ6Pe+r9RNl1D5s7aB/kE9yQVEWcSY= -github.com/openimsdk/tools v0.0.50-alpha.57/go.mod h1:muCtxguNJv8lFwLei27UASu2Nvg4ERSeN0R4K5tivk0= +github.com/openimsdk/protocol v0.0.72-alpha.67 h1:zlLbVkoT0OYsjO2RCutQuDFllcfNvZfdYchvlR6UIe0= +github.com/openimsdk/protocol v0.0.72-alpha.67/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= +github.com/openimsdk/tools v0.0.50-alpha.58 h1:hkFL02Bzzp/l5x+tb7kJ9zes7hilh65EQ4qEIthsQX4= +github.com/openimsdk/tools v0.0.50-alpha.58/go.mod h1:muCtxguNJv8lFwLei27UASu2Nvg4ERSeN0R4K5tivk0= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/internal/rpc/conversation/conversation.go b/internal/rpc/conversation/conversation.go index 9fcd5abb1..696ada152 100644 --- a/internal/rpc/conversation/conversation.go +++ b/internal/rpc/conversation/conversation.go @@ -769,7 +769,8 @@ func (c *conversationServer) ClearUserConversationMsg(ctx context.Context, req * if err != nil { return nil, err } - for _, conversation := range conversations { + latestMsgDestructTime := time.UnixMilli(req.Timestamp) + for i, conversation := range conversations { if conversation.IsMsgDestruct == false || conversation.MsgDestructTime == 0 { continue } @@ -778,17 +779,37 @@ func (c *conversationServer) ClearUserConversationMsg(ctx context.Context, req * if err != nil { return nil, err } - if resp.Seq == 0 { + if resp.Seq <= 0 { + log.ZDebug(ctx, "ClearUserConversationMsg GetLastMessageSeqByTime seq <= 0", "index", i, "conversationID", conversation.ConversationID, "ownerUserID", conversation.OwnerUserID, "msgDestructTime", conversation.MsgDestructTime, "seq", resp.Seq) + if err := c.setConversationMinSeqAndLatestMsgDestructTime(ctx, conversation.ConversationID, conversation.OwnerUserID, -1, latestMsgDestructTime); err != nil { + return nil, err + } continue } - _, err = c.SetConversationMinSeq(ctx, &pbconversation.SetConversationMinSeqReq{ - ConversationID: conversation.ConversationID, - OwnerUserID: []string{conversation.OwnerUserID}, - MinSeq: resp.Seq + 1, - }) - if err != nil { + resp.Seq++ + if err := c.setConversationMinSeqAndLatestMsgDestructTime(ctx, conversation.ConversationID, conversation.OwnerUserID, resp.Seq, latestMsgDestructTime); err != nil { return nil, err } + log.ZDebug(ctx, "ClearUserConversationMsg set min seq", "index", i, "conversationID", conversation.ConversationID, "ownerUserID", conversation.OwnerUserID, "seq", resp.Seq, "msgDestructTime", conversation.MsgDestructTime) } - return &pbconversation.ClearUserConversationMsgResp{}, nil + return &pbconversation.ClearUserConversationMsgResp{Count: int32(len(conversations))}, nil +} + +func (c *conversationServer) setConversationMinSeqAndLatestMsgDestructTime(ctx context.Context, conversationID string, ownerUserID string, minSeq int64, latestMsgDestructTime time.Time) error { + update := map[string]any{ + "latest_msg_destruct_time": latestMsgDestructTime, + } + if minSeq >= 0 { + req := &pbmsg.SetUserConversationMinSeqReq{ConversationID: conversationID, OwnerUserID: []string{ownerUserID}, MinSeq: minSeq} + if _, err := pbmsg.SetUserConversationMinSeqCaller.Invoke(ctx, req); err != nil { + return err + } + update["min_seq"] = minSeq + } + + if err := c.conversationDatabase.UpdateUsersConversationField(ctx, []string{ownerUserID}, conversationID, update); err != nil { + return err + } + c.conversationNotificationSender.ConversationChangeNotification(ctx, ownerUserID, []string{conversationID}) + return nil } diff --git a/internal/rpc/msg/clear.go b/internal/rpc/msg/clear.go index eb7a4b7a6..7a2d36300 100644 --- a/internal/rpc/msg/clear.go +++ b/internal/rpc/msg/clear.go @@ -27,10 +27,11 @@ func (m *msgServer) DestructMsgs(ctx context.Context, req *msg.DestructMsgsReq) if err != nil { return nil, err } - for _, doc := range docs { + for i, doc := range docs { if err := m.MsgDatabase.DeleteDoc(ctx, doc.DocID); err != nil { return nil, err } + log.ZDebug(ctx, "DestructMsgs delete doc", "index", i, "docID", doc.DocID) index := strings.LastIndex(doc.DocID, ":") if index < 0 { continue @@ -51,9 +52,11 @@ func (m *msgServer) DestructMsgs(ctx context.Context, req *msg.DestructMsgsReq) if conversationID == "" { continue } + minSeq++ if err := m.MsgDatabase.SetMinSeq(ctx, conversationID, minSeq); err != nil { return nil, err } + log.ZDebug(ctx, "DestructMsgs delete doc set min seq", "index", i, "docID", doc.DocID, "conversationID", conversationID, "setMinSeq", minSeq) } return &msg.DestructMsgsResp{Count: int32(len(docs))}, nil } diff --git a/internal/rpc/third/s3.go b/internal/rpc/third/s3.go index e1b42ca33..8796fe824 100644 --- a/internal/rpc/third/s3.go +++ b/internal/rpc/third/s3.go @@ -295,22 +295,19 @@ func (t *thirdServer) DeleteOutdatedData(ctx context.Context, req *third.DeleteO if err != nil { return nil, err } - keyCount := make(map[string]int) - for _, obj := range models { - keyCount[obj.Key]++ - } - for _, obj := range models { - count, err := t.s3dataBase.GetKeyCount(ctx, engine, obj.Key) - if err != nil { - return nil, err - } + for i, obj := range models { if err := t.s3dataBase.DeleteSpecifiedData(ctx, engine, []string{obj.Name}); err != nil { return nil, errs.Wrap(err) } if err := t.s3dataBase.DelS3Key(ctx, engine, obj.Name); err != nil { return nil, err } - if int(count) <= keyCount[obj.Key] { + count, err := t.s3dataBase.GetKeyCount(ctx, engine, obj.Key) + if err != nil { + return nil, err + } + log.ZDebug(ctx, "delete s3 object record", "index", i, "s3", obj, "count", count) + if count == 0 { if err := t.s3.DeleteObject(ctx, obj.Key); err != nil { return nil, err } diff --git a/pkg/common/storage/database/mgo/msg.go b/pkg/common/storage/database/mgo/msg.go index e01465466..f37176695 100644 --- a/pkg/common/storage/database/mgo/msg.go +++ b/pkg/common/storage/database/mgo/msg.go @@ -1318,16 +1318,14 @@ func (m *MsgMgo) GetLastMessageSeqByTime(ctx context.Context, conversationID str { "$match": bson.M{ "doc_id": bson.M{ - "$regex": fmt.Sprintf("^%s:", conversationID), + "$regex": fmt.Sprintf("^%s", conversationID), }, }, }, { "$match": bson.M{ "msgs.msg.send_time": bson.M{ - "msgs.msg.send_time": bson.M{ - "$lte": time, - }, + "$lte": time, }, }, }, @@ -1342,6 +1340,7 @@ func (m *MsgMgo) GetLastMessageSeqByTime(ctx context.Context, conversationID str { "$project": bson.M{ "_id": 0, + "doc_id": 1, "msgs.msg.send_time": 1, "msgs.msg.seq": 1, }, @@ -1356,6 +1355,9 @@ func (m *MsgMgo) GetLastMessageSeqByTime(ctx context.Context, conversationID str } var seq int64 for _, v := range res[0].Msg { + if v.Msg == nil { + continue + } if v.Msg.SendTime <= time { seq = v.Msg.Seq } diff --git a/pkg/common/storage/database/mgo/msg_test.go b/pkg/common/storage/database/mgo/msg_test.go index 7218ef1ad..992090552 100644 --- a/pkg/common/storage/database/mgo/msg_test.go +++ b/pkg/common/storage/database/mgo/msg_test.go @@ -95,13 +95,13 @@ func TestName4(t *testing.T) { defer cancel() cli := Result(mongo.Connect(ctx, options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.66:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) - msg, err := NewConversationMongo(cli.Database("openim_v3")) + msg, err := NewMsgMongo(cli.Database("openim_v3")) if err != nil { panic(err) } - ts := time.Now().UnixMilli() + ts := time.Now().Add(-time.Hour * 24 * 5).UnixMilli() t.Log(ts) - res, err := msg.FindRandConversation(ctx, ts, 10) + res, err := msg.GetLastMessageSeqByTime(ctx, "sg_1523453548", ts) if err != nil { panic(err) } From 2e8ca5f5b938fa929b0123c2a66d246aa3537ed2 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 24 Dec 2024 17:50:05 +0800 Subject: [PATCH 10/42] upgrading pkg tools --- go.mod | 2 +- go.sum | 4 ++-- internal/msggateway/client.go | 4 ++-- internal/msgtransfer/init.go | 2 +- internal/msgtransfer/online_history_msg_handler.go | 2 +- internal/rpc/msg/send.go | 2 +- pkg/rpccache/subscriber.go | 3 ++- 7 files changed, 10 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 26f167a19..05fe6db15 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.72-alpha.68 - github.com/openimsdk/tools v0.0.50-alpha.60 + github.com/openimsdk/tools v0.0.50-alpha.61 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index c0385481c..1fc3c33db 100644 --- a/go.sum +++ b/go.sum @@ -349,8 +349,8 @@ github.com/openimsdk/gomake v0.0.15-alpha.2 h1:5Q8yl8ezy2yx+q8/ucU/t4kJnDfCzNOrk github.com/openimsdk/gomake v0.0.15-alpha.2/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.72-alpha.68 h1:Ekn6S9Ftt12Xs/p9kJ39RDr2gSwIczz+MmSHQE4lAek= github.com/openimsdk/protocol v0.0.72-alpha.68/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= -github.com/openimsdk/tools v0.0.50-alpha.60 h1:dYqYpSdSN5o6CxlEjua2USfwfUiG0tUWFBpqghTjbWE= -github.com/openimsdk/tools v0.0.50-alpha.60/go.mod h1:muCtxguNJv8lFwLei27UASu2Nvg4ERSeN0R4K5tivk0= +github.com/openimsdk/tools v0.0.50-alpha.61 h1:zKEZwrj+fUVuyC6KR3kZp9zFaCCIFgoSbHO0r0mZ6h4= +github.com/openimsdk/tools v0.0.50-alpha.61/go.mod h1:JowL2jYr8tu4vcQe+5hJh4v3BtSx1T0CIS3pgU/Mw+U= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/internal/msggateway/client.go b/internal/msggateway/client.go index 161300d6d..1040f2be2 100644 --- a/internal/msggateway/client.go +++ b/internal/msggateway/client.go @@ -131,7 +131,7 @@ func (c *Client) readMessage() { defer func() { if r := recover(); r != nil { c.closedErr = ErrPanic - log.ZPanic(c.ctx, "socket have panic err:", r) + log.ZPanic(c.ctx, "socket have panic err:", errs.ErrPanic(r)) } c.close() }() @@ -376,7 +376,7 @@ func (c *Client) activeHeartbeat(ctx context.Context) { go func() { defer func() { if r := recover(); r != nil { - log.ZPanic(ctx, "activeHeartbeat Panic", r) + log.ZPanic(ctx, "activeHeartbeat Panic", errs.ErrPanic(r)) } }() log.ZDebug(ctx, "server initiative send heartbeat start.") diff --git a/internal/msgtransfer/init.go b/internal/msgtransfer/init.go index 5cb613123..ee9f06644 100644 --- a/internal/msgtransfer/init.go +++ b/internal/msgtransfer/init.go @@ -195,7 +195,7 @@ func (m *MsgTransfer) Start(index int, config *Config, client discovery.SvcDisco go func() { defer func() { if r := recover(); r != nil { - log.ZPanic(m.ctx, "MsgTransfer Start Panic", r) + log.ZPanic(m.ctx, "MsgTransfer Start Panic", errs.ErrPanic(r)) } }() if err := prommetrics.TransferInit(listener); err != nil && !errors.Is(err, http.ErrServerClosed) { diff --git a/internal/msgtransfer/online_history_msg_handler.go b/internal/msgtransfer/online_history_msg_handler.go index 83b075061..7b989070b 100644 --- a/internal/msgtransfer/online_history_msg_handler.go +++ b/internal/msgtransfer/online_history_msg_handler.go @@ -361,7 +361,7 @@ func (och *OnlineHistoryRedisConsumerHandler) handleNotification(ctx context.Con func (och *OnlineHistoryRedisConsumerHandler) HandleUserHasReadSeqMessages(ctx context.Context) { defer func() { if r := recover(); r != nil { - log.ZPanic(ctx, "HandleUserHasReadSeqMessages Panic", r) + log.ZPanic(ctx, "HandleUserHasReadSeqMessages Panic", errs.ErrPanic(r)) } }() diff --git a/internal/rpc/msg/send.go b/internal/rpc/msg/send.go index b9bbf615f..f01134f8f 100644 --- a/internal/rpc/msg/send.go +++ b/internal/rpc/msg/send.go @@ -88,7 +88,7 @@ func (m *msgServer) setConversationAtInfo(nctx context.Context, msg *sdkws.MsgDa defer func() { if r := recover(); r != nil { - log.ZPanic(nctx, "setConversationAtInfo Panic", r) + log.ZPanic(nctx, "setConversationAtInfo Panic", errs.ErrPanic(r)) } }() diff --git a/pkg/rpccache/subscriber.go b/pkg/rpccache/subscriber.go index d28d1aa29..0cb35bebf 100644 --- a/pkg/rpccache/subscriber.go +++ b/pkg/rpccache/subscriber.go @@ -17,6 +17,7 @@ package rpccache import ( "context" "encoding/json" + "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/redis/go-redis/v9" ) @@ -24,7 +25,7 @@ import ( func subscriberRedisDeleteCache(ctx context.Context, client redis.UniversalClient, channel string, del func(ctx context.Context, key ...string)) { defer func() { if r := recover(); r != nil { - log.ZPanic(ctx, "subscriberRedisDeleteCache Panic", r) + log.ZPanic(ctx, "subscriberRedisDeleteCache Panic", errs.ErrPanic(r)) } }() for message := range client.Subscribe(ctx, channel).Channel() { From 030c4493dcad2f745e218a3a4e233c31a506f1d5 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 27 Dec 2024 17:19:46 +0800 Subject: [PATCH 11/42] fix --- internal/api/jssdk/jssdk.go | 21 ++++++++++++------- internal/api/router.go | 6 ++++-- .../msgtransfer/online_history_msg_handler.go | 2 +- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/internal/api/jssdk/jssdk.go b/internal/api/jssdk/jssdk.go index 036cb027a..3c0911207 100644 --- a/internal/api/jssdk/jssdk.go +++ b/internal/api/jssdk/jssdk.go @@ -20,16 +20,23 @@ const ( defaultGetActiveConversation = 100 ) -func NewJSSdkApi() *JSSdk { - return &JSSdk{} +func NewJSSdkApi(userClient *rpcli.UserClient, relationClient *rpcli.RelationClient, groupClient *rpcli.GroupClient, + conversationClient *rpcli.ConversationClient, msgClient *rpcli.MsgClient) *JSSdk { + return &JSSdk{ + userClient: userClient, + relationClient: relationClient, + groupClient: groupClient, + conversationClient: conversationClient, + msgClient: msgClient, + } } type JSSdk struct { - userClient rpcli.UserClient - relationClient rpcli.RelationClient - groupClient rpcli.GroupClient - conversationClient rpcli.ConversationClient - msgClient rpcli.MsgClient + userClient *rpcli.UserClient + relationClient *rpcli.RelationClient + groupClient *rpcli.GroupClient + conversationClient *rpcli.ConversationClient + msgClient *rpcli.MsgClient } func (x *JSSdk) GetActiveConversations(c *gin.Context) { diff --git a/internal/api/router.go b/internal/api/router.go index 5c4fa2137..e516d8ca8 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -100,8 +100,8 @@ func newGinRouter(ctx context.Context, client discovery.SvcDiscoveryRegistry, cf case BestSpeed: r.Use(gzip.Gzip(gzip.BestSpeed)) } - r.Use(prommetricsGin(), gin.RecoveryWithWriter(gin.DefaultErrorWriter, mw.GinPanicErr), mw.CorsHandler(), mw.GinParseOperationID(), GinParseToken(rpcli.NewAuthClient(authConn))) - j := jssdk.NewJSSdkApi() + r.Use(prommetricsGin(), gin.RecoveryWithWriter(gin.DefaultErrorWriter, mw.GinPanicErr), mw.CorsHandler(), + mw.GinParseOperationID(), GinParseToken(rpcli.NewAuthClient(authConn))) u := NewUserApi(user.NewUserClient(userConn), client, cfg.Discovery.RpcService) { @@ -280,6 +280,8 @@ func newGinRouter(ctx context.Context, client discovery.SvcDiscoveryRegistry, cf } { + j := jssdk.NewJSSdkApi(rpcli.NewUserClient(userConn), rpcli.NewRelationClient(friendConn), + rpcli.NewGroupClient(groupConn), rpcli.NewConversationClient(conversationConn), rpcli.NewMsgClient(msgConn)) jssdk := r.Group("/jssdk") jssdk.POST("/get_conversations", j.GetConversations) jssdk.POST("/get_active_conversations", j.GetActiveConversations) diff --git a/internal/msgtransfer/online_history_msg_handler.go b/internal/msgtransfer/online_history_msg_handler.go index 7b989070b..6334c95fd 100644 --- a/internal/msgtransfer/online_history_msg_handler.go +++ b/internal/msgtransfer/online_history_msg_handler.go @@ -116,7 +116,7 @@ func NewOnlineHistoryRedisConsumerHandler(ctx context.Context, client discovery. och.redisMessageBatches = b och.historyConsumerGroup = historyConsumerGroup - return &och, err + return &och, nil } func (och *OnlineHistoryRedisConsumerHandler) do(ctx context.Context, channelID int, val *batcher.Msg[sarama.ConsumerMessage]) { ctx = mcontext.WithTriggerIDContext(ctx, val.TriggerID()) From f0d88829d86257443081f11e66363a874ed32a87 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Sun, 29 Dec 2024 14:13:39 +0800 Subject: [PATCH 12/42] fix --- go.mod | 4 ++-- go.sum | 4 ++-- pkg/rpccache/online.go | 47 +++--------------------------------------- pkg/rpcli/group.go | 24 +++++++++++++++++++++ 4 files changed, 31 insertions(+), 48 deletions(-) diff --git a/go.mod b/go.mod index 3cbbff3c9..387cb7304 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.72-alpha.68 + github.com/openimsdk/protocol v0.0.72-alpha.69 github.com/openimsdk/tools v0.0.50-alpha.62 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 @@ -42,7 +42,6 @@ require ( github.com/spf13/viper v1.18.2 go.etcd.io/etcd/client/v3 v3.5.13 go.uber.org/automaxprocs v1.5.3 - golang.org/x/exp v0.0.0-20230905200255-921286631fa9 golang.org/x/sync v0.8.0 k8s.io/api v0.31.2 k8s.io/apimachinery v0.31.2 @@ -187,6 +186,7 @@ require ( go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/arch v0.7.0 // indirect + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect golang.org/x/image v0.15.0 // indirect golang.org/x/net v0.29.0 // indirect golang.org/x/oauth2 v0.23.0 // indirect diff --git a/go.sum b/go.sum index 846ef721d..d207325e9 100644 --- a/go.sum +++ b/go.sum @@ -347,8 +347,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.15-alpha.2 h1:5Q8yl8ezy2yx+q8/ucU/t4kJnDfCzNOrkXcDACCqtyM= github.com/openimsdk/gomake v0.0.15-alpha.2/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.72-alpha.68 h1:Ekn6S9Ftt12Xs/p9kJ39RDr2gSwIczz+MmSHQE4lAek= -github.com/openimsdk/protocol v0.0.72-alpha.68/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= +github.com/openimsdk/protocol v0.0.72-alpha.69 h1:b22oY2XTdBR/BePqA73KsrM3GDF3Vk8YcBEXZU4ArJc= +github.com/openimsdk/protocol v0.0.72-alpha.69/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= github.com/openimsdk/tools v0.0.50-alpha.62 h1:e/m1XL7+EXbkOoxr/En/612WcOPKOUHPBj0++gG6MuQ= github.com/openimsdk/tools v0.0.50-alpha.62/go.mod h1:JowL2jYr8tu4vcQe+5hJh4v3BtSx1T0CIS3pgU/Mw+U= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= diff --git a/pkg/rpccache/online.go b/pkg/rpccache/online.go index 8f5323477..32b2f2889 100644 --- a/pkg/rpccache/online.go +++ b/pkg/rpccache/online.go @@ -206,16 +206,6 @@ func (o *OnlineCache) GetUserOnlinePlatform(ctx context.Context, userID string) return platformIDs, nil } -// func (o *OnlineCache) GetUserOnlinePlatformBatch(ctx context.Context, userIDs []string) (map[string]int32, error) { -// platformIDs, err := o.getUserOnlinePlatform(ctx, userIDs) -// if err != nil { -// return nil, err -// } -// tmp := make([]int32, len(platformIDs)) -// copy(tmp, platformIDs) -// return platformIDs, nil -// } - func (o *OnlineCache) GetUserOnline(ctx context.Context, userID string) (bool, error) { platformIDs, err := o.getUserOnlinePlatform(ctx, userID) if err != nil { @@ -225,6 +215,9 @@ func (o *OnlineCache) GetUserOnline(ctx context.Context, userID string) (bool, e } func (o *OnlineCache) getUserOnlinePlatformBatch(ctx context.Context, userIDs []string) (map[string][]int32, error) { + if len(userIDs) == 0 { + return nil, nil + } platformIDsMap, err := o.lruCache.GetBatch(userIDs, func(missingUsers []string) (map[string][]int32, error) { platformIDsMap := make(map[string][]int32) usersStatus, err := o.client.GetUsersOnlinePlatform(ctx, missingUsers) @@ -281,40 +274,6 @@ func (o *OnlineCache) GetUsersOnline(ctx context.Context, userIDs []string) ([]s return userIDs, offlineUserIDs, nil } -//func (o *OnlineCache) GetUsersOnline(ctx context.Context, userIDs []string) ([]string, error) { -// onlineUserIDs := make([]string, 0, len(userIDs)) -// for _, userID := range userIDs { -// online, err := o.GetUserOnline(ctx, userID) -// if err != nil { -// return nil, err -// } -// if online { -// onlineUserIDs = append(onlineUserIDs, userID) -// } -// } -// log.ZDebug(ctx, "OnlineCache GetUsersOnline", "userIDs", userIDs, "onlineUserIDs", onlineUserIDs) -// return onlineUserIDs, nil -//} -// -//func (o *OnlineCache) GetGroupOnline(ctx context.Context, groupID string) ([]string, error) { -// userIDs, err := o.group.GetGroupMemberIDs(ctx, groupID) -// if err != nil { -// return nil, err -// } -// var onlineUserIDs []string -// for _, userID := range userIDs { -// online, err := o.GetUserOnline(ctx, userID) -// if err != nil { -// return nil, err -// } -// if online { -// onlineUserIDs = append(onlineUserIDs, userID) -// } -// } -// log.ZDebug(ctx, "OnlineCache GetGroupOnline", "groupID", groupID, "onlineUserIDs", onlineUserIDs, "allUserID", userIDs) -// return onlineUserIDs, nil -//} - func (o *OnlineCache) setUserOnline(userID string, platformIDs []int32) { switch o.fullUserCache { case true: diff --git a/pkg/rpcli/group.go b/pkg/rpcli/group.go index b51283c5d..623c5cc12 100644 --- a/pkg/rpcli/group.go +++ b/pkg/rpcli/group.go @@ -46,3 +46,27 @@ func (x *GroupClient) GetGroupMemberUserIDs(ctx context.Context, groupID string) req := &group.GetGroupMemberUserIDsReq{GroupID: groupID} return extractField(ctx, x.GroupClient.GetGroupMemberUserIDs, req, (*group.GetGroupMemberUserIDsResp).GetUserIDs) } + +func (x *GroupClient) GetGroupMembersInfo(ctx context.Context, groupID string, userIDs []string) ([]*sdkws.GroupMemberFullInfo, error) { + if len(userIDs) == 0 { + return nil, nil + } + req := &group.GetGroupMembersInfoReq{GroupID: groupID, UserIDs: userIDs} + return extractField(ctx, x.GroupClient.GetGroupMembersInfo, req, (*group.GetGroupMembersInfoResp).GetMembers) +} + +func (x *GroupClient) GetGroupMemberInfo(ctx context.Context, groupID string, userID string) (*sdkws.GroupMemberFullInfo, error) { + return firstValue(x.GetGroupMembersInfo(ctx, groupID, []string{userID})) +} + +func (x *GroupClient) GetGroupMemberMapInfo(ctx context.Context, groupID string, userIDs []string) (map[string]*sdkws.GroupMemberFullInfo, error) { + members, err := x.GetGroupMembersInfo(ctx, groupID, userIDs) + if err != nil { + return nil, err + } + memberMap := make(map[string]*sdkws.GroupMemberFullInfo) + for _, member := range members { + memberMap[member.UserID] = member + } + return memberMap, nil +} From d091d5f4bfda89a4a4693d07401eef7ced972e53 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Mon, 30 Dec 2024 15:53:07 +0800 Subject: [PATCH 13/42] optimize log output --- go.mod | 2 +- go.sum | 4 ++-- internal/msggateway/hub_server.go | 9 --------- internal/push/offlinepush/dummy/push.go | 7 +++++-- 4 files changed, 8 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index 387cb7304..efdac43f8 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.72-alpha.69 - github.com/openimsdk/tools v0.0.50-alpha.62 + github.com/openimsdk/tools v0.0.50-alpha.63 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index d207325e9..f87e038d0 100644 --- a/go.sum +++ b/go.sum @@ -349,8 +349,8 @@ github.com/openimsdk/gomake v0.0.15-alpha.2 h1:5Q8yl8ezy2yx+q8/ucU/t4kJnDfCzNOrk github.com/openimsdk/gomake v0.0.15-alpha.2/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.72-alpha.69 h1:b22oY2XTdBR/BePqA73KsrM3GDF3Vk8YcBEXZU4ArJc= github.com/openimsdk/protocol v0.0.72-alpha.69/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= -github.com/openimsdk/tools v0.0.50-alpha.62 h1:e/m1XL7+EXbkOoxr/En/612WcOPKOUHPBj0++gG6MuQ= -github.com/openimsdk/tools v0.0.50-alpha.62/go.mod h1:JowL2jYr8tu4vcQe+5hJh4v3BtSx1T0CIS3pgU/Mw+U= +github.com/openimsdk/tools v0.0.50-alpha.63 h1:dPoVvg4KWqYX/xtK3j96TwX2A/4jwT5S5XIHvSM9hTY= +github.com/openimsdk/tools v0.0.50-alpha.63/go.mod h1:B+oqV0zdewN7OiEHYJm+hW+8/Te7B8tHHgD8rK5ZLZk= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/internal/msggateway/hub_server.go b/internal/msggateway/hub_server.go index 52afe495b..490711cfe 100644 --- a/internal/msggateway/hub_server.go +++ b/internal/msggateway/hub_server.go @@ -96,10 +96,6 @@ func NewServer(longConnServer LongConnServer, conf *Config, ready func(srv *Serv return s } -func (s *Server) OnlinePushMsg(context context.Context, req *msggateway.OnlinePushMsgReq) (*msggateway.OnlinePushMsgResp, error) { - panic("implement me") -} - func (s *Server) GetUsersOnlineStatus(ctx context.Context, req *msggateway.GetUsersOnlineStatusReq) (*msggateway.GetUsersOnlineStatusResp, error) { if !authverify.IsAppManagerUid(ctx, s.config.Share.IMAdminUserID) { return nil, errs.ErrNoPermission.WrapMsg("only app manager") @@ -133,11 +129,6 @@ func (s *Server) GetUsersOnlineStatus(ctx context.Context, req *msggateway.GetUs return &resp, nil } -func (s *Server) OnlineBatchPushOneMsg(ctx context.Context, req *msggateway.OnlineBatchPushOneMsgReq) (*msggateway.OnlineBatchPushOneMsgResp, error) { - // todo implement - return nil, nil -} - func (s *Server) pushToUser(ctx context.Context, userID string, msgData *sdkws.MsgData) *msggateway.SingleMsgToUserResults { clients, ok := s.LongConnServer.GetUserAllCons(userID) if !ok { diff --git a/internal/push/offlinepush/dummy/push.go b/internal/push/offlinepush/dummy/push.go index 0bccaf4a4..1babbc798 100644 --- a/internal/push/offlinepush/dummy/push.go +++ b/internal/push/offlinepush/dummy/push.go @@ -18,6 +18,7 @@ import ( "context" "github.com/openimsdk/open-im-server/v3/internal/push/offlinepush/options" "github.com/openimsdk/tools/log" + "sync/atomic" ) func NewClient() *Dummy { @@ -25,10 +26,12 @@ func NewClient() *Dummy { } type Dummy struct { + v atomic.Bool } func (d *Dummy) Push(ctx context.Context, userIDs []string, title, content string, opts *options.Opts) error { - log.ZDebug(ctx, "dummy push") - log.ZWarn(ctx, "Dummy push", nil, "ps", "The offline push is not configured. To configure it, please go to config/openim-push.yml.") + if d.v.CompareAndSwap(false, true) { + log.ZWarn(ctx, "dummy push", nil, "ps", "the offline push is not configured. to configure it, please go to config/openim-push.yml") + } return nil } From 3059436c6c684ad5bbdee9c12096014ce644bed2 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 31 Dec 2024 18:18:08 +0800 Subject: [PATCH 14/42] feat: support GetLastMessage --- go.mod | 2 +- go.sum | 4 +- internal/msggateway/client.go | 2 + internal/msggateway/constant.go | 1 + internal/msggateway/message_handler.go | 13 ++++++ internal/rpc/msg/sync_msg.go | 8 ++++ pkg/common/storage/controller/msg.go | 25 ++++++++++- pkg/common/storage/database/mgo/msg.go | 62 ++++++++++++++++++++++++++ pkg/common/storage/database/msg.go | 1 + 9 files changed, 114 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index efdac43f8..03a7a4d4d 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.72-alpha.69 + github.com/openimsdk/protocol v0.0.72-alpha.70 github.com/openimsdk/tools v0.0.50-alpha.63 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index f87e038d0..4c297134d 100644 --- a/go.sum +++ b/go.sum @@ -347,8 +347,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.15-alpha.2 h1:5Q8yl8ezy2yx+q8/ucU/t4kJnDfCzNOrkXcDACCqtyM= github.com/openimsdk/gomake v0.0.15-alpha.2/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.72-alpha.69 h1:b22oY2XTdBR/BePqA73KsrM3GDF3Vk8YcBEXZU4ArJc= -github.com/openimsdk/protocol v0.0.72-alpha.69/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= +github.com/openimsdk/protocol v0.0.72-alpha.70 h1:j7vB81+rTthijRda2b8tlli9oWvPxr4yXHwZ8nPZIBQ= +github.com/openimsdk/protocol v0.0.72-alpha.70/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= github.com/openimsdk/tools v0.0.50-alpha.63 h1:dPoVvg4KWqYX/xtK3j96TwX2A/4jwT5S5XIHvSM9hTY= github.com/openimsdk/tools v0.0.50-alpha.63/go.mod h1:B+oqV0zdewN7OiEHYJm+hW+8/Te7B8tHHgD8rK5ZLZk= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= diff --git a/internal/msggateway/client.go b/internal/msggateway/client.go index 1040f2be2..289076435 100644 --- a/internal/msggateway/client.go +++ b/internal/msggateway/client.go @@ -242,6 +242,8 @@ func (c *Client) handleMessage(message []byte) error { resp, messageErr = c.setAppBackgroundStatus(ctx, binaryReq) case WsSubUserOnlineStatus: resp, messageErr = c.longConnServer.SubUserOnlineStatus(ctx, c, binaryReq) + case WsPullConvLastMessage: + resp, messageErr = c.longConnServer.GetLastMessage(ctx, binaryReq) default: return fmt.Errorf( "ReqIdentifier failed,sendID:%s,msgIncr:%s,reqIdentifier:%d", diff --git a/internal/msggateway/constant.go b/internal/msggateway/constant.go index a825c0519..b77cc44c3 100644 --- a/internal/msggateway/constant.go +++ b/internal/msggateway/constant.go @@ -52,6 +52,7 @@ const ( WsLogoutMsg = 2003 WsSetBackgroundStatus = 2004 WsSubUserOnlineStatus = 2005 + WsPullConvLastMessage = 2006 WSDataError = 3001 ) diff --git a/internal/msggateway/message_handler.go b/internal/msggateway/message_handler.go index 9b59867d6..ca15e1ef6 100644 --- a/internal/msggateway/message_handler.go +++ b/internal/msggateway/message_handler.go @@ -108,6 +108,7 @@ type MessageHandler interface { GetSeqMessage(ctx context.Context, data *Req) ([]byte, error) UserLogout(ctx context.Context, data *Req) ([]byte, error) SetUserDeviceBackground(ctx context.Context, data *Req) ([]byte, bool, error) + GetLastMessage(ctx context.Context, data *Req) ([]byte, error) } var _ MessageHandler = (*GrpcHandler)(nil) @@ -266,3 +267,15 @@ func (g *GrpcHandler) SetUserDeviceBackground(ctx context.Context, data *Req) ([ } return nil, req.IsBackground, nil } + +func (g *GrpcHandler) GetLastMessage(ctx context.Context, data *Req) ([]byte, error) { + var req msg.GetLastMessageReq + if err := proto.Unmarshal(data.Data, &req); err != nil { + return nil, err + } + resp, err := g.msgClient.GetLastMessage(ctx, &req) + if err != nil { + return nil, err + } + return proto.Marshal(resp) +} diff --git a/internal/rpc/msg/sync_msg.go b/internal/rpc/msg/sync_msg.go index 7d4ffa3e6..6cf1c21d3 100644 --- a/internal/rpc/msg/sync_msg.go +++ b/internal/rpc/msg/sync_msg.go @@ -245,3 +245,11 @@ func (m *msgServer) SearchMessage(ctx context.Context, req *msg.SearchMessageReq func (m *msgServer) GetServerTime(ctx context.Context, _ *msg.GetServerTimeReq) (*msg.GetServerTimeResp, error) { return &msg.GetServerTimeResp{ServerTime: timeutil.GetCurrentTimestampByMill()}, nil } + +func (m *msgServer) GetLastMessage(ctx context.Context, req *msg.GetLastMessageReq) (*msg.GetLastMessageResp, error) { + msgs, err := m.MsgDatabase.GetLastMessage(ctx, req.ConversationIDs, req.UserID) + if err != nil { + return nil, err + } + return &msg.GetLastMessageResp{Msgs: msgs}, nil +} diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index d5ad12584..a93d581eb 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -97,6 +97,8 @@ type CommonMsgDatabase interface { DeleteDoc(ctx context.Context, docID string) error GetLastMessageSeqByTime(ctx context.Context, conversationID string, time int64) (int64, error) + + GetLastMessage(ctx context.Context, conversationIDS []string, userID string) (map[string]*sdkws.MsgData, error) } func NewCommonMsgDatabase(msgDocModel database.Msg, msg cache.MsgCache, seqUser cache.SeqUser, seqConversation cache.SeqConversationCache, kafkaConf *config.Kafka) (CommonMsgDatabase, error) { @@ -811,8 +813,29 @@ func (db *commonMsgDatabase) GetMessageBySeqs(ctx context.Context, conversationI if v, ok := seqMsgs[seq]; ok { res = append(res, convert.MsgDB2Pb(v.Msg)) } else { - res = append(res, &sdkws.MsgData{Seq: seq}) + res = append(res, &sdkws.MsgData{Seq: seq, Status: constant.MsgStatusHasDeleted}) } } return res, nil } + +func (db *commonMsgDatabase) GetLastMessage(ctx context.Context, conversationIDs []string, userID string) (map[string]*sdkws.MsgData, error) { + res := make(map[string]*sdkws.MsgData) + for _, conversationID := range conversationIDs { + if _, ok := res[conversationID]; ok { + continue + } + msg, err := db.msgDocDatabase.GetLastMessage(ctx, conversationID) + if err != nil { + if errs.Unwrap(err) == mongo.ErrNoDocuments { + continue + } + return nil, err + } + tmp := []*model.MsgInfoModel{msg} + db.handlerDeleteAndRevoked(ctx, userID, tmp) + db.handlerQuote(ctx, userID, conversationID, tmp) + res[conversationID] = convert.MsgDB2Pb(msg.Msg) + } + return res, nil +} diff --git a/pkg/common/storage/database/mgo/msg.go b/pkg/common/storage/database/mgo/msg.go index 03ebff611..c440d4442 100644 --- a/pkg/common/storage/database/mgo/msg.go +++ b/pkg/common/storage/database/mgo/msg.go @@ -997,6 +997,68 @@ func (m *MsgMgo) GetLastMessageSeqByTime(ctx context.Context, conversationID str return seq, nil } +func (m *MsgMgo) GetLastMessage(ctx context.Context, conversationID string) (*model.MsgInfoModel, error) { + pipeline := []bson.M{ + { + "$match": bson.M{ + "doc_id": bson.M{ + "$regex": fmt.Sprintf("^%s", conversationID), + }, + }, + }, + { + "$match": bson.M{ + "msgs.msg.status": bson.M{ + "$lt": constant.MsgStatusHasDeleted, + }, + }, + }, + { + "$sort": bson.M{ + "_id": -1, + }, + }, + { + "$limit": 1, + }, + { + "$project": bson.M{ + "_id": 0, + "doc_id": 0, + }, + }, + { + "$unwind": "$msgs", + }, + { + "$match": bson.M{ + "msgs.msg.status": bson.M{ + "$lt": constant.MsgStatusHasDeleted, + }, + }, + }, + { + "$sort": bson.M{ + "msgs.msg.seq": -1, + }, + }, + { + "$limit": 1, + }, + } + type Result struct { + Msgs *model.MsgInfoModel `bson:"msgs"` + } + res, err := mongoutil.Aggregate[*Result](ctx, m.coll, pipeline) + if err != nil { + return nil, err + } + if len(res) == 0 { + return nil, errs.Wrap(mongo.ErrNoDocuments) + } + return res[0].Msgs, nil +} + func (m *MsgMgo) onlyFindDocIndex(ctx context.Context, docID string, indexes []int64) ([]*model.MsgInfoModel, error) { if len(indexes) == 0 { return nil, nil diff --git a/pkg/common/storage/database/msg.go b/pkg/common/storage/database/msg.go index b44e70296..e3c4e8ece 100644 --- a/pkg/common/storage/database/msg.go +++ b/pkg/common/storage/database/msg.go @@ -39,5 +39,6 @@ type Msg interface { DeleteDoc(ctx context.Context, docID string) error GetRandBeforeMsg(ctx context.Context, ts int64, limit int) ([]*model.MsgDocModel, error) GetLastMessageSeqByTime(ctx context.Context, conversationID string, time int64) (int64, error) + GetLastMessage(ctx context.Context, conversationID string) (*model.MsgInfoModel, error) FindSeqs(ctx context.Context, conversationID string, seqs []int64) ([]*model.MsgInfoModel, error) } From f3d5634e569bebfa9e493a48f2cb7957a55d60c4 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 31 Dec 2024 18:21:53 +0800 Subject: [PATCH 15/42] feat: support GetLastMessage --- internal/msggateway/client.go | 4 ++-- internal/msggateway/constant.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/msggateway/client.go b/internal/msggateway/client.go index 289076435..bdb62aece 100644 --- a/internal/msggateway/client.go +++ b/internal/msggateway/client.go @@ -236,14 +236,14 @@ func (c *Client) handleMessage(message []byte) error { resp, messageErr = c.longConnServer.GetSeqMessage(ctx, binaryReq) case WSGetConvMaxReadSeq: resp, messageErr = c.longConnServer.GetConversationsHasReadAndMaxSeq(ctx, binaryReq) + case WsPullConvLastMessage: + resp, messageErr = c.longConnServer.GetLastMessage(ctx, binaryReq) case WsLogoutMsg: resp, messageErr = c.longConnServer.UserLogout(ctx, binaryReq) case WsSetBackgroundStatus: resp, messageErr = c.setAppBackgroundStatus(ctx, binaryReq) case WsSubUserOnlineStatus: resp, messageErr = c.longConnServer.SubUserOnlineStatus(ctx, c, binaryReq) - case WsPullConvLastMessage: - resp, messageErr = c.longConnServer.GetLastMessage(ctx, binaryReq) default: return fmt.Errorf( "ReqIdentifier failed,sendID:%s,msgIncr:%s,reqIdentifier:%d", diff --git a/internal/msggateway/constant.go b/internal/msggateway/constant.go index b77cc44c3..1e7ab3bb7 100644 --- a/internal/msggateway/constant.go +++ b/internal/msggateway/constant.go @@ -47,12 +47,12 @@ const ( WSSendSignalMsg = 1004 WSPullMsg = 1005 WSGetConvMaxReadSeq = 1006 + WsPullConvLastMessage = 1007 WSPushMsg = 2001 WSKickOnlineMsg = 2002 WsLogoutMsg = 2003 WsSetBackgroundStatus = 2004 WsSubUserOnlineStatus = 2005 - WsPullConvLastMessage = 2006 WSDataError = 3001 ) From 819997adeb03e2461963eb59e4e0dffbbe125bb5 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 2 Jan 2025 16:26:48 +0800 Subject: [PATCH 16/42] feat: s3 switch --- pkg/common/storage/database/mgo/object.go | 12 ++ pkg/common/storage/database/object.go | 4 + tools/s3/internal/conversion.go | 202 ++++++++++++++++++++++ tools/s3/main.go | 23 +++ 4 files changed, 241 insertions(+) create mode 100644 tools/s3/internal/conversion.go create mode 100644 tools/s3/main.go diff --git a/pkg/common/storage/database/mgo/object.go b/pkg/common/storage/database/mgo/object.go index 624eb84a0..ee48c6693 100644 --- a/pkg/common/storage/database/mgo/object.go +++ b/pkg/common/storage/database/mgo/object.go @@ -112,3 +112,15 @@ func (o *S3Mongo) FindExpirationObject(ctx context.Context, engine string, expir func (o *S3Mongo) GetKeyCount(ctx context.Context, engine string, key string) (int64, error) { return mongoutil.Count(ctx, o.coll, bson.M{"engine": engine, "key": key}) } + +func (o *S3Mongo) GetEngineCount(ctx context.Context, engine string) (int64, error) { + return mongoutil.Count(ctx, o.coll, bson.M{"engine": engine}) +} + +func (o *S3Mongo) GetEngineInfo(ctx context.Context, engine string, limit int, skip int) ([]*model.Object, error) { + return mongoutil.Find[*model.Object](ctx, o.coll, bson.M{"engine": engine}, options.Find().SetLimit(int64(limit)).SetSkip(int64(skip))) +} + +func (o *S3Mongo) UpdateEngine(ctx context.Context, oldEngine, oldName string, newEngine string) error { + return mongoutil.UpdateOne(ctx, o.coll, bson.M{"engine": oldEngine, "name": oldName}, bson.M{"$set": bson.M{"engine": newEngine}}, false) +} diff --git a/pkg/common/storage/database/object.go b/pkg/common/storage/database/object.go index 5541a159b..a0e4ebe2b 100644 --- a/pkg/common/storage/database/object.go +++ b/pkg/common/storage/database/object.go @@ -27,4 +27,8 @@ type ObjectInfo interface { Delete(ctx context.Context, engine string, name []string) error FindExpirationObject(ctx context.Context, engine string, expiration time.Time, needDelType []string, count int64) ([]*model.Object, error) GetKeyCount(ctx context.Context, engine string, key string) (int64, error) + + GetEngineCount(ctx context.Context, engine string) (int64, error) + GetEngineInfo(ctx context.Context, engine string, limit int, skip int) ([]*model.Object, error) + UpdateEngine(ctx context.Context, oldEngine, oldName string, newEngine string) error } diff --git a/tools/s3/internal/conversion.go b/tools/s3/internal/conversion.go new file mode 100644 index 000000000..ba2174535 --- /dev/null +++ b/tools/s3/internal/conversion.go @@ -0,0 +1,202 @@ +package internal + +import ( + "context" + "errors" + "fmt" + "github.com/mitchellh/mapstructure" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" + "github.com/openimsdk/tools/db/mongoutil" + "github.com/openimsdk/tools/db/redisutil" + "github.com/openimsdk/tools/s3" + "github.com/openimsdk/tools/s3/aws" + "github.com/openimsdk/tools/s3/cos" + "github.com/openimsdk/tools/s3/kodo" + "github.com/openimsdk/tools/s3/minio" + "github.com/openimsdk/tools/s3/oss" + "github.com/spf13/viper" + "go.mongodb.org/mongo-driver/mongo" + "log" + "net/http" + "path/filepath" + "time" +) + +const defaultTimeout = time.Second * 10 + +func readConf(path string, val any) error { + v := viper.New() + v.SetConfigFile(path) + if err := v.ReadInConfig(); err != nil { + return err + } + fn := func(config *mapstructure.DecoderConfig) { + config.TagName = "mapstructure" + } + return v.Unmarshal(val, fn) +} + +func getS3(path string, name string, thirdConf *config.Third) (s3.Interface, error) { + switch name { + case "minio": + ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout) + defer cancel() + var minioConf config.Minio + if err := readConf(filepath.Join(path, minioConf.GetConfigFileName()), &minioConf); err != nil { + return nil, err + } + var redisConf config.Redis + if err := readConf(filepath.Join(path, redisConf.GetConfigFileName()), &redisConf); err != nil { + return nil, err + } + rdb, err := redisutil.NewRedisClient(ctx, redisConf.Build()) + if err != nil { + return nil, err + } + return minio.NewMinio(ctx, redis.NewMinioCache(rdb), *minioConf.Build()) + case "cos": + return cos.NewCos(*thirdConf.Object.Cos.Build()) + case "oss": + return oss.NewOSS(*thirdConf.Object.Oss.Build()) + case "kodo": + return kodo.NewKodo(*thirdConf.Object.Kodo.Build()) + case "aws": + return aws.NewAws(*thirdConf.Object.Aws.Build()) + default: + return nil, fmt.Errorf("invalid object enable: %s", name) + } +} + +func getMongo(path string) (database.ObjectInfo, error) { + var mongoConf config.Mongo + if err := readConf(filepath.Join(path, mongoConf.GetConfigFileName()), &mongoConf); err != nil { + return nil, err + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout) + defer cancel() + mgocli, err := mongoutil.NewMongoDB(ctx, mongoConf.Build()) + if err != nil { + return nil, err + } + return mgo.NewS3Mongo(mgocli.GetDB()) +} + +func Main(path string, engine string) error { + var thirdConf config.Third + if err := readConf(filepath.Join(path, thirdConf.GetConfigFileName()), &thirdConf); err != nil { + return err + } + if thirdConf.Object.Enable == engine { + return errors.New("same s3 storage") + } + s3db, err := getMongo(path) + if err != nil { + return err + } + oldS3, err := getS3(path, engine, &thirdConf) + if err != nil { + return err + } + newS3, err := getS3(path, thirdConf.Object.Enable, &thirdConf) + if err != nil { + return err + } + count, err := getEngineCount(s3db, oldS3.Engine()) + if err != nil { + return err + } + log.Printf("engine %s count: %d", oldS3.Engine(), count) + var skip int + for i := 1; i <= count+1; i++ { + log.Printf("start %d/%d", i, count) + start := time.Now() + res, err := doObject(s3db, newS3, oldS3, skip) + if err != nil { + log.Printf("end [%s] %d/%d error %s", time.Since(start), i, count, err) + return err + } + log.Printf("end [%s] %d/%d result %+v", time.Since(start), i, count, *res) + if res.Skip { + skip++ + } + if res.End { + break + } + } + return nil +} + +func getEngineCount(db database.ObjectInfo, name string) (int, error) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout) + defer cancel() + count, err := db.GetEngineCount(ctx, name) + if err != nil { + return 0, err + } + return int(count), nil +} + +func doObject(db database.ObjectInfo, newS3, oldS3 s3.Interface, skip int) (*Result, error) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout) + defer cancel() + infos, err := db.GetEngineInfo(ctx, oldS3.Engine(), 1, skip) + if err != nil { + return nil, err + } + if len(infos) == 0 { + return &Result{End: true}, nil + } + obj := infos[0] + if _, err := db.Take(ctx, newS3.Engine(), obj.Name); err == nil { + return &Result{Skip: true}, nil + } else if !errors.Is(err, mongo.ErrNoDocuments) { + return nil, err + } + downloadURL, err := oldS3.AccessURL(ctx, obj.Key, time.Hour, &s3.AccessURLOption{}) + if err != nil { + return nil, err + } + putURL, err := newS3.PresignedPutObject(ctx, obj.Key, time.Hour) + if err != nil { + return nil, err + } + downloadResp, err := http.Get(downloadURL) + if err != nil { + return nil, err + } + defer downloadResp.Body.Close() + switch downloadResp.StatusCode { + case http.StatusNotFound: + return &Result{Skip: true}, nil + case http.StatusOK: + default: + return nil, fmt.Errorf("download object failed %s", downloadResp.Status) + } + log.Printf("file size %d", obj.Size) + request, err := http.NewRequest(http.MethodPut, putURL, downloadResp.Body) + if err != nil { + return nil, err + } + putResp, err := http.DefaultClient.Do(request) + if err != nil { + return nil, err + } + defer putResp.Body.Close() + if putResp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("put object failed %s", putResp.Status) + } + ctx, cancel = context.WithTimeout(context.Background(), defaultTimeout) + defer cancel() + if err := db.UpdateEngine(ctx, obj.Engine, obj.Name, newS3.Engine()); err != nil { + return nil, err + } + return &Result{}, nil +} + +type Result struct { + Skip bool + End bool +} diff --git a/tools/s3/main.go b/tools/s3/main.go new file mode 100644 index 000000000..1e661c9a7 --- /dev/null +++ b/tools/s3/main.go @@ -0,0 +1,23 @@ +package main + +import ( + "flag" + "fmt" + "github.com/openimsdk/open-im-server/v3/tools/s3/internal" + "os" +) + +func main() { + var ( + name string + config string + ) + flag.StringVar(&name, "name", "", "old previous storage name") + flag.StringVar(&config, "config", "", "config directory") + flag.Parse() + if err := internal.Main(config, name); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + fmt.Fprintln(os.Stdout, "success") +} From 755520aa371cba62ab5c7cb3652e67d0288b6b35 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 2 Jan 2025 17:09:28 +0800 Subject: [PATCH 17/42] feat: s3 switch --- tools/s3/README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 tools/s3/README.md diff --git a/tools/s3/README.md b/tools/s3/README.md new file mode 100644 index 000000000..ac30347d8 --- /dev/null +++ b/tools/s3/README.md @@ -0,0 +1,12 @@ +# After s3 switches the storage engine, convert the data + +- build +```shell +go build -o s3convert main.go +``` + +- start +```shell +./s3convert -config -name +# ./s3convert -config ./../../config -name minio +``` From 7a8346bdd312b1a2b22f4c1b718ff36182da090c Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 7 Jan 2025 11:21:31 +0800 Subject: [PATCH 18/42] fix: GetUsersOnline --- pkg/rpccache/online.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/rpccache/online.go b/pkg/rpccache/online.go index 32b2f2889..b5308bbe8 100644 --- a/pkg/rpccache/online.go +++ b/pkg/rpccache/online.go @@ -270,8 +270,8 @@ func (o *OnlineCache) GetUsersOnline(ctx context.Context, userIDs []string) ([]s } } - log.ZInfo(ctx, "get users online", "online users length", len(userIDs), "offline users length", len(offlineUserIDs), "cost", time.Since(t)) - return userIDs, offlineUserIDs, nil + log.ZInfo(ctx, "get users online", "online users length", len(onlineUserIDs), "offline users length", len(offlineUserIDs), "cost", time.Since(t)) + return onlineUserIDs, offlineUserIDs, nil } func (o *OnlineCache) setUserOnline(userID string, platformIDs []int32) { From 67586614b8b28e4220a6fb8602ec29646e58fffe Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 9 Jan 2025 16:54:17 +0800 Subject: [PATCH 19/42] feat: SendBusinessNotification supported configuration parameters --- internal/api/msg.go | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/internal/api/msg.go b/internal/api/msg.go index fc235354c..2fdbd7483 100644 --- a/internal/api/msg.go +++ b/internal/api/msg.go @@ -248,16 +248,21 @@ func (m *MessageApi) SendMessage(c *gin.Context) { func (m *MessageApi) SendBusinessNotification(c *gin.Context) { req := struct { - Key string `json:"key"` - Data string `json:"data"` - SendUserID string `json:"sendUserID" binding:"required"` - RecvUserID string `json:"recvUserID" binding:"required"` + Key string `json:"key"` + Data string `json:"data"` + SendUserID string `json:"sendUserID" binding:"required"` + RecvUserID string `json:"recvUserID" binding:"required"` + SendMsg bool `json:"sendMsg"` + ReliabilityLevel *int `json:"reliabilityLevel"` + UnreadCount bool `json:"unreadCount"` }{} if err := c.BindJSON(&req); err != nil { apiresp.GinError(c, errs.ErrArgs.WithDetail(err.Error()).Wrap()) return } - + if req.ReliabilityLevel == nil { + req.ReliabilityLevel = datautil.ToPtr(1) + } if !authverify.IsAppManagerUid(c, m.imAdminUserID) { apiresp.GinError(c, errs.ErrNoPermission.WrapMsg("only app manager can send message")) return @@ -278,9 +283,9 @@ func (m *MessageApi) SendBusinessNotification(c *gin.Context) { CreateTime: timeutil.GetCurrentTimestampByMill(), ClientMsgID: idutil.GetMsgIDByMD5(mcontext.GetOpUserID(c)), Options: config.GetOptionsByNotification(config.NotificationConfig{ - IsSendMsg: false, - ReliabilityLevel: 1, - UnreadCount: false, + IsSendMsg: req.SendMsg, + ReliabilityLevel: *req.ReliabilityLevel, + UnreadCount: req.UnreadCount, }), }, } From 05528afbbb57221f4cbfc29204b9f32702e9e1cc Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 9 Jan 2025 17:04:47 +0800 Subject: [PATCH 20/42] feat: SendBusinessNotification supported configuration parameters --- internal/api/msg.go | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/internal/api/msg.go b/internal/api/msg.go index 2fdbd7483..2af524a73 100644 --- a/internal/api/msg.go +++ b/internal/api/msg.go @@ -251,7 +251,8 @@ func (m *MessageApi) SendBusinessNotification(c *gin.Context) { Key string `json:"key"` Data string `json:"data"` SendUserID string `json:"sendUserID" binding:"required"` - RecvUserID string `json:"recvUserID" binding:"required"` + RecvUserID string `json:"recvUserID"` + RecvGroupID string `json:"recvGroupID"` SendMsg bool `json:"sendMsg"` ReliabilityLevel *int `json:"reliabilityLevel"` UnreadCount bool `json:"unreadCount"` @@ -260,6 +261,20 @@ func (m *MessageApi) SendBusinessNotification(c *gin.Context) { apiresp.GinError(c, errs.ErrArgs.WithDetail(err.Error()).Wrap()) return } + if req.RecvUserID == "" && req.RecvGroupID == "" { + apiresp.GinError(c, errs.ErrArgs.WrapMsg("recvUserID and recvGroupID cannot be empty at the same time")) + return + } + if req.RecvUserID != "" && req.RecvGroupID != "" { + apiresp.GinError(c, errs.ErrArgs.WrapMsg("recvUserID and recvGroupID cannot be set at the same time")) + return + } + var sessionType int32 + if req.RecvUserID != "" { + sessionType = constant.SingleChatType + } else { + sessionType = constant.ReadGroupChatType + } if req.ReliabilityLevel == nil { req.ReliabilityLevel = datautil.ToPtr(1) } @@ -269,8 +284,9 @@ func (m *MessageApi) SendBusinessNotification(c *gin.Context) { } sendMsgReq := msg.SendMsgReq{ MsgData: &sdkws.MsgData{ - SendID: req.SendUserID, - RecvID: req.RecvUserID, + SendID: req.SendUserID, + RecvID: req.RecvUserID, + GroupID: req.RecvGroupID, Content: []byte(jsonutil.StructToJsonString(&sdkws.NotificationElem{ Detail: jsonutil.StructToJsonString(&struct { Key string `json:"key"` @@ -279,7 +295,7 @@ func (m *MessageApi) SendBusinessNotification(c *gin.Context) { })), MsgFrom: constant.SysMsgType, ContentType: constant.BusinessNotification, - SessionType: constant.SingleChatType, + SessionType: sessionType, CreateTime: timeutil.GetCurrentTimestampByMill(), ClientMsgID: idutil.GetMsgIDByMD5(mcontext.GetOpUserID(c)), Options: config.GetOptionsByNotification(config.NotificationConfig{ From e94ce825d6611fe59346d7f55e81c794275a5fa4 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 9 Jan 2025 17:13:33 +0800 Subject: [PATCH 21/42] feat: SendBusinessNotification supported configuration parameters --- internal/api/msg.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/api/msg.go b/internal/api/msg.go index 2af524a73..b21e792db 100644 --- a/internal/api/msg.go +++ b/internal/api/msg.go @@ -255,7 +255,6 @@ func (m *MessageApi) SendBusinessNotification(c *gin.Context) { RecvGroupID string `json:"recvGroupID"` SendMsg bool `json:"sendMsg"` ReliabilityLevel *int `json:"reliabilityLevel"` - UnreadCount bool `json:"unreadCount"` }{} if err := c.BindJSON(&req); err != nil { apiresp.GinError(c, errs.ErrArgs.WithDetail(err.Error()).Wrap()) @@ -301,7 +300,7 @@ func (m *MessageApi) SendBusinessNotification(c *gin.Context) { Options: config.GetOptionsByNotification(config.NotificationConfig{ IsSendMsg: req.SendMsg, ReliabilityLevel: *req.ReliabilityLevel, - UnreadCount: req.UnreadCount, + UnreadCount: false, }), }, } From f31d196ae89601f0690391efd843e10f2a1bcbb7 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 10 Jan 2025 10:59:20 +0800 Subject: [PATCH 22/42] feat: seq conversion failed without exiting --- tools/seq/main.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/seq/main.go b/tools/seq/main.go index 16da9f156..ef2875fe1 100644 --- a/tools/seq/main.go +++ b/tools/seq/main.go @@ -3,8 +3,10 @@ package main import ( "flag" "fmt" - "github.com/openimsdk/open-im-server/v3/tools/seq/internal" + "os" "time" + + "github.com/openimsdk/open-im-server/v3/tools/seq/internal" ) func main() { @@ -17,6 +19,8 @@ func main() { flag.Parse() if err := internal.Main(config, time.Duration(second)*time.Second); err != nil { fmt.Println("seq task", err) + os.Exit(1) + return } fmt.Println("seq task success!") } From 961f43c077b27efefe867721216dcb72c0020590 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 22 Jan 2025 14:22:30 +0800 Subject: [PATCH 23/42] fix: DeleteDoc crash --- pkg/common/storage/controller/msg.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index a93d581eb..b92f9b510 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -18,11 +18,12 @@ import ( "context" "encoding/json" "errors" - "github.com/openimsdk/tools/utils/jsonutil" "strconv" "strings" "time" + "github.com/openimsdk/tools/utils/jsonutil" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" @@ -722,13 +723,13 @@ func (db *commonMsgDatabase) DeleteDoc(ctx context.Context, docID string) error if index <= 0 { return errs.ErrInternalServer.WrapMsg("docID is invalid", "docID", docID) } - index, err := strconv.Atoi(docID[index+1:]) + docIndex, err := strconv.Atoi(docID[index+1:]) if err != nil { return errs.WrapMsg(err, "strconv.Atoi", "docID", docID) } conversationID := docID[:index] seqs := make([]int64, db.msgTable.GetSingleGocMsgNum()) - minSeq := db.msgTable.GetMinSeq(index) + minSeq := db.msgTable.GetMinSeq(docIndex) for i := range seqs { seqs[i] = minSeq + int64(i) } From 96baa5a0ffac3d7c4d4727876ffe418ec6be4e3b Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Wed, 22 Jan 2025 14:38:27 +0800 Subject: [PATCH 24/42] fix: check error in BatchSetTokenMapByUidPid (#3076) --- pkg/common/storage/controller/auth.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/pkg/common/storage/controller/auth.go b/pkg/common/storage/controller/auth.go index 2885b985a..ee2a06f53 100644 --- a/pkg/common/storage/controller/auth.go +++ b/pkg/common/storage/controller/auth.go @@ -60,16 +60,15 @@ func (a *authDatabase) BatchSetTokenMapByUidPid(ctx context.Context, tokens []st setMap := make(map[string]map[string]any) for _, token := range tokens { claims, err := tokenverify.GetClaimFromToken(token, authverify.Secret(a.accessSecret)) - key := cachekey.GetTokenKey(claims.UserID, claims.PlatformID) if err != nil { continue + } + key := cachekey.GetTokenKey(claims.UserID, claims.PlatformID) + if v, ok := setMap[key]; ok { + v[token] = constant.KickedToken } else { - if v, ok := setMap[key]; ok { - v[token] = constant.KickedToken - } else { - setMap[key] = map[string]any{ - token: constant.KickedToken, - } + setMap[key] = map[string]any{ + token: constant.KickedToken, } } } From bbb5473d26a847a4a35b01a3c9d70da16234591d Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Wed, 22 Jan 2025 15:24:40 +0800 Subject: [PATCH 25/42] fix: DeleteDoc crash (#3078) * pb * fix: Modifying other fields while setting IsPrivateChat does not take effect * fix: quote message error revoke * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * upgrading pkg tools * fix * fix * optimize log output * feat: support GetLastMessage * feat: support GetLastMessage * feat: s3 switch * feat: s3 switch * fix: GetUsersOnline * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: seq conversion failed without exiting * fix: DeleteDoc crash --- pkg/common/storage/controller/msg.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index a93d581eb..b92f9b510 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -18,11 +18,12 @@ import ( "context" "encoding/json" "errors" - "github.com/openimsdk/tools/utils/jsonutil" "strconv" "strings" "time" + "github.com/openimsdk/tools/utils/jsonutil" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" @@ -722,13 +723,13 @@ func (db *commonMsgDatabase) DeleteDoc(ctx context.Context, docID string) error if index <= 0 { return errs.ErrInternalServer.WrapMsg("docID is invalid", "docID", docID) } - index, err := strconv.Atoi(docID[index+1:]) + docIndex, err := strconv.Atoi(docID[index+1:]) if err != nil { return errs.WrapMsg(err, "strconv.Atoi", "docID", docID) } conversationID := docID[:index] seqs := make([]int64, db.msgTable.GetSingleGocMsgNum()) - minSeq := db.msgTable.GetMinSeq(index) + minSeq := db.msgTable.GetMinSeq(docIndex) for i := range seqs { seqs[i] = minSeq + int64(i) } From 83c7b7134c78dffdc470a9a19d6e4be4fb0e7c5e Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Thu, 23 Jan 2025 15:18:39 +0800 Subject: [PATCH 26/42] refactor: improve workflows logic. (#3072) * refactor: improve workflows logic. * update args. * remove unused contents. * update milestone merge contents. * update contents. --- ...cker-build-and-release-services-images.yml | 13 +- .github/workflows/go-build-test.yml | 137 +++++++++---- .github/workflows/merge-from-milestone.yml | 180 ++++++------------ .github/workflows/publish-docker-image.yml | 83 ++++---- 4 files changed, 197 insertions(+), 216 deletions(-) diff --git a/.github/workflows/docker-build-and-release-services-images.yml b/.github/workflows/docker-build-and-release-services-images.yml index ed4c7ad38..407589f1e 100644 --- a/.github/workflows/docker-build-and-release-services-images.yml +++ b/.github/workflows/docker-build-and-release-services-images.yml @@ -19,26 +19,26 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3.8.0 - name: Log in to Docker Hub - uses: docker/login-action@v2 + uses: docker/login-action@v3.3.0 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Log in to GitHub Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3.3.0 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Log in to Aliyun Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3.3.0 with: registry: registry.cn-hangzhou.aliyuncs.com username: ${{ secrets.ALIREGISTRY_USERNAME }} @@ -46,7 +46,7 @@ jobs: - name: Extract metadata for Docker (tags, labels) id: meta - uses: docker/metadata-action@v5 + uses: docker/metadata-action@v5.6.0 with: tags: | type=ref,event=tag @@ -54,7 +54,6 @@ jobs: type=ref,event=branch type=semver,pattern={{version}} type=semver,pattern=v{{version}} - # type=semver,pattern={{major}}.{{minor}} type=semver,pattern=release-{{raw}} type=sha type=raw,value=${{ github.event.inputs.tag }} diff --git a/.github/workflows/go-build-test.yml b/.github/workflows/go-build-test.yml index 10a4154d6..4033603e6 100644 --- a/.github/workflows/go-build-test.yml +++ b/.github/workflows/go-build-test.yml @@ -4,7 +4,7 @@ on: push: pull_request: paths-ignore: - - '**/*.md' + - "**/*.md" workflow_dispatch: @@ -18,7 +18,7 @@ jobs: strategy: matrix: os: [ubuntu-latest] - go_version: ["1.21.x", "1.22.x"] + go_version: ["1.22.x"] steps: - name: Checkout Server repository @@ -37,27 +37,20 @@ jobs: - name: Set up infra services uses: hoverkraft-tech/compose-action@v2.0.1 - # Uncomment and set the correct path to your docker-compose file with: compose-file: "./docker-compose.yml" - # run: | - # sudo docker compose up -d - # sudo sleep 30 # Increased sleep time for better stability - # timeout-minutes: 60 # Increased timeout for Docker setup + # - name: Get Internal IP Address + # id: get-ip + # run: | + # IP=$(hostname -I | awk '{print $1}') + # echo "The IP Address is: $IP" + # echo "::set-output name=ip::$IP" - - # - name: Get Internal IP Address - # id: get-ip - # run: | - # IP=$(hostname -I | awk '{print $1}') - # echo "The IP Address is: $IP" - # echo "::set-output name=ip::$IP" - - # - name: Update .env - # run: | - # sed -i 's|externalAddress:.*|externalAddress: "http://${{ steps.get-ip.outputs.ip }}:10005"|' config/minio.yml - # cat config/minio.yml + # - name: Update .env + # run: | + # sed -i 's|externalAddress:.*|externalAddress: "http://${{ steps.get-ip.outputs.ip }}:10005"|' config/minio.yml + # cat config/minio.yml - name: Build and test Server Services run: | @@ -85,6 +78,90 @@ jobs: mage start mage check + - name: Test Server and Chat + run: | + check_error() { + echo "Response: $1" + errCode=$(echo $1 | jq -r '.errCode') + if [ "$errCode" != "0" ]; then + errMsg=$(echo $1 | jq -r '.errMsg') + echo "Error: $errMsg" + exit 1 + fi + } + + # Test register + response1=$(curl -X POST -H "Content-Type: application/json" -H "operationID: imAdmin" -d '{ + "verifyCode": "666666", + "platform": 3, + "autoLogin": true, + "user":{ + "nickname": "test12312", + "areaCode":"+86", + "phoneNumber": "12345678190", + "password":"test123456" + } + }' http://127.0.0.1:10008/account/register) + check_error "$response1" + userID1=$(echo $response1 | jq -r '.data.userID') + echo "userID1: $userID1" + + response2=$(curl -X POST -H "Content-Type: application/json" -H "operationID: imAdmin" -d '{ + "verifyCode": "666666", + "platform": 3, + "autoLogin": true, + "user":{ + "nickname": "test22312", + "areaCode":"+86", + "phoneNumber": "12345678290", + "password":"test123456" + } + }' http://127.0.0.1:10008/account/register) + check_error "$response2" + userID2=$(echo $response2 | jq -r '.data.userID') + echo "userID2: $userID2" + + # Test login + login_response=$(curl -X POST -H "Content-Type: application/json" -H "operationID: imAdmin" -d '{ + "platform": 3, + "areaCode":"+86", + "phoneNumber": "12345678190", + "password":"test123456" + }' http://localhost:10008/account/login) + check_error "$login_response" + + # Test get admin token + get_admin_token_response=$(curl -X POST -H "Content-Type: application/json" -H "operationID: imAdmin" -d '{ + "secret": "openIM123", + "platformID": 2, + "userID": "imAdmin" + }' http://127.0.0.1:10002/auth/get_admin_token) + check_error "$get_admin_token_response" + adminToken=$(echo $get_admin_token_response | jq -r '.data.token') + echo "adminToken: $adminToken" + + # Test send message + send_msg_response=$(curl -X POST -H "Content-Type: application/json" -H "operationID: imAdmin" -H "token: $adminToken" -d '{ + "sendID": "'$userID1'", + "recvID": "'$userID2'", + "senderPlatformID": 3, + "content": { + "content": "hello!!" + }, + "contentType": 101, + "sessionType": 1 + }' http://127.0.0.1:10002/msg/send_msg) + check_error "$send_msg_response" + + # Test get users + get_users_response=$(curl -X POST -H "Content-Type: application/json" -H "operationID: imAdmin" -H "token: $adminToken" -d '{ + "pagination": { + "pageNumber": 1, + "showNumber": 100 + } + }' http://127.0.0.1:10002/user/get_users) + check_error "$get_users_response" + go-test: name: Benchmark Test with go ${{ matrix.go_version }} on ${{ matrix.os }} runs-on: ${{ matrix.os }} @@ -93,11 +170,11 @@ jobs: env: SDK_DIR: openim-sdk-core CONFIG_PATH: config/notification.yml - # pull-requests: write + strategy: matrix: - os: [ ubuntu-latest ] - go_version: [ "1.22.x" ] + os: [ubuntu-latest] + go_version: ["1.22.x"] steps: - name: Checkout Server repository @@ -106,7 +183,8 @@ jobs: - name: Checkout SDK repository uses: actions/checkout@v4 with: - repository: 'openimsdk/openim-sdk-core' + repository: "openimsdk/openim-sdk-core" + ref: "release-v3.8" path: ${{ env.SDK_DIR }} - name: Set up Go ${{ matrix.go_version }} @@ -119,11 +197,6 @@ jobs: go install github.com/magefile/mage@latest go mod download - - name: Install yq - run: | - sudo wget https://github.com/mikefarah/yq/releases/download/v4.34.1/yq_linux_amd64 -O /usr/bin/yq - sudo chmod +x /usr/bin/yq - - name: Modify Server Configuration run: | yq e '.groupCreated.isSendMsg = true' -i ${{ env.CONFIG_PATH }} @@ -183,11 +256,3 @@ jobs: run: | CONTAINER_NAME="${{ github.event.repository.name }}-container" docker logs $CONTAINER_NAME - - # - name: Cleanup Docker Container - # run: | - # CONTAINER_NAME="${{ github.event.repository.name }}-container" - # IMAGE_NAME="${{ github.event.repository.name }}-test" - # docker stop $CONTAINER_NAME - # docker rm $CONTAINER_NAME - # docker rmi $IMAGE_NAME diff --git a/.github/workflows/merge-from-milestone.yml b/.github/workflows/merge-from-milestone.yml index 44b4f81f4..8e09c9ab7 100644 --- a/.github/workflows/merge-from-milestone.yml +++ b/.github/workflows/merge-from-milestone.yml @@ -1,4 +1,4 @@ -name: Create Pre-Release PR from Milestone +name: Create Individual PRs from Milestone permissions: contents: write @@ -9,24 +9,24 @@ on: workflow_dispatch: inputs: milestone_name: - description: 'Milestone name to collect closed PRs from' + description: "Milestone name to collect closed PRs from" required: true - default: 'v3.8.2' + default: "v3.8.4" target_branch: - description: 'Target branch to merge the consolidated PR' + description: "Target branch to merge the consolidated PR" required: true - default: 'pre-release-v3.8.2' + default: "pre-release-v3.8.4" env: - MILESTONE_NAME: ${{ github.event.inputs.milestone_name || 'v3.8.2' }} - TARGET_BRANCH: ${{ github.event.inputs.target_branch || 'pre-release-v3.8.2' }} + MILESTONE_NAME: ${{ github.event.inputs.milestone_name || 'v3.8.4' }} + TARGET_BRANCH: ${{ github.event.inputs.target_branch || 'pre-release-v3.8.4' }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} BOT_TOKEN: ${{ secrets.BOT_TOKEN }} LABEL_NAME: cherry-picked - TEMP_DIR: /tmp # Using /tmp as the temporary directory + TEMP_DIR: /tmp jobs: - cherry_pick_milestone_prs: + merge_milestone_prs: runs-on: ubuntu-latest steps: - name: Setup temp directory @@ -47,7 +47,6 @@ jobs: - name: Setup Git User for OpenIM-Robot run: | - # Set up Git credentials for the bot git config --global user.email "OpenIM-Robot@users.noreply.github.com" git config --global user.name "OpenIM-Robot" @@ -83,136 +82,65 @@ jobs: if ! echo "$labels" | grep -q "${LABEL_NAME}"; then echo "PR #$pr_number does not have the 'cherry-picked' label. Adding to the list." echo "$pr_number" >> ${{ env.TEMP_DIR }}/pr_numbers.txt - else - echo "PR #$pr_number already has the 'cherry-picked' label. Skipping." fi done - # Sort the filtered PR numbers sort -n ${{ env.TEMP_DIR }}/pr_numbers.txt -o ${{ env.TEMP_DIR }}/pr_numbers.txt - echo "Filtered and sorted PR numbers:" - cat ${{ env.TEMP_DIR }}/pr_numbers.txt || echo "No closed PR numbers found for milestone." - - - name: Fetch Merge Commits for PRs and Generate Title and Body + - name: Create Individual PRs run: | - # Ensure the files are initialized - > ${{ env.TEMP_DIR }}/commit_hashes.txt - > ${{ env.TEMP_DIR }}/pr_title.txt - > ${{ env.TEMP_DIR }}/pr_body.txt - - # Write description to the PR body - echo "### Description:" >> ${{ env.TEMP_DIR }}/pr_body.txt - echo "Merging PRs from milestone \`$MILESTONE_NAME\` into target branch \`$TARGET_BRANCH\`." >> ${{ env.TEMP_DIR }}/pr_body.txt - echo "" >> ${{ env.TEMP_DIR }}/pr_body.txt - echo "### Need Merge PRs:" >> ${{ env.TEMP_DIR }}/pr_body.txt - - pr_numbers_in_title="" - - # Process sorted PR numbers and generate commit hashes for pr_number in $(cat ${{ env.TEMP_DIR }}/pr_numbers.txt); do - echo "Processing PR #$pr_number" pr_details=$(curl -s -H "Authorization: token $BOT_TOKEN" \ -H "Accept: application/vnd.github+json" \ "https://api.github.com/repos/${{ github.repository }}/pulls/$pr_number") pr_title=$(echo "$pr_details" | jq -r '.title') + pr_body=$(echo "$pr_details" | jq -r '.body') + pr_creator=$(echo "$pr_details" | jq -r '.user.login') merge_commit=$(echo "$pr_details" | jq -r '.merge_commit_sha') short_commit_hash=$(echo "$merge_commit" | cut -c 1-7) - # Append PR details to the body - echo "- $pr_title: (#$pr_number) ($short_commit_hash)" >> ${{ env.TEMP_DIR }}/pr_body.txt + if [ "$merge_commit" != "null" ]; then + git fetch origin + + echo "Checking out target branch: $TARGET_BRANCH" + git checkout $TARGET_BRANCH - if [ "$merge_commit" != "null" ];then - echo "$merge_commit" >> ${{ env.TEMP_DIR }}/commit_hashes.txt - echo "#$pr_number" >> ${{ env.TEMP_DIR }}/pr_title.txt - pr_numbers_in_title="$pr_numbers_in_title #$pr_number" + echo "Pulling latest changes from target branch: $TARGET_BRANCH" + git pull origin $TARGET_BRANCH + + cherry_pick_branch="cherry-pick-${short_commit_hash}" + git checkout -b $cherry_pick_branch + + echo "Cherry-picking commit: $merge_commit" + if ! git cherry-pick "$merge_commit" --strategy=recursive -X theirs; then + echo "Cherry-pick encountered conflicts, attempting to continue..." + git cherry-pick --continue || { echo "Cherry-pick failed"; exit 1; } + fi + + git remote set-url origin "https://${BOT_TOKEN}@github.com/${{ github.repository }}.git" + + echo "Pushing branch: $cherry_pick_branch" + git push origin $cherry_pick_branch --force || { echo "Push failed"; exit 1; } + + new_pr_title="$pr_title [Created by @$pr_creator from #$pr_number]" + new_pr_body="$pr_body + > This PR is created from original PR #$pr_number." + + response=$(curl -s -X POST -H "Authorization: token $BOT_TOKEN" \ + -H "Accept: application/vnd.github+json" \ + https://api.github.com/repos/${{ github.repository }}/pulls \ + -d "$(jq -n --arg title "$new_pr_title" \ + --arg head "$cherry_pick_branch" \ + --arg base "$TARGET_BRANCH" \ + --arg body "$new_pr_body" \ + '{title: $title, head: $head, base: $base, body: $body}')") + + new_pr_number=$(echo "$response" | jq -r '.number') + echo "Created PR #$new_pr_number" + + curl -s -X POST -H "Authorization: token $GITHUB_TOKEN" \ + -H "Accept: application/vnd.github+json" \ + -d '{"labels": ["milestone-merge"]}' \ + "https://api.github.com/repos/${{ github.repository }}/issues/$new_pr_number/labels" fi done - - commit_hashes=$(cat ${{ env.TEMP_DIR }}/commit_hashes.txt | tr '\n' ' ') - first_commit_hash=$(head -n 1 ${{ env.TEMP_DIR }}/commit_hashes.txt) - cherry_pick_branch="cherry-pick-${first_commit_hash:0:7}" - echo "COMMIT_HASHES=$commit_hashes" >> $GITHUB_ENV - echo "CHERRY_PICK_BRANCH=$cherry_pick_branch" >> $GITHUB_ENV - echo "pr_numbers_in_title=$pr_numbers_in_title" >> $GITHUB_ENV - - - name: Pull and Cherry-pick Commits, Then Push - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - BOT_TOKEN: ${{ secrets.BOT_TOKEN }} - run: | - # Fetch and pull the latest changes from the target branch - git fetch origin - git checkout $TARGET_BRANCH - git pull origin $TARGET_BRANCH - - # Create a new branch for cherry-picking - git checkout -b $CHERRY_PICK_BRANCH - - # Cherry-pick the commits and handle conflicts - for commit_hash in $COMMIT_HASHES; do - echo "Attempting to cherry-pick commit $commit_hash" - if ! git cherry-pick "$commit_hash" --strategy=recursive -X theirs; then - echo "Conflict detected for $commit_hash. Resolving with incoming changes." - conflict_files=$(git diff --name-only --diff-filter=U) - echo "Conflicting files:" - echo "$conflict_files" - - for file in $conflict_files; do - if [ -f "$file" ]; then - echo "Resolving conflict for $file" - git add "$file" - else - echo "File $file has been deleted. Skipping." - git rm "$file" - fi - done - - echo "Conflicts resolved. Continuing cherry-pick." - git cherry-pick --continue - else - echo "Cherry-pick successful for commit $commit_hash." - fi - done - - # Push the cherry-pick branch to the repository - git remote set-url origin "https://${BOT_TOKEN}@github.com/${{ github.repository }}.git" - git push origin $CHERRY_PICK_BRANCH --force - - - name: Create Pull Request - run: | - # Prepare and create the PR - pr_title="deps: Merge ${{ env.pr_numbers_in_title }} PRs into $TARGET_BRANCH" - pr_body=$(cat ${{ env.TEMP_DIR }}/pr_body.txt) - - echo "Prepared PR title:" - echo "$pr_title" - echo "Prepared PR body:" - echo "$pr_body" - - # Create the PR using the GitHub API - response=$(curl -s -X POST -H "Authorization: token $BOT_TOKEN" \ - -H "Accept: application/vnd.github+json" \ - https://api.github.com/repos/${{ github.repository }}/pulls \ - -d "$(jq -n --arg title "$pr_title" \ - --arg head "$CHERRY_PICK_BRANCH" \ - --arg base "$TARGET_BRANCH" \ - --arg body "$pr_body" \ - '{title: $title, head: $head, base: $base, body: $body}')") - - pr_number=$(echo "$response" | jq -r '.number') - echo "$pr_number" > ${{ env.TEMP_DIR }}/created_pr_number.txt - echo "Created PR #$pr_number" - - - name: Add Label to Created Pull Request - run: | - # Add 'milestone-merge' label to the created PR - pr_number=$(cat ${{ env.TEMP_DIR }}/created_pr_number.txt) - echo "Adding label to PR #$pr_number" - - curl -s -X POST -H "Authorization: token $GITHUB_TOKEN" \ - -H "Accept: application/vnd.github+json" \ - -d '{"labels": ["milestone-merge"]}' \ - "https://api.github.com/repos/${{ github.repository }}/issues/$pr_number/labels" - - echo "Added 'milestone-merge' label to PR #$pr_number." diff --git a/.github/workflows/publish-docker-image.yml b/.github/workflows/publish-docker-image.yml index 40b79e61a..998a11cf3 100644 --- a/.github/workflows/publish-docker-image.yml +++ b/.github/workflows/publish-docker-image.yml @@ -25,11 +25,11 @@ jobs: with: path: main-repo - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + # - name: Set up QEMU + # uses: docker/setup-qemu-action@v3.3.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v3.8.0 - name: Build Docker image id: build @@ -38,11 +38,8 @@ jobs: context: ./main-repo load: true tags: "openim/openim-server:local" - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Save Docker image to file - run: docker save -o image.tar openim/openim-server:local + cache-from: type=gha,scope=build + cache-to: type=gha,mode=max,scope=build - name: Checkout compose repository uses: actions/checkout@v4 @@ -66,43 +63,12 @@ jobs: run: | cd ${{ github.workspace }}/compose-repo docker compose up -d - sleep 60 - # - name: Check openim-server health - # run: | - # timeout=300 - # interval=30 - # elapsed=0 - # while [[ $elapsed -le $timeout ]]; do - # if ! docker exec openim-server mage check; then - # echo "openim-server is not ready, waiting..." - # sleep $interval - # elapsed=$(($elapsed + $interval)) - # else - # echo "Health check successful" - # exit 0 - # fi - # done - # echo "Health check failed after 5 minutes" - # exit 1 - - # - name: Check openim-chat health - # if: success() - # run: | - # if ! docker exec openim-chat mage check; then - # echo "openim-chat check failed" - # exit 1 - # else - # echo "Health check successful" - # exit 0 - # fi - - - name: Load Docker image from file - run: docker load -i image.tar + docker compose ps - name: Extract metadata for Docker (tags, labels) id: meta - uses: docker/metadata-action@v5.5.1 + uses: docker/metadata-action@v5.6.0 with: images: | openim/openim-server @@ -112,29 +78,27 @@ jobs: type=ref,event=tag type=schedule type=ref,event=branch - type=semver,pattern={{version}} + # type=semver,pattern={{version}} type=semver,pattern=v{{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} type=semver,pattern=release-{{raw}} type=sha type=raw,value=${{ github.event.inputs.tag }} - name: Log in to Docker Hub - uses: docker/login-action@v2 + uses: docker/login-action@v3.3.0 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Log in to GitHub Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3.3.0 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Log in to Aliyun Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3.3.0 with: registry: registry.cn-hangzhou.aliyuncs.com username: ${{ secrets.ALIREGISTRY_USERNAME }} @@ -148,3 +112,28 @@ jobs: platforms: linux/amd64,linux/arm64 tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha,scope=build + cache-to: type=gha,mode=max,scope=build + + - name: Verify multi-platform support + run: | + images=("openim/openim-server" "ghcr.io/openimsdk/openim-server" "registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-server") + for image in "${images[@]}"; do + for tag in $(echo "${{ steps.meta.outputs.tags }}" | tr ',' '\n'); do + manifest=$(docker manifest inspect "$image:$tag" || echo "error") + if [[ "$manifest" == "error" ]]; then + echo "Manifest not found for $image:$tag" + exit 1 + fi + amd64_found=$(echo "$manifest" | jq '.manifests[] | select(.platform.architecture == "amd64")') + arm64_found=$(echo "$manifest" | jq '.manifests[] | select(.platform.architecture == "arm64")') + if [[ -z "$amd64_found" ]]; then + echo "Multi-platform support check failed for $image:$tag - missing amd64" + exit 1 + fi + if [[ -z "$arm64_found" ]]; then + echo "Multi-platform support check failed for $image:$tag - missing arm64" + exit 1 + fi + done + done From 623335c1452f54b2d0af4fee5d5fce4dc5643449 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 24 Jan 2025 15:40:58 +0800 Subject: [PATCH 27/42] fix: fill send time --- pkg/common/storage/database/mgo/msg.go | 134 ++++++++++++++++++++++++- pkg/common/storage/model/msg.go | 11 +- 2 files changed, 141 insertions(+), 4 deletions(-) diff --git a/pkg/common/storage/database/mgo/msg.go b/pkg/common/storage/database/mgo/msg.go index c440d4442..9e220072e 100644 --- a/pkg/common/storage/database/mgo/msg.go +++ b/pkg/common/storage/database/mgo/msg.go @@ -1091,22 +1091,150 @@ func (m *MsgMgo) onlyFindDocIndex(ctx context.Context, docID string, indexes []i return msgDocModel[0].Msg, nil } +//func (m *MsgMgo) FindSeqs(ctx context.Context, conversationID string, seqs []int64) ([]*model.MsgInfoModel, error) { +// if len(seqs) == 0 { +// return nil, nil +// } +// result := make([]*model.MsgInfoModel, 0, len(seqs)) +// for docID, seqs := range m.model.GetDocIDSeqsMap(conversationID, seqs) { +// res, err := m.onlyFindDocIndex(ctx, docID, datautil.Slice(seqs, m.model.GetMsgIndex)) +// if err != nil { +// return nil, err +// } +// for i, re := range res { +// if re == nil || re.Msg == nil { +// continue +// } +// result = append(result, res[i]) +// } +// } +// return result, nil +//} + +func (m *MsgMgo) findBeforeDocSendTime(ctx context.Context, docID string, limit int64) (int64, int64, error) { + if limit == 0 { + return 0, 0, nil + } + pipeline := []bson.M{ + { + "$match": bson.M{ + "doc_id": docID, + }, + }, + { + "$project": bson.M{ + "_id": 0, + "doc_id": 0, + //"msgs.msg.send_time": 1, + //"msgs.msg.seq": 1, + }, + }, + { + "$unwind": "$msgs", + }, + { + "$project": bson.M{ + //"_id": 0, + //"doc_id": 0, + "msgs.msg.send_time": 1, + "msgs.msg.seq": 1, + }, + }, + } + if limit > 0 { + pipeline = append(pipeline, bson.M{"$limit": limit}) + } + type Result struct { + Msgs *model.MsgInfoModel `bson:"msgs"` + } + res, err := mongoutil.Aggregate[Result](ctx, m.coll, pipeline) + if err != nil { + return 0, 0, err + } + for i := len(res) - 1; i > 0; i-- { + v := res[i] + if v.Msgs != nil && v.Msgs.Msg != nil && v.Msgs.Msg.SendTime > 0 { + return v.Msgs.Msg.Seq, v.Msgs.Msg.SendTime, nil + } + } + return 0, 0, nil +} + +func (m *MsgMgo) findBeforeSendTime(ctx context.Context, conversationID string, seq int64) (int64, int64, error) { + first := true + for i := m.model.GetDocIndex(seq); i >= 0; i-- { + limit := int64(-1) + if first { + first = false + limit = m.model.GetMsgIndex(seq) + } + docID := m.model.BuildDocIDByIndex(conversationID, i) + msgSeq, msgSendTime, err := m.findBeforeDocSendTime(ctx, docID, limit) + if err != nil { + return 0, 0, err + } + if msgSendTime > 0 { + return msgSeq, msgSendTime, nil + } + } + return 0, 0, nil +} + func (m *MsgMgo) FindSeqs(ctx context.Context, conversationID string, seqs []int64) ([]*model.MsgInfoModel, error) { if len(seqs) == 0 { return nil, nil } + var abnormalSeq []int64 result := make([]*model.MsgInfoModel, 0, len(seqs)) - for docID, seqs := range m.model.GetDocIDSeqsMap(conversationID, seqs) { - res, err := m.onlyFindDocIndex(ctx, docID, datautil.Slice(seqs, m.model.GetMsgIndex)) + for docID, docSeqs := range m.model.GetDocIDSeqsMap(conversationID, seqs) { + res, err := m.onlyFindDocIndex(ctx, docID, datautil.Slice(docSeqs, m.model.GetMsgIndex)) if err != nil { return nil, err } + if len(res) == 0 { + abnormalSeq = append(abnormalSeq, docSeqs...) + continue + } for i, re := range res { - if re == nil || re.Msg == nil { + if re == nil || re.Msg == nil || re.Msg.SendTime == 0 { + abnormalSeq = append(abnormalSeq, docSeqs[i]) continue } result = append(result, res[i]) } } + if len(abnormalSeq) > 0 { + datautil.Sort(abnormalSeq, false) + sendTime := make(map[int64]int64) + var ( + lastSeq int64 + lastSendTime int64 + ) + for _, seq := range abnormalSeq { + if lastSendTime > 0 && lastSeq <= seq { + sendTime[seq] = lastSendTime + continue + } + msgSeq, msgSendTime, err := m.findBeforeSendTime(ctx, conversationID, seq) + if err != nil { + return nil, err + } + if msgSendTime <= 0 { + break + } + sendTime[seq] = msgSendTime + lastSeq = msgSeq + lastSendTime = msgSendTime + } + for _, seq := range abnormalSeq { + result = append(result, &model.MsgInfoModel{ + Msg: &model.MsgDataModel{ + Seq: seq, + Status: constant.MsgStatusHasDeleted, + SendTime: sendTime[seq], + }, + }) + } + } return result, nil } diff --git a/pkg/common/storage/model/msg.go b/pkg/common/storage/model/msg.go index 69113032d..6cf63bfcd 100644 --- a/pkg/common/storage/model/msg.go +++ b/pkg/common/storage/model/msg.go @@ -15,9 +15,10 @@ package model import ( + "strconv" + "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/errs" - "strconv" ) const ( @@ -108,6 +109,10 @@ func (m *MsgDocModel) IsFull() bool { return m.Msg[len(m.Msg)-1].Msg != nil } +func (m *MsgDocModel) GetDocIndex(seq int64) int64 { + return (seq - 1) / singleGocMsgNum +} + func (m *MsgDocModel) GetDocID(conversationID string, seq int64) string { seqSuffix := (seq - 1) / singleGocMsgNum return m.indexGen(conversationID, seqSuffix) @@ -135,6 +140,10 @@ func (*MsgDocModel) indexGen(conversationID string, seqSuffix int64) string { return conversationID + ":" + strconv.FormatInt(seqSuffix, 10) } +func (*MsgDocModel) BuildDocIDByIndex(conversationID string, index int64) string { + return conversationID + ":" + strconv.FormatInt(index, 10) +} + func (*MsgDocModel) GenExceptionMessageBySeqs(seqs []int64) (exceptionMsg []*sdkws.MsgData) { for _, v := range seqs { msgModel := new(sdkws.MsgData) From 29c7a409468d7b0185b418477aa1723c60b5a4a7 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 24 Jan 2025 15:50:32 +0800 Subject: [PATCH 28/42] fix: fill send time --- pkg/common/storage/database/mgo/msg.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/common/storage/database/mgo/msg.go b/pkg/common/storage/database/mgo/msg.go index 9e220072e..83fefbfe6 100644 --- a/pkg/common/storage/database/mgo/msg.go +++ b/pkg/common/storage/database/mgo/msg.go @@ -1125,8 +1125,6 @@ func (m *MsgMgo) findBeforeDocSendTime(ctx context.Context, docID string, limit "$project": bson.M{ "_id": 0, "doc_id": 0, - //"msgs.msg.send_time": 1, - //"msgs.msg.seq": 1, }, }, { From 274a9bee6577ff080ed803de035f6953069a7804 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Fri, 24 Jan 2025 16:13:12 +0800 Subject: [PATCH 29/42] fix: the abnormal message has no sending time, causing the SDK to be abnormal (#3087) * pb * fix: Modifying other fields while setting IsPrivateChat does not take effect * fix: quote message error revoke * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * upgrading pkg tools * fix * fix * optimize log output * feat: support GetLastMessage * feat: support GetLastMessage * feat: s3 switch * feat: s3 switch * fix: GetUsersOnline * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: seq conversion failed without exiting * fix: DeleteDoc crash * fix: fill send time * fix: fill send time --- pkg/common/storage/database/mgo/msg.go | 132 ++++++++++++++++++++++++- pkg/common/storage/model/msg.go | 11 ++- 2 files changed, 139 insertions(+), 4 deletions(-) diff --git a/pkg/common/storage/database/mgo/msg.go b/pkg/common/storage/database/mgo/msg.go index c440d4442..83fefbfe6 100644 --- a/pkg/common/storage/database/mgo/msg.go +++ b/pkg/common/storage/database/mgo/msg.go @@ -1091,22 +1091,148 @@ func (m *MsgMgo) onlyFindDocIndex(ctx context.Context, docID string, indexes []i return msgDocModel[0].Msg, nil } +//func (m *MsgMgo) FindSeqs(ctx context.Context, conversationID string, seqs []int64) ([]*model.MsgInfoModel, error) { +// if len(seqs) == 0 { +// return nil, nil +// } +// result := make([]*model.MsgInfoModel, 0, len(seqs)) +// for docID, seqs := range m.model.GetDocIDSeqsMap(conversationID, seqs) { +// res, err := m.onlyFindDocIndex(ctx, docID, datautil.Slice(seqs, m.model.GetMsgIndex)) +// if err != nil { +// return nil, err +// } +// for i, re := range res { +// if re == nil || re.Msg == nil { +// continue +// } +// result = append(result, res[i]) +// } +// } +// return result, nil +//} + +func (m *MsgMgo) findBeforeDocSendTime(ctx context.Context, docID string, limit int64) (int64, int64, error) { + if limit == 0 { + return 0, 0, nil + } + pipeline := []bson.M{ + { + "$match": bson.M{ + "doc_id": docID, + }, + }, + { + "$project": bson.M{ + "_id": 0, + "doc_id": 0, + }, + }, + { + "$unwind": "$msgs", + }, + { + "$project": bson.M{ + //"_id": 0, + //"doc_id": 0, + "msgs.msg.send_time": 1, + "msgs.msg.seq": 1, + }, + }, + } + if limit > 0 { + pipeline = append(pipeline, bson.M{"$limit": limit}) + } + type Result struct { + Msgs *model.MsgInfoModel `bson:"msgs"` + } + res, err := mongoutil.Aggregate[Result](ctx, m.coll, pipeline) + if err != nil { + return 0, 0, err + } + for i := len(res) - 1; i > 0; i-- { + v := res[i] + if v.Msgs != nil && v.Msgs.Msg != nil && v.Msgs.Msg.SendTime > 0 { + return v.Msgs.Msg.Seq, v.Msgs.Msg.SendTime, nil + } + } + return 0, 0, nil +} + +func (m *MsgMgo) findBeforeSendTime(ctx context.Context, conversationID string, seq int64) (int64, int64, error) { + first := true + for i := m.model.GetDocIndex(seq); i >= 0; i-- { + limit := int64(-1) + if first { + first = false + limit = m.model.GetMsgIndex(seq) + } + docID := m.model.BuildDocIDByIndex(conversationID, i) + msgSeq, msgSendTime, err := m.findBeforeDocSendTime(ctx, docID, limit) + if err != nil { + return 0, 0, err + } + if msgSendTime > 0 { + return msgSeq, msgSendTime, nil + } + } + return 0, 0, nil +} + func (m *MsgMgo) FindSeqs(ctx context.Context, conversationID string, seqs []int64) ([]*model.MsgInfoModel, error) { if len(seqs) == 0 { return nil, nil } + var abnormalSeq []int64 result := make([]*model.MsgInfoModel, 0, len(seqs)) - for docID, seqs := range m.model.GetDocIDSeqsMap(conversationID, seqs) { - res, err := m.onlyFindDocIndex(ctx, docID, datautil.Slice(seqs, m.model.GetMsgIndex)) + for docID, docSeqs := range m.model.GetDocIDSeqsMap(conversationID, seqs) { + res, err := m.onlyFindDocIndex(ctx, docID, datautil.Slice(docSeqs, m.model.GetMsgIndex)) if err != nil { return nil, err } + if len(res) == 0 { + abnormalSeq = append(abnormalSeq, docSeqs...) + continue + } for i, re := range res { - if re == nil || re.Msg == nil { + if re == nil || re.Msg == nil || re.Msg.SendTime == 0 { + abnormalSeq = append(abnormalSeq, docSeqs[i]) continue } result = append(result, res[i]) } } + if len(abnormalSeq) > 0 { + datautil.Sort(abnormalSeq, false) + sendTime := make(map[int64]int64) + var ( + lastSeq int64 + lastSendTime int64 + ) + for _, seq := range abnormalSeq { + if lastSendTime > 0 && lastSeq <= seq { + sendTime[seq] = lastSendTime + continue + } + msgSeq, msgSendTime, err := m.findBeforeSendTime(ctx, conversationID, seq) + if err != nil { + return nil, err + } + if msgSendTime <= 0 { + break + } + sendTime[seq] = msgSendTime + lastSeq = msgSeq + lastSendTime = msgSendTime + } + for _, seq := range abnormalSeq { + result = append(result, &model.MsgInfoModel{ + Msg: &model.MsgDataModel{ + Seq: seq, + Status: constant.MsgStatusHasDeleted, + SendTime: sendTime[seq], + }, + }) + } + } return result, nil } diff --git a/pkg/common/storage/model/msg.go b/pkg/common/storage/model/msg.go index 69113032d..6cf63bfcd 100644 --- a/pkg/common/storage/model/msg.go +++ b/pkg/common/storage/model/msg.go @@ -15,9 +15,10 @@ package model import ( + "strconv" + "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/errs" - "strconv" ) const ( @@ -108,6 +109,10 @@ func (m *MsgDocModel) IsFull() bool { return m.Msg[len(m.Msg)-1].Msg != nil } +func (m *MsgDocModel) GetDocIndex(seq int64) int64 { + return (seq - 1) / singleGocMsgNum +} + func (m *MsgDocModel) GetDocID(conversationID string, seq int64) string { seqSuffix := (seq - 1) / singleGocMsgNum return m.indexGen(conversationID, seqSuffix) @@ -135,6 +140,10 @@ func (*MsgDocModel) indexGen(conversationID string, seqSuffix int64) string { return conversationID + ":" + strconv.FormatInt(seqSuffix, 10) } +func (*MsgDocModel) BuildDocIDByIndex(conversationID string, index int64) string { + return conversationID + ":" + strconv.FormatInt(index, 10) +} + func (*MsgDocModel) GenExceptionMessageBySeqs(seqs []int64) (exceptionMsg []*sdkws.MsgData) { for _, v := range seqs { msgModel := new(sdkws.MsgData) From d1b4d84628513b800d76404d89c29fc8070117e9 Mon Sep 17 00:00:00 2001 From: skiffer-git <72860476+skiffer-git@users.noreply.github.com> Date: Tue, 4 Feb 2025 23:25:46 +0800 Subject: [PATCH 30/42] Update LICENSE --- LICENSE | 210 ++++++-------------------------------------------------- 1 file changed, 22 insertions(+), 188 deletions(-) diff --git a/LICENSE b/LICENSE index 261eeb9e9..4591ca426 100644 --- a/LICENSE +++ b/LICENSE @@ -1,201 +1,35 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ +# Open Source License - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION +OpenIM is licensed under the Apache License 2.0, with the following additional conditions: - 1. Definitions. +1. OpenIM may be utilized commercially, including as a backend service for other applications or as an application development platform for enterprises. +A commercial license must be obtained from the producer if: + - Under no circumstances may you operate a multi-tenant or multi-business environment using the OpenIM source code, whether or not you have modified the repository code. In other words, a single instance of OpenIM may not simultaneously serve multiple enterprises, nor may it serve multiple lines of business within the same enterprise. + - If you intend to operate in such a multi-tenant or multi-business manner, you must obtain a commercial license from the producer in advance. - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. +2. As a contributor, you should agree that: - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. + a. The producer can adjust the open-source agreement to be more strict or more relaxed as deemed necessary. + b. Your contributed code may be used for commercial purposes, including but not limited to its cloud business operations. - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. +Apart from the specific conditions mentioned above, all other rights and restrictions follow the Apache License 2.0. Detailed information about the Apache License 2.0 can be found at http://www.apache.org/licenses/LICENSE-2.0. - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. +For any licensing-related questions or to obtain a commercial license, please contact contact@openim.io. - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). +© 2024 OpenIMSDK - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. +---------- - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. + http://www.apache.org/licenses/LICENSE-2.0 - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. From 86aa26ef62957ab72053cc32eb2b43490e8dd5d2 Mon Sep 17 00:00:00 2001 From: skiffer-git <72860476+skiffer-git@users.noreply.github.com> Date: Wed, 5 Feb 2025 08:28:22 +0800 Subject: [PATCH 31/42] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6b1779b25..3b88935eb 100644 --- a/README.md +++ b/README.md @@ -132,7 +132,8 @@ Thank you for contributing to building a powerful instant messaging solution! ## :closed_book: License -OpenIMSDK is available under the Apache License 2.0. See the [LICENSE file](https://github.com/openimsdk/open-im-server/blob/main/LICENSE) for more information. +For more details, please refer to [here](./LICENSE). + From 8b752aff6f0eabe0b0335d49b8e0dcd96a5fb917 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 6 Feb 2025 10:02:48 +0800 Subject: [PATCH 32/42] fix: crash caused by withdrawing messages from users who have left the group --- internal/rpc/msg/revoke.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/internal/rpc/msg/revoke.go b/internal/rpc/msg/revoke.go index 97de0f48a..c2fb5833f 100644 --- a/internal/rpc/msg/revoke.go +++ b/internal/rpc/msg/revoke.go @@ -17,9 +17,10 @@ package msg import ( "context" "encoding/json" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" "github.com/openimsdk/protocol/constant" @@ -79,8 +80,10 @@ func (m *msgServer) RevokeMsg(ctx context.Context, req *msg.RevokeMsgReq) (*msg. switch members[req.UserID].RoleLevel { case constant.GroupOwner: case constant.GroupAdmin: - if members[msgs[0].SendID].RoleLevel != constant.GroupOrdinaryUsers { - return nil, errs.ErrNoPermission.WrapMsg("no permission") + if sendMember, ok := members[msgs[0].SendID]; ok { + if sendMember.RoleLevel != constant.GroupOrdinaryUsers { + return nil, errs.ErrNoPermission.WrapMsg("no permission") + } } default: return nil, errs.ErrNoPermission.WrapMsg("no permission") From ed416f83768d76ab4acdd23bae926c0536bb55de Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Thu, 6 Feb 2025 10:38:46 +0800 Subject: [PATCH 33/42] fix: crash caused by withdrawing messages from users who have left the group (#3100) * pb * fix: Modifying other fields while setting IsPrivateChat does not take effect * fix: quote message error revoke * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * upgrading pkg tools * fix * fix * optimize log output * feat: support GetLastMessage * feat: support GetLastMessage * feat: s3 switch * feat: s3 switch * fix: GetUsersOnline * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: seq conversion failed without exiting * fix: DeleteDoc crash * fix: fill send time * fix: fill send time * fix: crash caused by withdrawing messages from users who have left the group --- internal/rpc/msg/revoke.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/internal/rpc/msg/revoke.go b/internal/rpc/msg/revoke.go index 97de0f48a..c2fb5833f 100644 --- a/internal/rpc/msg/revoke.go +++ b/internal/rpc/msg/revoke.go @@ -17,9 +17,10 @@ package msg import ( "context" "encoding/json" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" "github.com/openimsdk/protocol/constant" @@ -79,8 +80,10 @@ func (m *msgServer) RevokeMsg(ctx context.Context, req *msg.RevokeMsgReq) (*msg. switch members[req.UserID].RoleLevel { case constant.GroupOwner: case constant.GroupAdmin: - if members[msgs[0].SendID].RoleLevel != constant.GroupOrdinaryUsers { - return nil, errs.ErrNoPermission.WrapMsg("no permission") + if sendMember, ok := members[msgs[0].SendID]; ok { + if sendMember.RoleLevel != constant.GroupOrdinaryUsers { + return nil, errs.ErrNoPermission.WrapMsg("no permission") + } } default: return nil, errs.ErrNoPermission.WrapMsg("no permission") From 23976e4c74d3e6f08532f77296b10418498396bb Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 7 Feb 2025 14:33:17 +0800 Subject: [PATCH 34/42] fix: user msg timestamp --- internal/rpc/conversation/conversation.go | 5 +++-- pkg/common/storage/database/mgo/conversation.go | 9 ++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/internal/rpc/conversation/conversation.go b/internal/rpc/conversation/conversation.go index 53364ff86..76dd606aa 100644 --- a/internal/rpc/conversation/conversation.go +++ b/internal/rpc/conversation/conversation.go @@ -16,10 +16,11 @@ package conversation import ( "context" - "github.com/openimsdk/open-im-server/v3/pkg/rpcli" "sort" "time" + "github.com/openimsdk/open-im-server/v3/pkg/rpcli" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" @@ -773,7 +774,7 @@ func (c *conversationServer) ClearUserConversationMsg(ctx context.Context, req * if conversation.IsMsgDestruct == false || conversation.MsgDestructTime == 0 { continue } - seq, err := c.msgClient.GetLastMessageSeqByTime(ctx, conversation.ConversationID, req.Timestamp-conversation.MsgDestructTime) + seq, err := c.msgClient.GetLastMessageSeqByTime(ctx, conversation.ConversationID, req.Timestamp-(conversation.MsgDestructTime*1000)) if err != nil { return nil, err } diff --git a/pkg/common/storage/database/mgo/conversation.go b/pkg/common/storage/database/mgo/conversation.go index 851ec99c4..536827450 100644 --- a/pkg/common/storage/database/mgo/conversation.go +++ b/pkg/common/storage/database/mgo/conversation.go @@ -243,7 +243,14 @@ func (c *ConversationMgo) FindRandConversation(ctx context.Context, ts int64, li "$add": []any{ bson.M{ "$toLong": "$latest_msg_destruct_time", - }, "$msg_destruct_time"}, + }, + bson.M{ + "$multiply": []any{ + "$msg_destruct_time", + 1000, // convert to milliseconds + }, + }, + }, }, }, }, From 0bf076bb05bfd2bfcf5ee5e2c95a3fce6a62e3f1 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Sat, 8 Feb 2025 10:36:00 +0800 Subject: [PATCH 35/42] fix: the user sets the conversation timer cleanup timestamp unit incorrectly (#3102) * pb * fix: Modifying other fields while setting IsPrivateChat does not take effect * fix: quote message error revoke * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * upgrading pkg tools * fix * fix * optimize log output * feat: support GetLastMessage * feat: support GetLastMessage * feat: s3 switch * feat: s3 switch * fix: GetUsersOnline * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: seq conversion failed without exiting * fix: DeleteDoc crash * fix: fill send time * fix: fill send time * fix: crash caused by withdrawing messages from users who have left the group * fix: user msg timestamp --- internal/rpc/conversation/conversation.go | 5 +++-- pkg/common/storage/database/mgo/conversation.go | 9 ++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/internal/rpc/conversation/conversation.go b/internal/rpc/conversation/conversation.go index 53364ff86..76dd606aa 100644 --- a/internal/rpc/conversation/conversation.go +++ b/internal/rpc/conversation/conversation.go @@ -16,10 +16,11 @@ package conversation import ( "context" - "github.com/openimsdk/open-im-server/v3/pkg/rpcli" "sort" "time" + "github.com/openimsdk/open-im-server/v3/pkg/rpcli" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" @@ -773,7 +774,7 @@ func (c *conversationServer) ClearUserConversationMsg(ctx context.Context, req * if conversation.IsMsgDestruct == false || conversation.MsgDestructTime == 0 { continue } - seq, err := c.msgClient.GetLastMessageSeqByTime(ctx, conversation.ConversationID, req.Timestamp-conversation.MsgDestructTime) + seq, err := c.msgClient.GetLastMessageSeqByTime(ctx, conversation.ConversationID, req.Timestamp-(conversation.MsgDestructTime*1000)) if err != nil { return nil, err } diff --git a/pkg/common/storage/database/mgo/conversation.go b/pkg/common/storage/database/mgo/conversation.go index 851ec99c4..536827450 100644 --- a/pkg/common/storage/database/mgo/conversation.go +++ b/pkg/common/storage/database/mgo/conversation.go @@ -243,7 +243,14 @@ func (c *ConversationMgo) FindRandConversation(ctx context.Context, ts int64, li "$add": []any{ bson.M{ "$toLong": "$latest_msg_destruct_time", - }, "$msg_destruct_time"}, + }, + bson.M{ + "$multiply": []any{ + "$msg_destruct_time", + 1000, // convert to milliseconds + }, + }, + }, }, }, }, From ad8829c5a6d267f29fe0e506b962d86ad8ab1cdc Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Sat, 8 Feb 2025 12:03:29 +0800 Subject: [PATCH 36/42] build: keep conflict is true. (#3103) --- .github/workflows/merge-from-milestone.yml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/merge-from-milestone.yml b/.github/workflows/merge-from-milestone.yml index 8e09c9ab7..67aafcf0d 100644 --- a/.github/workflows/merge-from-milestone.yml +++ b/.github/workflows/merge-from-milestone.yml @@ -113,14 +113,26 @@ jobs: echo "Cherry-picking commit: $merge_commit" if ! git cherry-pick "$merge_commit" --strategy=recursive -X theirs; then - echo "Cherry-pick encountered conflicts, attempting to continue..." - git cherry-pick --continue || { echo "Cherry-pick failed"; exit 1; } + echo "Cherry-pick encountered conflicts, attempting to resolve..." + git status --porcelain | grep '^UU ' | cut -c 4- | while read -r file; do + echo "Resolving conflict in $file" + git add "$file" + done + git status --porcelain | grep '^AA ' | cut -c 4- | while read -r file; do + echo "Adding new file $file" + git add "$file" + done + git status --porcelain | grep '^DD ' | cut -c 4- | while read -r file; do + echo "Removing deleted file $file" + git rm "$file" + done + git cherry-pick --continue || { echo "Cherry-pick failed"; continue; } fi git remote set-url origin "https://${BOT_TOKEN}@github.com/${{ github.repository }}.git" echo "Pushing branch: $cherry_pick_branch" - git push origin $cherry_pick_branch --force || { echo "Push failed"; exit 1; } + git push origin $cherry_pick_branch --force || { echo "Push failed"; continue; } new_pr_title="$pr_title [Created by @$pr_creator from #$pr_number]" new_pr_body="$pr_body From 489571f4b6cdb8c8906a15aab4ad3fefa38ffbda Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Sat, 8 Feb 2025 15:26:35 +0800 Subject: [PATCH 37/42] fix: solve stop when merge failed (#3106) --- .github/workflows/merge-from-milestone.yml | 35 +++++++++++++--------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/.github/workflows/merge-from-milestone.yml b/.github/workflows/merge-from-milestone.yml index 67aafcf0d..1f5762ccb 100644 --- a/.github/workflows/merge-from-milestone.yml +++ b/.github/workflows/merge-from-milestone.yml @@ -113,26 +113,33 @@ jobs: echo "Cherry-picking commit: $merge_commit" if ! git cherry-pick "$merge_commit" --strategy=recursive -X theirs; then - echo "Cherry-pick encountered conflicts, attempting to resolve..." - git status --porcelain | grep '^UU ' | cut -c 4- | while read -r file; do - echo "Resolving conflict in $file" - git add "$file" + echo "Conflict detected for $merge_commit. Resolving with incoming changes." + conflict_files=$(git diff --name-only --diff-filter=U) + echo "Conflicting files:" + echo "$conflict_files" + + for file in $conflict_files; do + if [ -f "$file" ]; then + echo "Resolving conflict for $file" + git add "$file" + else + echo "File $file has been deleted. Skipping." + git rm "$file" + fi done - git status --porcelain | grep '^AA ' | cut -c 4- | while read -r file; do - echo "Adding new file $file" - git add "$file" - done - git status --porcelain | grep '^DD ' | cut -c 4- | while read -r file; do - echo "Removing deleted file $file" - git rm "$file" - done - git cherry-pick --continue || { echo "Cherry-pick failed"; continue; } + + echo "Conflicts resolved. Continuing cherry-pick." + git cherry-pick --continue || { echo "Cherry-pick failed, but continuing to create PR."; } + else + echo "Cherry-pick successful for commit $merge_commit." fi git remote set-url origin "https://${BOT_TOKEN}@github.com/${{ github.repository }}.git" echo "Pushing branch: $cherry_pick_branch" - git push origin $cherry_pick_branch --force || { echo "Push failed"; continue; } + if ! git push origin $cherry_pick_branch --force; then + echo "Push failed, but continuing to create PR..." + fi new_pr_title="$pr_title [Created by @$pr_creator from #$pr_number]" new_pr_body="$pr_body From 8e6448d7a7a75a34876ce8921bd7bcdd162ef40b Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Mon, 10 Feb 2025 10:57:03 +0800 Subject: [PATCH 38/42] seq read config --- tools/seq/internal/seq.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tools/seq/internal/seq.go b/tools/seq/internal/seq.go index c931cda5d..3ec67e2db 100644 --- a/tools/seq/internal/seq.go +++ b/tools/seq/internal/seq.go @@ -15,16 +15,17 @@ import ( "syscall" "time" + "github.com/mitchellh/mapstructure" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/db/redisutil" "github.com/openimsdk/tools/utils/runtimeenv" "github.com/redis/go-redis/v9" + "github.com/spf13/viper" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" - "gopkg.in/yaml.v3" ) const ( @@ -45,13 +46,16 @@ func readConfig[T any](dir string, name string) (*T, error) { if runtimeenv.PrintRuntimeEnvironment() == config.KUBERNETES { dir = os.Getenv(config.MountConfigFilePath) } - - data, err := os.ReadFile(filepath.Join(dir, name)) - if err != nil { + v := viper.New() + v.SetConfigFile(filepath.Join(dir, name)) + if err := v.ReadInConfig(); err != nil { return nil, err } + fn := func(config *mapstructure.DecoderConfig) { + config.TagName = "mapstructure" + } var conf T - if err := yaml.Unmarshal(data, &conf); err != nil { + if err := v.Unmarshal(&conf, fn); err != nil { return nil, err } return &conf, nil From aebd6d6dd69f03317a96de72603869b44557a021 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Mon, 10 Feb 2025 11:28:09 +0800 Subject: [PATCH 39/42] seq read config --- tools/seq/internal/seq.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/seq/internal/seq.go b/tools/seq/internal/seq.go index 3ec67e2db..574e7cef9 100644 --- a/tools/seq/internal/seq.go +++ b/tools/seq/internal/seq.go @@ -47,6 +47,9 @@ func readConfig[T any](dir string, name string) (*T, error) { dir = os.Getenv(config.MountConfigFilePath) } v := viper.New() + v.SetEnvPrefix(config.EnvPrefixMap[name]) + v.AutomaticEnv() + v.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) v.SetConfigFile(filepath.Join(dir, name)) if err := v.ReadInConfig(); err != nil { return nil, err From a3c43a49d829be586454c3793c440e39fb4ae019 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Mon, 10 Feb 2025 11:44:57 +0800 Subject: [PATCH 40/42] fix: seq conversion not reading env in docker environment (#3130) * pb * fix: Modifying other fields while setting IsPrivateChat does not take effect * fix: quote message error revoke * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * upgrading pkg tools * fix * fix * optimize log output * feat: support GetLastMessage * feat: support GetLastMessage * feat: s3 switch * feat: s3 switch * fix: GetUsersOnline * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: seq conversion failed without exiting * fix: DeleteDoc crash * fix: fill send time * fix: fill send time * fix: crash caused by withdrawing messages from users who have left the group * fix: user msg timestamp * seq read config * seq read config --- tools/seq/internal/seq.go | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/tools/seq/internal/seq.go b/tools/seq/internal/seq.go index c931cda5d..574e7cef9 100644 --- a/tools/seq/internal/seq.go +++ b/tools/seq/internal/seq.go @@ -15,16 +15,17 @@ import ( "syscall" "time" + "github.com/mitchellh/mapstructure" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/db/redisutil" "github.com/openimsdk/tools/utils/runtimeenv" "github.com/redis/go-redis/v9" + "github.com/spf13/viper" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" - "gopkg.in/yaml.v3" ) const ( @@ -45,13 +46,19 @@ func readConfig[T any](dir string, name string) (*T, error) { if runtimeenv.PrintRuntimeEnvironment() == config.KUBERNETES { dir = os.Getenv(config.MountConfigFilePath) } - - data, err := os.ReadFile(filepath.Join(dir, name)) - if err != nil { + v := viper.New() + v.SetEnvPrefix(config.EnvPrefixMap[name]) + v.AutomaticEnv() + v.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) + v.SetConfigFile(filepath.Join(dir, name)) + if err := v.ReadInConfig(); err != nil { return nil, err } + fn := func(config *mapstructure.DecoderConfig) { + config.TagName = "mapstructure" + } var conf T - if err := yaml.Unmarshal(data, &conf); err != nil { + if err := v.Unmarshal(&conf, fn); err != nil { return nil, err } return &conf, nil From 4569781fb70f8bd9d5b470e0e60d9ef46e4b168c Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 12 Feb 2025 18:32:06 +0800 Subject: [PATCH 41/42] fix: the source message of the reference is withdrawn, and the referenced message is deleted --- internal/msgtransfer/online_msg_to_mongo_handler.go | 12 ++++++++---- pkg/common/storage/controller/msg.go | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/internal/msgtransfer/online_msg_to_mongo_handler.go b/internal/msgtransfer/online_msg_to_mongo_handler.go index d8836d54e..8405be7fe 100644 --- a/internal/msgtransfer/online_msg_to_mongo_handler.go +++ b/internal/msgtransfer/online_msg_to_mongo_handler.go @@ -73,10 +73,14 @@ func (mc *OnlineHistoryMongoConsumerHandler) handleChatWs2Mongo(ctx context.Cont } else { prommetrics.MsgInsertMongoSuccessCounter.Inc() } - var seqs []int64 - for _, msg := range msgFromMQ.MsgData { - seqs = append(seqs, msg.Seq) - } + //var seqs []int64 + //for _, msg := range msgFromMQ.MsgData { + // seqs = append(seqs, msg.Seq) + //} + //if err := mc.msgTransferDatabase.DeleteMessagesFromCache(ctx, msgFromMQ.ConversationID, seqs); err != nil { + // log.ZError(ctx, "remove cache msg from redis err", err, "msg", + // msgFromMQ.MsgData, "conversationID", msgFromMQ.ConversationID) + //} } func (*OnlineHistoryMongoConsumerHandler) Setup(_ sarama.ConsumerGroupSession) error { return nil } diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index b92f9b510..0069dc7cc 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -310,7 +310,7 @@ func (db *commonMsgDatabase) handlerDBMsg(ctx context.Context, cache map[int64][ log.ZError(ctx, "json.Unmarshal", err) return } - if quoteMsg.QuoteMessage == nil || quoteMsg.QuoteMessage.Content == "" { + if quoteMsg.QuoteMessage == nil { return } if quoteMsg.QuoteMessage.Content == "e30=" { From e37ea50b9493e45bc1581bbf74e126050a846303 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Wed, 12 Feb 2025 18:46:38 +0800 Subject: [PATCH 42/42] fix: the source message of the reference is withdrawn, and the referenced message is deleted (#3137) * pb * fix: Modifying other fields while setting IsPrivateChat does not take effect * fix: quote message error revoke * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * upgrading pkg tools * fix * fix * optimize log output * feat: support GetLastMessage * feat: support GetLastMessage * feat: s3 switch * feat: s3 switch * fix: GetUsersOnline * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: seq conversion failed without exiting * fix: DeleteDoc crash * fix: fill send time * fix: fill send time * fix: crash caused by withdrawing messages from users who have left the group * fix: user msg timestamp * seq read config * seq read config * fix: the source message of the reference is withdrawn, and the referenced message is deleted --- internal/msgtransfer/online_msg_to_mongo_handler.go | 12 ++++++++---- pkg/common/storage/controller/msg.go | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/internal/msgtransfer/online_msg_to_mongo_handler.go b/internal/msgtransfer/online_msg_to_mongo_handler.go index d8836d54e..8405be7fe 100644 --- a/internal/msgtransfer/online_msg_to_mongo_handler.go +++ b/internal/msgtransfer/online_msg_to_mongo_handler.go @@ -73,10 +73,14 @@ func (mc *OnlineHistoryMongoConsumerHandler) handleChatWs2Mongo(ctx context.Cont } else { prommetrics.MsgInsertMongoSuccessCounter.Inc() } - var seqs []int64 - for _, msg := range msgFromMQ.MsgData { - seqs = append(seqs, msg.Seq) - } + //var seqs []int64 + //for _, msg := range msgFromMQ.MsgData { + // seqs = append(seqs, msg.Seq) + //} + //if err := mc.msgTransferDatabase.DeleteMessagesFromCache(ctx, msgFromMQ.ConversationID, seqs); err != nil { + // log.ZError(ctx, "remove cache msg from redis err", err, "msg", + // msgFromMQ.MsgData, "conversationID", msgFromMQ.ConversationID) + //} } func (*OnlineHistoryMongoConsumerHandler) Setup(_ sarama.ConsumerGroupSession) error { return nil } diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index b92f9b510..0069dc7cc 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -310,7 +310,7 @@ func (db *commonMsgDatabase) handlerDBMsg(ctx context.Context, cache map[int64][ log.ZError(ctx, "json.Unmarshal", err) return } - if quoteMsg.QuoteMessage == nil || quoteMsg.QuoteMessage.Content == "" { + if quoteMsg.QuoteMessage == nil { return } if quoteMsg.QuoteMessage.Content == "e30=" {