From c1632f8e65634bc2a8a8d859a317dcac88b6fd8d Mon Sep 17 00:00:00 2001 From: Gordon <1432970085@qq.com> Date: Wed, 18 May 2022 18:37:42 +0800 Subject: [PATCH 01/76] tag message sender not notification --- internal/rpc/msg/tag_send_msg.go | 1 + pkg/common/constant/constant.go | 1 + 2 files changed, 2 insertions(+) diff --git a/internal/rpc/msg/tag_send_msg.go b/internal/rpc/msg/tag_send_msg.go index 195640498..a79a90ec6 100644 --- a/internal/rpc/msg/tag_send_msg.go +++ b/internal/rpc/msg/tag_send_msg.go @@ -27,6 +27,7 @@ func TagSendMessage(operationID string, user *db.User, recvID, content string, s msgData.SenderNickname = user.Nickname msgData.Options = map[string]bool{} msgData.Options[constant.IsSenderConversationUpdate] = false + msgData.Options[constant.IsSenderNotificationPush] = false msgData.CreateTime = utils.GetCurrentTimestampByMill() msgData.ClientMsgID = utils.GetMsgID(user.UserID) msgData.SenderPlatformID = senderPlatformID diff --git a/pkg/common/constant/constant.go b/pkg/common/constant/constant.go index 33e350003..621bf9eb1 100644 --- a/pkg/common/constant/constant.go +++ b/pkg/common/constant/constant.go @@ -144,6 +144,7 @@ const ( IsSenderSync = "senderSync" IsNotPrivate = "notPrivate" IsSenderConversationUpdate = "senderConversationUpdate" + IsSenderNotificationPush = "senderNotificationPush" //GroupStatus GroupOk = 0 From d92d6e84d0a79c46aeca6d2c50ddd56c7ff1c2be Mon Sep 17 00:00:00 2001 From: Gordon <1432970085@qq.com> Date: Wed, 18 May 2022 18:58:51 +0800 Subject: [PATCH 02/76] push message --- internal/push/logic/push_to_client.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/push/logic/push_to_client.go b/internal/push/logic/push_to_client.go index b8125be5a..4ee83aa11 100644 --- a/internal/push/logic/push_to_client.go +++ b/internal/push/logic/push_to_client.go @@ -57,6 +57,9 @@ func MsgToUser(pushMsg *pbPush.PushMsgReq) { if isOfflinePush && pushMsg.PushToUserID != pushMsg.MsgData.SendID { for _, v := range wsResult { if v.ResultCode == 0 { + if utils.IsContainInt32(v.RecvPlatFormID, pushTerminal) { + break + } continue } if utils.IsContainInt32(v.RecvPlatFormID, pushTerminal) { From 967a4c774842162048b4b7d71ba072ad39034f3e Mon Sep 17 00:00:00 2001 From: Gordon <1432970085@qq.com> Date: Wed, 18 May 2022 21:09:49 +0800 Subject: [PATCH 03/76] statistic update --- pkg/statistics/statistics.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/statistics/statistics.go b/pkg/statistics/statistics.go index 800f5864b..50d6a2814 100644 --- a/pkg/statistics/statistics.go +++ b/pkg/statistics/statistics.go @@ -17,6 +17,7 @@ func (s *Statistics) output() { t := time.NewTicker(time.Duration(s.SleepTime) * time.Second) defer t.Stop() var sum uint64 + var timeIntervalNum uint64 for { sum = *s.AllCount select { @@ -27,7 +28,8 @@ func (s *Statistics) output() { } else { intervalCount = *s.AllCount - sum } - log.NewWarn("", " system stat ", s.ModuleName, s.PrintArgs, intervalCount, "total:", *s.AllCount) + timeIntervalNum++ + log.NewWarn("", " system stat ", s.ModuleName, s.PrintArgs, intervalCount, "total:", *s.AllCount, "intervalNum", timeIntervalNum) } } From 4f165f3b2718450d050c726c81c535406705982a Mon Sep 17 00:00:00 2001 From: Gordon <1432970085@qq.com> Date: Wed, 18 May 2022 21:58:32 +0800 Subject: [PATCH 04/76] statistic update --- pkg/statistics/statistics.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/statistics/statistics.go b/pkg/statistics/statistics.go index 50d6a2814..63653c42f 100644 --- a/pkg/statistics/statistics.go +++ b/pkg/statistics/statistics.go @@ -9,7 +9,7 @@ type Statistics struct { AllCount *uint64 ModuleName string PrintArgs string - SleepTime int + SleepTime uint64 } func (s *Statistics) output() { @@ -29,12 +29,12 @@ func (s *Statistics) output() { intervalCount = *s.AllCount - sum } timeIntervalNum++ - log.NewWarn("", " system stat ", s.ModuleName, s.PrintArgs, intervalCount, "total:", *s.AllCount, "intervalNum", timeIntervalNum) + log.NewWarn("", " system stat ", s.ModuleName, s.PrintArgs, intervalCount, "total:", *s.AllCount, "intervalNum", timeIntervalNum, "avg", (*s.AllCount)/(timeIntervalNum)/s.SleepTime) } } func NewStatistics(allCount *uint64, moduleName, printArgs string, sleepTime int) *Statistics { - p := &Statistics{AllCount: allCount, ModuleName: moduleName, SleepTime: sleepTime, PrintArgs: printArgs} + p := &Statistics{AllCount: allCount, ModuleName: moduleName, SleepTime: uint64(sleepTime), PrintArgs: printArgs} go p.output() return p } From ee0a71de1b306abe3b3e2490af63a4b450903b85 Mon Sep 17 00:00:00 2001 From: Gordon <1432970085@qq.com> Date: Thu, 19 May 2022 12:25:46 +0800 Subject: [PATCH 05/76] concurrent consumption of messages --- config/config.yaml | 4 +- internal/msg_transfer/logic/db.go | 20 ++- internal/msg_transfer/logic/init.go | 1 + .../logic/online_history_msg_handler.go | 141 +++++++++++++++++- pkg/common/config/config.go | 3 +- 5 files changed, 151 insertions(+), 18 deletions(-) diff --git a/config/config.yaml b/config/config.yaml index 736b15bce..0f1c5c4ec 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -222,7 +222,9 @@ secret: tuoyun multiloginpolicy: 1 #chat log insert to db -chatPersistenceMysql: true +chatpersistencemysql: true +#可靠性存储 +reliablestorage: true #token config tokenpolicy: diff --git a/internal/msg_transfer/logic/db.go b/internal/msg_transfer/logic/db.go index 806b782c0..98a684cc7 100644 --- a/internal/msg_transfer/logic/db.go +++ b/internal/msg_transfer/logic/db.go @@ -2,22 +2,20 @@ package logic import ( "Open_IM/pkg/common/db" - "Open_IM/pkg/common/log" pbMsg "Open_IM/pkg/proto/chat" - "Open_IM/pkg/utils" ) func saveUserChat(uid string, msg *pbMsg.MsgDataToMQ) error { - time := utils.GetCurrentTimestampByMill() - seq, err := db.DB.IncrUserSeq(uid) - if err != nil { - log.NewError(msg.OperationID, "data insert to redis err", err.Error(), msg.String()) - return err - } - msg.MsgData.Seq = uint32(seq) + //time := utils.GetCurrentTimestampByMill() + //seq, err := db.DB.IncrUserSeq(uid) + //if err != nil { + // log.NewError(msg.OperationID, "data insert to redis err", err.Error(), msg.String()) + // return err + //} + //msg.MsgData.Seq = uint32(seq) pbSaveData := pbMsg.MsgDataToDB{} pbSaveData.MsgData = msg.MsgData - log.NewInfo(msg.OperationID, "IncrUserSeq cost time", utils.GetCurrentTimestampByMill()-time) + //log.NewInfo(msg.OperationID, "IncrUserSeq cost time", utils.GetCurrentTimestampByMill()-time) return db.DB.SaveUserChatMongo2(uid, pbSaveData.MsgData.SendTime, &pbSaveData) -// return db.DB.SaveUserChatMongo2(uid, pbSaveData.MsgData.SendTime, &pbSaveData) + // return db.DB.SaveUserChatMongo2(uid, pbSaveData.MsgData.SendTime, &pbSaveData) } diff --git a/internal/msg_transfer/logic/init.go b/internal/msg_transfer/logic/init.go index 8f5cc0695..bef20e8c7 100644 --- a/internal/msg_transfer/logic/init.go +++ b/internal/msg_transfer/logic/init.go @@ -12,6 +12,7 @@ import ( const OnlineTopicBusy = 1 const OnlineTopicVacancy = 0 +const Msg = 2 var ( persistentCH PersistentConsumerHandler diff --git a/internal/msg_transfer/logic/online_history_msg_handler.go b/internal/msg_transfer/logic/online_history_msg_handler.go index dd1f22634..fb94ffe5f 100644 --- a/internal/msg_transfer/logic/online_history_msg_handler.go +++ b/internal/msg_transfer/logic/online_history_msg_handler.go @@ -3,6 +3,7 @@ package logic import ( "Open_IM/pkg/common/config" "Open_IM/pkg/common/constant" + "Open_IM/pkg/common/db" kfk "Open_IM/pkg/common/kafka" "Open_IM/pkg/common/log" "Open_IM/pkg/grpc-etcdv3/getcdv3" @@ -17,7 +18,11 @@ import ( "time" ) -type fcb func(msg []byte, msgKey string) +type MsgChannelValue struct { + userID string + msg *pbMsg.MsgDataToMQ +} +type fcb func(cMsg *sarama.ConsumerMessage, msgKey string, sess sarama.ConsumerGroupSession) type Cmd2Value struct { Cmd int Value interface{} @@ -26,12 +31,22 @@ type OnlineHistoryConsumerHandler struct { msgHandle map[string]fcb historyConsumerGroup *kfk.MConsumerGroup cmdCh chan Cmd2Value + msgCh chan Cmd2Value } func (och *OnlineHistoryConsumerHandler) Init(cmdCh chan Cmd2Value) { och.msgHandle = make(map[string]fcb) och.cmdCh = cmdCh - och.msgHandle[config.Config.Kafka.Ws2mschat.Topic] = och.handleChatWs2Mongo + och.msgCh = make(chan Cmd2Value, 1000) + if config.Config.ReliableStorage { + och.msgHandle[config.Config.Kafka.Ws2mschat.Topic] = och.handleChatWs2Mongo + } else { + och.msgHandle[config.Config.Kafka.Ws2mschat.Topic] = och.handleChatWs2MongoLowReliability + for i := 0; i < 10; i++ { + go och.Run() + + } + } och.historyConsumerGroup = kfk.NewMConsumerGroup(&kfk.MConsumerGroupConfig{KafkaVersion: sarama.V0_10_2_0, OffsetsInitial: sarama.OffsetNewest, IsReturnErr: false}, []string{config.Config.Kafka.Ws2mschat.Topic}, config.Config.Kafka.Ws2mschat.Addr, config.Config.Kafka.ConsumerGroupID.MsgToMongo) @@ -61,7 +76,28 @@ func sendCmd(ch chan Cmd2Value, value Cmd2Value, timeout int64) error { return errors.New("send cmd timeout") } } -func (och *OnlineHistoryConsumerHandler) handleChatWs2Mongo(msg []byte, msgKey string) { +func (och *OnlineHistoryConsumerHandler) Run() { + for { + select { + case cmd := <-och.msgCh: + switch cmd.Cmd { + case Msg: + msgChannelValue := cmd.Value.(MsgChannelValue) + err := saveUserChat(msgChannelValue.userID, msgChannelValue.msg) + if err != nil { + singleMsgFailedCount++ + log.NewError(msgChannelValue.msg.OperationID, "single data insert to mongo err", err.Error(), msgChannelValue.msg.String()) + } else { + singleMsgSuccessCountMutex.Lock() + singleMsgSuccessCount++ + singleMsgSuccessCountMutex.Unlock() + } + } + } + } +} +func (och *OnlineHistoryConsumerHandler) handleChatWs2Mongo(cMsg *sarama.ConsumerMessage, msgKey string, sess sarama.ConsumerGroupSession) { + msg := cMsg.Value now := time.Now() msgFromMQ := pbMsg.MsgDataToMQ{} err := proto.Unmarshal(msg, &msgFromMQ) @@ -126,6 +162,102 @@ func (och *OnlineHistoryConsumerHandler) handleChatWs2Mongo(msg []byte, msgKey s log.NewError(msgFromMQ.OperationID, "SessionType error", msgFromMQ.String()) return } + sess.MarkMessage(cMsg, "") + log.NewDebug(msgFromMQ.OperationID, "msg_transfer handle topic data to database success...", msgFromMQ.String()) +} +func (och *OnlineHistoryConsumerHandler) handleChatWs2MongoLowReliability(cMsg *sarama.ConsumerMessage, msgKey string, sess sarama.ConsumerGroupSession) { + msg := cMsg.Value + now := time.Now() + msgFromMQ := pbMsg.MsgDataToMQ{} + err := proto.Unmarshal(msg, &msgFromMQ) + if err != nil { + log.Error("msg_transfer Unmarshal msg err", "", "msg", string(msg), "err", err.Error()) + return + } + operationID := msgFromMQ.OperationID + log.NewInfo(operationID, "msg come mongo!!!", "", "msg", string(msg)) + //Control whether to store offline messages (mongo) + isHistory := utils.GetSwitchFromOptions(msgFromMQ.MsgData.Options, constant.IsHistory) + //Control whether to store history messages (mysql) + isPersist := utils.GetSwitchFromOptions(msgFromMQ.MsgData.Options, constant.IsPersistent) + isSenderSync := utils.GetSwitchFromOptions(msgFromMQ.MsgData.Options, constant.IsSenderSync) + switch msgFromMQ.MsgData.SessionType { + case constant.SingleChatType: + log.NewDebug(msgFromMQ.OperationID, "msg_transfer msg type = SingleChatType", isHistory, isPersist) + if isHistory { + seq, err := db.DB.IncrUserSeq(msgKey) + if err != nil { + log.NewError(operationID, "data insert to redis err", err.Error(), string(msg)) + return + } + sess.MarkMessage(cMsg, "") + msgFromMQ.MsgData.Seq = uint32(seq) + log.Debug(operationID, "send ch msg is ", msgFromMQ.String()) + och.msgCh <- Cmd2Value{Cmd: Msg, Value: MsgChannelValue{msgKey, &msgFromMQ}} + //err := saveUserChat(msgKey, &msgFromMQ) + //if err != nil { + // singleMsgFailedCount++ + // log.NewError(operationID, "single data insert to mongo err", err.Error(), msgFromMQ.String()) + // return + //} + //singleMsgSuccessCountMutex.Lock() + //singleMsgSuccessCount++ + //singleMsgSuccessCountMutex.Unlock() + //log.NewDebug(msgFromMQ.OperationID, "sendMessageToPush cost time ", time.Since(now)) + } + if !isSenderSync && msgKey == msgFromMQ.MsgData.SendID { + } else { + go sendMessageToPush(&msgFromMQ, msgKey) + } + log.NewDebug(operationID, "saveUserChat cost time ", time.Since(now)) + case constant.GroupChatType: + log.NewDebug(msgFromMQ.OperationID, "msg_transfer msg type = GroupChatType", isHistory, isPersist) + if isHistory { + seq, err := db.DB.IncrUserSeq(msgKey) + if err != nil { + log.NewError(operationID, "data insert to redis err", err.Error(), string(msg)) + return + } + sess.MarkMessage(cMsg, "") + msgFromMQ.MsgData.Seq = uint32(seq) + log.Debug(operationID, "send ch msg is ", msgFromMQ.String()) + och.msgCh <- Cmd2Value{Cmd: Msg, Value: MsgChannelValue{msgKey, &msgFromMQ}} + //err := saveUserChat(msgFromMQ.MsgData.RecvID, &msgFromMQ) + //if err != nil { + // log.NewError(operationID, "group data insert to mongo err", msgFromMQ.String(), msgFromMQ.MsgData.RecvID, err.Error()) + // return + //} + //groupMsgCount++ + } + go sendMessageToPush(&msgFromMQ, msgFromMQ.MsgData.RecvID) + case constant.NotificationChatType: + log.NewDebug(msgFromMQ.OperationID, "msg_transfer msg type = NotificationChatType", isHistory, isPersist) + if isHistory { + seq, err := db.DB.IncrUserSeq(msgKey) + if err != nil { + log.NewError(operationID, "data insert to redis err", err.Error(), string(msg)) + return + } + sess.MarkMessage(cMsg, "") + msgFromMQ.MsgData.Seq = uint32(seq) + log.Debug(operationID, "send ch msg is ", msgFromMQ.String()) + och.msgCh <- Cmd2Value{Cmd: Msg, Value: MsgChannelValue{msgKey, &msgFromMQ}} + //err := saveUserChat(msgKey, &msgFromMQ) + //if err != nil { + // log.NewError(operationID, "single data insert to mongo err", err.Error(), msgFromMQ.String()) + // return + //} + //log.NewDebug(msgFromMQ.OperationID, "sendMessageToPush cost time ", time.Since(now)) + } + if !isSenderSync && msgKey == msgFromMQ.MsgData.SendID { + } else { + go sendMessageToPush(&msgFromMQ, msgKey) + } + log.NewDebug(operationID, "saveUserChat cost time ", time.Since(now)) + default: + log.NewError(msgFromMQ.OperationID, "SessionType error", msgFromMQ.String()) + return + } log.NewDebug(msgFromMQ.OperationID, "msg_transfer handle topic data to database success...", msgFromMQ.String()) } @@ -138,8 +270,7 @@ func (och *OnlineHistoryConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupS SetOnlineTopicStatus(OnlineTopicBusy) //och.TriggerCmd(OnlineTopicBusy) log.NewDebug("", "online kafka get info to mongo", "msgTopic", msg.Topic, "msgPartition", msg.Partition, "online", msg.Offset, claim.HighWaterMarkOffset()) - och.msgHandle[msg.Topic](msg.Value, string(msg.Key)) - sess.MarkMessage(msg, "") + och.msgHandle[msg.Topic](msg, string(msg.Key), sess) if claim.HighWaterMarkOffset()-msg.Offset <= 1 { log.Debug("", "online msg consume end", claim.HighWaterMarkOffset(), msg.Offset) SetOnlineTopicStatus(OnlineTopicVacancy) diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index e898e5aa6..b7454c37b 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -209,7 +209,8 @@ type config struct { } Secret string `yaml:"secret"` MultiLoginPolicy int `yaml:"multiloginpolicy"` - ChatPersistenceMysql bool `yaml:"chatPersistenceMysql"` + ChatPersistenceMysql bool `yaml:"chatpersistencemysql"` + ReliableStorage bool `yaml:"reliablestorage"` TokenPolicy struct { AccessSecret string `yaml:"accessSecret"` From 98542bd5d5d659abc2e38f1b5d1a103895295422 Mon Sep 17 00:00:00 2001 From: Gordon <1432970085@qq.com> Date: Thu, 19 May 2022 12:28:14 +0800 Subject: [PATCH 06/76] concurrent consumption of messages --- config/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.yaml b/config/config.yaml index 0f1c5c4ec..ee973fe37 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -224,7 +224,7 @@ multiloginpolicy: 1 #chat log insert to db chatpersistencemysql: true #可靠性存储 -reliablestorage: true +reliablestorage: false #token config tokenpolicy: From 143e2908ad23c65d7b557323ef7963491185546f Mon Sep 17 00:00:00 2001 From: Gordon <1432970085@qq.com> Date: Thu, 19 May 2022 14:18:05 +0800 Subject: [PATCH 07/76] concurrent consumption of messages --- .../logic/offline_history_msg_handler.go | 139 +++++++++++++++++- 1 file changed, 132 insertions(+), 7 deletions(-) diff --git a/internal/msg_transfer/logic/offline_history_msg_handler.go b/internal/msg_transfer/logic/offline_history_msg_handler.go index 876beb30e..cf3f206cb 100644 --- a/internal/msg_transfer/logic/offline_history_msg_handler.go +++ b/internal/msg_transfer/logic/offline_history_msg_handler.go @@ -3,6 +3,7 @@ package logic import ( "Open_IM/pkg/common/config" "Open_IM/pkg/common/constant" + "Open_IM/pkg/common/db" kfk "Open_IM/pkg/common/kafka" "Open_IM/pkg/common/log" pbMsg "Open_IM/pkg/proto/chat" @@ -15,20 +16,50 @@ import ( type OfflineHistoryConsumerHandler struct { msgHandle map[string]fcb cmdCh chan Cmd2Value + msgCh chan Cmd2Value historyConsumerGroup *kfk.MConsumerGroup } func (mc *OfflineHistoryConsumerHandler) Init(cmdCh chan Cmd2Value) { mc.msgHandle = make(map[string]fcb) mc.cmdCh = cmdCh - mc.msgHandle[config.Config.Kafka.Ws2mschatOffline.Topic] = mc.handleChatWs2Mongo + mc.msgCh = make(chan Cmd2Value, 1000) + if config.Config.ReliableStorage { + mc.msgHandle[config.Config.Kafka.Ws2mschatOffline.Topic] = mc.handleChatWs2Mongo + } else { + mc.msgHandle[config.Config.Kafka.Ws2mschatOffline.Topic] = mc.handleChatWs2MongoLowReliability + for i := 0; i < 10; i++ { + go mc.Run() + + } + } mc.historyConsumerGroup = kfk.NewMConsumerGroup(&kfk.MConsumerGroupConfig{KafkaVersion: sarama.V0_10_2_0, OffsetsInitial: sarama.OffsetNewest, IsReturnErr: false}, []string{config.Config.Kafka.Ws2mschatOffline.Topic}, config.Config.Kafka.Ws2mschatOffline.Addr, config.Config.Kafka.ConsumerGroupID.MsgToMongoOffline) } - -func (mc *OfflineHistoryConsumerHandler) handleChatWs2Mongo(msg []byte, msgKey string) { +func (mc *OfflineHistoryConsumerHandler) Run() { + for { + select { + case cmd := <-mc.msgCh: + switch cmd.Cmd { + case Msg: + msgChannelValue := cmd.Value.(MsgChannelValue) + err := saveUserChat(msgChannelValue.userID, msgChannelValue.msg) + if err != nil { + singleMsgFailedCount++ + log.NewError(msgChannelValue.msg.OperationID, "single data insert to mongo err", err.Error(), msgChannelValue.msg.String()) + } else { + singleMsgSuccessCountMutex.Lock() + singleMsgSuccessCount++ + singleMsgSuccessCountMutex.Unlock() + } + } + } + } +} +func (mc *OfflineHistoryConsumerHandler) handleChatWs2Mongo(cMsg *sarama.ConsumerMessage, msgKey string, sess sarama.ConsumerGroupSession) { + msg := cMsg.Value now := time.Now() msgFromMQ := pbMsg.MsgDataToMQ{} err := proto.Unmarshal(msg, &msgFromMQ) @@ -95,6 +126,102 @@ func (mc *OfflineHistoryConsumerHandler) handleChatWs2Mongo(msg []byte, msgKey s log.NewError(msgFromMQ.OperationID, "SessionType error", msgFromMQ.String()) return } + sess.MarkMessage(cMsg, "") + log.NewDebug(msgFromMQ.OperationID, "msg_transfer handle topic data to database success...", msgFromMQ.String()) +} +func (mc *OfflineHistoryConsumerHandler) handleChatWs2MongoLowReliability(cMsg *sarama.ConsumerMessage, msgKey string, sess sarama.ConsumerGroupSession) { + msg := cMsg.Value + now := time.Now() + msgFromMQ := pbMsg.MsgDataToMQ{} + err := proto.Unmarshal(msg, &msgFromMQ) + if err != nil { + log.Error("msg_transfer Unmarshal msg err", "", "msg", string(msg), "err", err.Error()) + return + } + operationID := msgFromMQ.OperationID + log.NewInfo(operationID, "msg come mongo!!!", "", "msg", string(msg)) + //Control whether to store offline messages (mongo) + isHistory := utils.GetSwitchFromOptions(msgFromMQ.MsgData.Options, constant.IsHistory) + //Control whether to store history messages (mysql) + isPersist := utils.GetSwitchFromOptions(msgFromMQ.MsgData.Options, constant.IsPersistent) + isSenderSync := utils.GetSwitchFromOptions(msgFromMQ.MsgData.Options, constant.IsSenderSync) + switch msgFromMQ.MsgData.SessionType { + case constant.SingleChatType: + log.NewDebug(msgFromMQ.OperationID, "msg_transfer msg type = SingleChatType", isHistory, isPersist) + if isHistory { + seq, err := db.DB.IncrUserSeq(msgKey) + if err != nil { + log.NewError(operationID, "data insert to redis err", err.Error(), string(msg)) + return + } + sess.MarkMessage(cMsg, "") + msgFromMQ.MsgData.Seq = uint32(seq) + log.Debug(operationID, "send ch msg is ", msgFromMQ.String()) + mc.msgCh <- Cmd2Value{Cmd: Msg, Value: MsgChannelValue{msgKey, &msgFromMQ}} + //err := saveUserChat(msgKey, &msgFromMQ) + //if err != nil { + // singleMsgFailedCount++ + // log.NewError(operationID, "single data insert to mongo err", err.Error(), msgFromMQ.String()) + // return + //} + //singleMsgSuccessCountMutex.Lock() + //singleMsgSuccessCount++ + //singleMsgSuccessCountMutex.Unlock() + //log.NewDebug(msgFromMQ.OperationID, "sendMessageToPush cost time ", time.Since(now)) + } + if !isSenderSync && msgKey == msgFromMQ.MsgData.SendID { + } else { + go sendMessageToPush(&msgFromMQ, msgKey) + } + log.NewDebug(operationID, "saveUserChat cost time ", time.Since(now)) + case constant.GroupChatType: + log.NewDebug(msgFromMQ.OperationID, "msg_transfer msg type = GroupChatType", isHistory, isPersist) + if isHistory { + seq, err := db.DB.IncrUserSeq(msgKey) + if err != nil { + log.NewError(operationID, "data insert to redis err", err.Error(), string(msg)) + return + } + sess.MarkMessage(cMsg, "") + msgFromMQ.MsgData.Seq = uint32(seq) + log.Debug(operationID, "send ch msg is ", msgFromMQ.String()) + mc.msgCh <- Cmd2Value{Cmd: Msg, Value: MsgChannelValue{msgKey, &msgFromMQ}} + //err := saveUserChat(msgFromMQ.MsgData.RecvID, &msgFromMQ) + //if err != nil { + // log.NewError(operationID, "group data insert to mongo err", msgFromMQ.String(), msgFromMQ.MsgData.RecvID, err.Error()) + // return + //} + //groupMsgCount++ + } + go sendMessageToPush(&msgFromMQ, msgFromMQ.MsgData.RecvID) + case constant.NotificationChatType: + log.NewDebug(msgFromMQ.OperationID, "msg_transfer msg type = NotificationChatType", isHistory, isPersist) + if isHistory { + seq, err := db.DB.IncrUserSeq(msgKey) + if err != nil { + log.NewError(operationID, "data insert to redis err", err.Error(), string(msg)) + return + } + sess.MarkMessage(cMsg, "") + msgFromMQ.MsgData.Seq = uint32(seq) + log.Debug(operationID, "send ch msg is ", msgFromMQ.String()) + mc.msgCh <- Cmd2Value{Cmd: Msg, Value: MsgChannelValue{msgKey, &msgFromMQ}} + //err := saveUserChat(msgKey, &msgFromMQ) + //if err != nil { + // log.NewError(operationID, "single data insert to mongo err", err.Error(), msgFromMQ.String()) + // return + //} + //log.NewDebug(msgFromMQ.OperationID, "sendMessageToPush cost time ", time.Since(now)) + } + if !isSenderSync && msgKey == msgFromMQ.MsgData.SendID { + } else { + go sendMessageToPush(&msgFromMQ, msgKey) + } + log.NewDebug(operationID, "saveUserChat cost time ", time.Since(now)) + default: + log.NewError(msgFromMQ.OperationID, "SessionType error", msgFromMQ.String()) + return + } log.NewDebug(msgFromMQ.OperationID, "msg_transfer handle topic data to database success...", msgFromMQ.String()) } @@ -110,8 +237,7 @@ func (mc *OfflineHistoryConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupS for msg := range claim.Messages() { if GetOnlineTopicStatus() == OnlineTopicVacancy { log.NewDebug("", "vacancy offline kafka get info to mongo", "msgTopic", msg.Topic, "msgPartition", msg.Partition, "msg", string(msg.Value)) - mc.msgHandle[msg.Topic](msg.Value, string(msg.Key)) - sess.MarkMessage(msg, "") + mc.msgHandle[msg.Topic](msg, string(msg.Key), sess) } else { select { case <-mc.cmdCh: @@ -119,8 +245,7 @@ func (mc *OfflineHistoryConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupS case <-time.After(time.Millisecond * time.Duration(100)): log.NewDebug("", "timeout offline kafka get info to mongo", "msgTopic", msg.Topic, "msgPartition", msg.Partition, "msg", string(msg.Value)) } - mc.msgHandle[msg.Topic](msg.Value, string(msg.Key)) - sess.MarkMessage(msg, "") + mc.msgHandle[msg.Topic](msg, string(msg.Key), sess) } } From a2888b009d37de7634217522ba1b8757a651d524 Mon Sep 17 00:00:00 2001 From: Gordon <1432970085@qq.com> Date: Thu, 19 May 2022 14:21:38 +0800 Subject: [PATCH 08/76] concurrent consumption of messages --- internal/msg_transfer/logic/persistent_msg_handler.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/msg_transfer/logic/persistent_msg_handler.go b/internal/msg_transfer/logic/persistent_msg_handler.go index 6e330a664..7418e629e 100644 --- a/internal/msg_transfer/logic/persistent_msg_handler.go +++ b/internal/msg_transfer/logic/persistent_msg_handler.go @@ -32,7 +32,8 @@ func (pc *PersistentConsumerHandler) Init() { } -func (pc *PersistentConsumerHandler) handleChatWs2Mysql(msg []byte, msgKey string) { +func (pc *PersistentConsumerHandler) handleChatWs2Mysql(cMsg *sarama.ConsumerMessage, msgKey string, _ sarama.ConsumerGroupSession) { + msg := cMsg.Value log.NewInfo("msg come here mysql!!!", "", "msg", string(msg)) var tag bool msgFromMQ := pbMsg.MsgDataToMQ{} @@ -71,7 +72,7 @@ func (pc *PersistentConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupSessi claim sarama.ConsumerGroupClaim) error { for msg := range claim.Messages() { log.NewDebug("", "kafka get info to mysql", "msgTopic", msg.Topic, "msgPartition", msg.Partition, "msg", string(msg.Value)) - pc.msgHandle[msg.Topic](msg.Value, string(msg.Key)) + pc.msgHandle[msg.Topic](msg, string(msg.Key), sess) sess.MarkMessage(msg, "") } return nil From d086de81036a5f8bff98285f8fb0c7a5d3511f3e Mon Sep 17 00:00:00 2001 From: Gordon <1432970085@qq.com> Date: Thu, 19 May 2022 18:08:44 +0800 Subject: [PATCH 09/76] concurrent consumption of messages --- .../logic/offline_history_msg_handler.go | 108 +++++----------- .../logic/online_history_msg_handler.go | 119 ++++++------------ 2 files changed, 68 insertions(+), 159 deletions(-) diff --git a/internal/msg_transfer/logic/offline_history_msg_handler.go b/internal/msg_transfer/logic/offline_history_msg_handler.go index cf3f206cb..c5cac9a82 100644 --- a/internal/msg_transfer/logic/offline_history_msg_handler.go +++ b/internal/msg_transfer/logic/offline_history_msg_handler.go @@ -45,14 +45,20 @@ func (mc *OfflineHistoryConsumerHandler) Run() { switch cmd.Cmd { case Msg: msgChannelValue := cmd.Value.(MsgChannelValue) - err := saveUserChat(msgChannelValue.userID, msgChannelValue.msg) + msg := msgChannelValue.msg + log.Debug(msg.OperationID, "msg arrived channel", msg.String()) + isSenderSync := utils.GetSwitchFromOptions(msg.MsgData.Options, constant.IsSenderSync) + err := saveUserChat(msgChannelValue.userID, &msg) if err != nil { singleMsgFailedCount++ - log.NewError(msgChannelValue.msg.OperationID, "single data insert to mongo err", err.Error(), msgChannelValue.msg.String()) + log.NewError(msg.OperationID, "single data insert to mongo err", err.Error(), msg.String()) } else { singleMsgSuccessCountMutex.Lock() singleMsgSuccessCount++ singleMsgSuccessCountMutex.Unlock() + if !(!isSenderSync && msgChannelValue.userID == msg.MsgData.SendID) { + go sendMessageToPush(&msg, msgChannelValue.userID) + } } } } @@ -131,7 +137,6 @@ func (mc *OfflineHistoryConsumerHandler) handleChatWs2Mongo(cMsg *sarama.Consume } func (mc *OfflineHistoryConsumerHandler) handleChatWs2MongoLowReliability(cMsg *sarama.ConsumerMessage, msgKey string, sess sarama.ConsumerGroupSession) { msg := cMsg.Value - now := time.Now() msgFromMQ := pbMsg.MsgDataToMQ{} err := proto.Unmarshal(msg, &msgFromMQ) if err != nil { @@ -142,87 +147,32 @@ func (mc *OfflineHistoryConsumerHandler) handleChatWs2MongoLowReliability(cMsg * log.NewInfo(operationID, "msg come mongo!!!", "", "msg", string(msg)) //Control whether to store offline messages (mongo) isHistory := utils.GetSwitchFromOptions(msgFromMQ.MsgData.Options, constant.IsHistory) - //Control whether to store history messages (mysql) - isPersist := utils.GetSwitchFromOptions(msgFromMQ.MsgData.Options, constant.IsPersistent) isSenderSync := utils.GetSwitchFromOptions(msgFromMQ.MsgData.Options, constant.IsSenderSync) - switch msgFromMQ.MsgData.SessionType { - case constant.SingleChatType: - log.NewDebug(msgFromMQ.OperationID, "msg_transfer msg type = SingleChatType", isHistory, isPersist) - if isHistory { - seq, err := db.DB.IncrUserSeq(msgKey) - if err != nil { - log.NewError(operationID, "data insert to redis err", err.Error(), string(msg)) - return - } - sess.MarkMessage(cMsg, "") - msgFromMQ.MsgData.Seq = uint32(seq) - log.Debug(operationID, "send ch msg is ", msgFromMQ.String()) - mc.msgCh <- Cmd2Value{Cmd: Msg, Value: MsgChannelValue{msgKey, &msgFromMQ}} - //err := saveUserChat(msgKey, &msgFromMQ) - //if err != nil { - // singleMsgFailedCount++ - // log.NewError(operationID, "single data insert to mongo err", err.Error(), msgFromMQ.String()) - // return - //} - //singleMsgSuccessCountMutex.Lock() - //singleMsgSuccessCount++ - //singleMsgSuccessCountMutex.Unlock() - //log.NewDebug(msgFromMQ.OperationID, "sendMessageToPush cost time ", time.Since(now)) + if isHistory { + seq, err := db.DB.IncrUserSeq(msgKey) + if err != nil { + log.NewError(operationID, "data insert to redis err", err.Error(), string(msg)) + return } - if !isSenderSync && msgKey == msgFromMQ.MsgData.SendID { - } else { + sess.MarkMessage(cMsg, "") + msgFromMQ.MsgData.Seq = uint32(seq) + log.Debug(operationID, "send ch msg is ", msgFromMQ.String()) + mc.msgCh <- Cmd2Value{Cmd: Msg, Value: MsgChannelValue{msgKey, msgFromMQ}} + //err := saveUserChat(msgKey, &msgFromMQ) + //if err != nil { + // singleMsgFailedCount++ + // log.NewError(operationID, "single data insert to mongo err", err.Error(), msgFromMQ.String()) + // return + //} + //singleMsgSuccessCountMutex.Lock() + //singleMsgSuccessCount++ + //singleMsgSuccessCountMutex.Unlock() + //log.NewDebug(msgFromMQ.OperationID, "sendMessageToPush cost time ", time.Since(now)) + } else { + if !(!isSenderSync && msgKey == msgFromMQ.MsgData.SendID) { go sendMessageToPush(&msgFromMQ, msgKey) } - log.NewDebug(operationID, "saveUserChat cost time ", time.Since(now)) - case constant.GroupChatType: - log.NewDebug(msgFromMQ.OperationID, "msg_transfer msg type = GroupChatType", isHistory, isPersist) - if isHistory { - seq, err := db.DB.IncrUserSeq(msgKey) - if err != nil { - log.NewError(operationID, "data insert to redis err", err.Error(), string(msg)) - return - } - sess.MarkMessage(cMsg, "") - msgFromMQ.MsgData.Seq = uint32(seq) - log.Debug(operationID, "send ch msg is ", msgFromMQ.String()) - mc.msgCh <- Cmd2Value{Cmd: Msg, Value: MsgChannelValue{msgKey, &msgFromMQ}} - //err := saveUserChat(msgFromMQ.MsgData.RecvID, &msgFromMQ) - //if err != nil { - // log.NewError(operationID, "group data insert to mongo err", msgFromMQ.String(), msgFromMQ.MsgData.RecvID, err.Error()) - // return - //} - //groupMsgCount++ - } - go sendMessageToPush(&msgFromMQ, msgFromMQ.MsgData.RecvID) - case constant.NotificationChatType: - log.NewDebug(msgFromMQ.OperationID, "msg_transfer msg type = NotificationChatType", isHistory, isPersist) - if isHistory { - seq, err := db.DB.IncrUserSeq(msgKey) - if err != nil { - log.NewError(operationID, "data insert to redis err", err.Error(), string(msg)) - return - } - sess.MarkMessage(cMsg, "") - msgFromMQ.MsgData.Seq = uint32(seq) - log.Debug(operationID, "send ch msg is ", msgFromMQ.String()) - mc.msgCh <- Cmd2Value{Cmd: Msg, Value: MsgChannelValue{msgKey, &msgFromMQ}} - //err := saveUserChat(msgKey, &msgFromMQ) - //if err != nil { - // log.NewError(operationID, "single data insert to mongo err", err.Error(), msgFromMQ.String()) - // return - //} - //log.NewDebug(msgFromMQ.OperationID, "sendMessageToPush cost time ", time.Since(now)) - } - if !isSenderSync && msgKey == msgFromMQ.MsgData.SendID { - } else { - go sendMessageToPush(&msgFromMQ, msgKey) - } - log.NewDebug(operationID, "saveUserChat cost time ", time.Since(now)) - default: - log.NewError(msgFromMQ.OperationID, "SessionType error", msgFromMQ.String()) - return } - log.NewDebug(msgFromMQ.OperationID, "msg_transfer handle topic data to database success...", msgFromMQ.String()) } func (OfflineHistoryConsumerHandler) Setup(_ sarama.ConsumerGroupSession) error { return nil } diff --git a/internal/msg_transfer/logic/online_history_msg_handler.go b/internal/msg_transfer/logic/online_history_msg_handler.go index fb94ffe5f..ea787c107 100644 --- a/internal/msg_transfer/logic/online_history_msg_handler.go +++ b/internal/msg_transfer/logic/online_history_msg_handler.go @@ -20,7 +20,7 @@ import ( type MsgChannelValue struct { userID string - msg *pbMsg.MsgDataToMQ + msg pbMsg.MsgDataToMQ } type fcb func(cMsg *sarama.ConsumerMessage, msgKey string, sess sarama.ConsumerGroupSession) type Cmd2Value struct { @@ -83,14 +83,29 @@ func (och *OnlineHistoryConsumerHandler) Run() { switch cmd.Cmd { case Msg: msgChannelValue := cmd.Value.(MsgChannelValue) - err := saveUserChat(msgChannelValue.userID, msgChannelValue.msg) + msg := msgChannelValue.msg + log.Debug(msg.OperationID, "msg arrived channel", msg.String()) + isSenderSync := utils.GetSwitchFromOptions(msg.MsgData.Options, constant.IsSenderSync) + //switch msgChannelValue.msg.MsgData.SessionType { + //case constant.SingleChatType: + //case constant.GroupChatType: + //case constant.NotificationChatType: + //default: + // log.NewError(msgFromMQ.OperationID, "SessionType error", msgFromMQ.String()) + // return + //} + + err := saveUserChat(msgChannelValue.userID, &msg) if err != nil { singleMsgFailedCount++ - log.NewError(msgChannelValue.msg.OperationID, "single data insert to mongo err", err.Error(), msgChannelValue.msg.String()) + log.NewError(msg.OperationID, "single data insert to mongo err", err.Error(), msg.String()) } else { singleMsgSuccessCountMutex.Lock() singleMsgSuccessCount++ singleMsgSuccessCountMutex.Unlock() + if !(!isSenderSync && msgChannelValue.userID == msg.MsgData.SendID) { + go sendMessageToPush(&msg, msgChannelValue.userID) + } } } } @@ -167,7 +182,6 @@ func (och *OnlineHistoryConsumerHandler) handleChatWs2Mongo(cMsg *sarama.Consume } func (och *OnlineHistoryConsumerHandler) handleChatWs2MongoLowReliability(cMsg *sarama.ConsumerMessage, msgKey string, sess sarama.ConsumerGroupSession) { msg := cMsg.Value - now := time.Now() msgFromMQ := pbMsg.MsgDataToMQ{} err := proto.Unmarshal(msg, &msgFromMQ) if err != nil { @@ -178,87 +192,32 @@ func (och *OnlineHistoryConsumerHandler) handleChatWs2MongoLowReliability(cMsg * log.NewInfo(operationID, "msg come mongo!!!", "", "msg", string(msg)) //Control whether to store offline messages (mongo) isHistory := utils.GetSwitchFromOptions(msgFromMQ.MsgData.Options, constant.IsHistory) - //Control whether to store history messages (mysql) - isPersist := utils.GetSwitchFromOptions(msgFromMQ.MsgData.Options, constant.IsPersistent) isSenderSync := utils.GetSwitchFromOptions(msgFromMQ.MsgData.Options, constant.IsSenderSync) - switch msgFromMQ.MsgData.SessionType { - case constant.SingleChatType: - log.NewDebug(msgFromMQ.OperationID, "msg_transfer msg type = SingleChatType", isHistory, isPersist) - if isHistory { - seq, err := db.DB.IncrUserSeq(msgKey) - if err != nil { - log.NewError(operationID, "data insert to redis err", err.Error(), string(msg)) - return - } - sess.MarkMessage(cMsg, "") - msgFromMQ.MsgData.Seq = uint32(seq) - log.Debug(operationID, "send ch msg is ", msgFromMQ.String()) - och.msgCh <- Cmd2Value{Cmd: Msg, Value: MsgChannelValue{msgKey, &msgFromMQ}} - //err := saveUserChat(msgKey, &msgFromMQ) - //if err != nil { - // singleMsgFailedCount++ - // log.NewError(operationID, "single data insert to mongo err", err.Error(), msgFromMQ.String()) - // return - //} - //singleMsgSuccessCountMutex.Lock() - //singleMsgSuccessCount++ - //singleMsgSuccessCountMutex.Unlock() - //log.NewDebug(msgFromMQ.OperationID, "sendMessageToPush cost time ", time.Since(now)) + if isHistory { + seq, err := db.DB.IncrUserSeq(msgKey) + if err != nil { + log.NewError(operationID, "data insert to redis err", err.Error(), string(msg)) + return } - if !isSenderSync && msgKey == msgFromMQ.MsgData.SendID { - } else { + sess.MarkMessage(cMsg, "") + msgFromMQ.MsgData.Seq = uint32(seq) + log.Debug(operationID, "send ch msg is ", msgFromMQ.String()) + och.msgCh <- Cmd2Value{Cmd: Msg, Value: MsgChannelValue{msgKey, msgFromMQ}} + //err := saveUserChat(msgKey, &msgFromMQ) + //if err != nil { + // singleMsgFailedCount++ + // log.NewError(operationID, "single data insert to mongo err", err.Error(), msgFromMQ.String()) + // return + //} + //singleMsgSuccessCountMutex.Lock() + //singleMsgSuccessCount++ + //singleMsgSuccessCountMutex.Unlock() + //log.NewDebug(msgFromMQ.OperationID, "sendMessageToPush cost time ", time.Since(now)) + } else { + if !(!isSenderSync && msgKey == msgFromMQ.MsgData.SendID) { go sendMessageToPush(&msgFromMQ, msgKey) } - log.NewDebug(operationID, "saveUserChat cost time ", time.Since(now)) - case constant.GroupChatType: - log.NewDebug(msgFromMQ.OperationID, "msg_transfer msg type = GroupChatType", isHistory, isPersist) - if isHistory { - seq, err := db.DB.IncrUserSeq(msgKey) - if err != nil { - log.NewError(operationID, "data insert to redis err", err.Error(), string(msg)) - return - } - sess.MarkMessage(cMsg, "") - msgFromMQ.MsgData.Seq = uint32(seq) - log.Debug(operationID, "send ch msg is ", msgFromMQ.String()) - och.msgCh <- Cmd2Value{Cmd: Msg, Value: MsgChannelValue{msgKey, &msgFromMQ}} - //err := saveUserChat(msgFromMQ.MsgData.RecvID, &msgFromMQ) - //if err != nil { - // log.NewError(operationID, "group data insert to mongo err", msgFromMQ.String(), msgFromMQ.MsgData.RecvID, err.Error()) - // return - //} - //groupMsgCount++ - } - go sendMessageToPush(&msgFromMQ, msgFromMQ.MsgData.RecvID) - case constant.NotificationChatType: - log.NewDebug(msgFromMQ.OperationID, "msg_transfer msg type = NotificationChatType", isHistory, isPersist) - if isHistory { - seq, err := db.DB.IncrUserSeq(msgKey) - if err != nil { - log.NewError(operationID, "data insert to redis err", err.Error(), string(msg)) - return - } - sess.MarkMessage(cMsg, "") - msgFromMQ.MsgData.Seq = uint32(seq) - log.Debug(operationID, "send ch msg is ", msgFromMQ.String()) - och.msgCh <- Cmd2Value{Cmd: Msg, Value: MsgChannelValue{msgKey, &msgFromMQ}} - //err := saveUserChat(msgKey, &msgFromMQ) - //if err != nil { - // log.NewError(operationID, "single data insert to mongo err", err.Error(), msgFromMQ.String()) - // return - //} - //log.NewDebug(msgFromMQ.OperationID, "sendMessageToPush cost time ", time.Since(now)) - } - if !isSenderSync && msgKey == msgFromMQ.MsgData.SendID { - } else { - go sendMessageToPush(&msgFromMQ, msgKey) - } - log.NewDebug(operationID, "saveUserChat cost time ", time.Since(now)) - default: - log.NewError(msgFromMQ.OperationID, "SessionType error", msgFromMQ.String()) - return } - log.NewDebug(msgFromMQ.OperationID, "msg_transfer handle topic data to database success...", msgFromMQ.String()) } func (OnlineHistoryConsumerHandler) Setup(_ sarama.ConsumerGroupSession) error { return nil } From d4f81f086cf496ae26eed03987e1a9ea0804f622 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Fri, 20 May 2022 11:00:11 +0800 Subject: [PATCH 10/76] batch to mongo --- internal/msg_transfer/logic/db.go | 23 ++++++--- pkg/common/db/batch_insert_chat.go | 78 ++++++++++++++++++++++++++++++ pkg/common/db/mongoModel.go | 47 +++++++++++++++++- 3 files changed, 139 insertions(+), 9 deletions(-) create mode 100644 pkg/common/db/batch_insert_chat.go diff --git a/internal/msg_transfer/logic/db.go b/internal/msg_transfer/logic/db.go index 98a684cc7..74106999c 100644 --- a/internal/msg_transfer/logic/db.go +++ b/internal/msg_transfer/logic/db.go @@ -2,20 +2,27 @@ package logic import ( "Open_IM/pkg/common/db" + "Open_IM/pkg/common/log" pbMsg "Open_IM/pkg/proto/chat" + "Open_IM/pkg/utils" ) func saveUserChat(uid string, msg *pbMsg.MsgDataToMQ) error { - //time := utils.GetCurrentTimestampByMill() - //seq, err := db.DB.IncrUserSeq(uid) - //if err != nil { - // log.NewError(msg.OperationID, "data insert to redis err", err.Error(), msg.String()) - // return err - //} - //msg.MsgData.Seq = uint32(seq) + time := utils.GetCurrentTimestampByMill() + seq, err := db.DB.IncrUserSeq(uid) + if err != nil { + log.NewError(msg.OperationID, "data insert to redis err", err.Error(), msg.String()) + return err + } + msg.MsgData.Seq = uint32(seq) pbSaveData := pbMsg.MsgDataToDB{} pbSaveData.MsgData = msg.MsgData - //log.NewInfo(msg.OperationID, "IncrUserSeq cost time", utils.GetCurrentTimestampByMill()-time) + log.NewInfo(msg.OperationID, "IncrUserSeq cost time", utils.GetCurrentTimestampByMill()-time) return db.DB.SaveUserChatMongo2(uid, pbSaveData.MsgData.SendTime, &pbSaveData) // return db.DB.SaveUserChatMongo2(uid, pbSaveData.MsgData.SendTime, &pbSaveData) } + +func saveUserChatList(uid string, msgList []*pbMsg.MsgDataToMQ, operationID string) error { + log.Info(operationID, utils.GetSelfFuncName(), "args ", uid, len(msgList)) + return db.DB.BatchInsertChat(uid, msgList, operationID) +} diff --git a/pkg/common/db/batch_insert_chat.go b/pkg/common/db/batch_insert_chat.go new file mode 100644 index 000000000..9c23244f6 --- /dev/null +++ b/pkg/common/db/batch_insert_chat.go @@ -0,0 +1,78 @@ +package db + +import ( + "Open_IM/pkg/common/config" + "Open_IM/pkg/common/log" + pbMsg "Open_IM/pkg/proto/chat" + "Open_IM/pkg/utils" + "context" + "errors" + "github.com/garyburd/redigo/redis" + "github.com/golang/protobuf/proto" + "go.mongodb.org/mongo-driver/bson" + "time" +) + +func (d *DataBases) BatchInsertChat(userID string, msgList []*pbMsg.MsgDataToMQ, operationID string) error { + if len(msgList) > GetSingleGocMsgNum() { + return errors.New("too large") + } + currentMaxSeq, err := d.GetUserMaxSeq(userID) + if err == nil { + + } else if err == redis.ErrNil { + currentMaxSeq = 0 + } else { + return utils.Wrap(err, "") + } + + remain := currentMaxSeq % uint64(GetSingleGocMsgNum()) + insertCounter := uint64(0) + msgListToMongo := make([]MsgInfo, 0) + msgListToMongoNext := make([]MsgInfo, 0) + seqUid := "" + seqUidNext := "" + for _, m := range msgList { + currentMaxSeq++ + sMsg := MsgInfo{} + sMsg.SendTime = m.MsgData.SendTime + m.MsgData.Seq = uint32(currentMaxSeq) + if sMsg.Msg, err = proto.Marshal(m.MsgData); err != nil { + return utils.Wrap(err, "") + } + if insertCounter < remain { + msgListToMongo = append(msgListToMongo, sMsg) + insertCounter++ + seqUid = getSeqUid(userID, uint32(currentMaxSeq)) + } else { + msgListToMongoNext = append(msgListToMongoNext, sMsg) + seqUidNext = getSeqUid(userID, uint32(currentMaxSeq)) + } + } + ctx, _ := context.WithTimeout(context.Background(), time.Duration(config.Config.Mongo.DBTimeout)*time.Second) + c := d.mongoClient.Database(config.Config.Mongo.DBDatabase).Collection(cChat) + newTime := getCurrentTimestampByMill() + + if seqUid != "" { + filter := bson.M{"uid": seqUid} + log.NewDebug(operationID, "filter ", seqUid) + err := c.FindOneAndUpdate(ctx, filter, bson.M{"$push": bson.M{"msg": bson.M{"$each": msgListToMongo}}}).Err() + if err != nil { + log.Error(operationID, "FindOneAndUpdate failed ", err.Error(), filter) + return utils.Wrap(err, "") + } + } + + if seqUidNext != "" { + filter := bson.M{"uid": seqUidNext} + sChat := UserChat{} + sChat.UID = seqUidNext + sChat.Msg = msgListToMongoNext + if _, err = c.InsertOne(ctx, &sChat); err != nil { + log.NewError(operationID, "InsertOne failed", filter, err.Error(), sChat) + return utils.Wrap(err, "") + } + } + log.NewDebug(operationID, "find mgo uid cost time", getCurrentTimestampByMill()-newTime) + return nil +} diff --git a/pkg/common/db/mongoModel.go b/pkg/common/db/mongoModel.go index 08d605113..96206c6f2 100644 --- a/pkg/common/db/mongoModel.go +++ b/pkg/common/db/mongoModel.go @@ -31,6 +31,10 @@ const cWorkMoment = "work_moment" const cCommentMsg = "comment_msg" const singleGocMsgNum = 5000 +func GetSingleGocMsgNum() int { + return singleGocMsgNum +} + type MsgInfo struct { SendTime int64 Msg []byte @@ -351,7 +355,7 @@ func (d *DataBases) SaveUserChatMongo2(uid string, sendTime int64, m *pbMsg.MsgD return utils.Wrap(err, "") } err = c.FindOneAndUpdate(ctx, filter, bson.M{"$push": bson.M{"msg": sMsg}}).Err() - log.NewDebug(operationID, "get mgoSession cost time", getCurrentTimestampByMill()-newTime) + log.NewWarn(operationID, "get mgoSession cost time", getCurrentTimestampByMill()-newTime) if err != nil { sChat := UserChat{} sChat.UID = seqUid @@ -368,6 +372,47 @@ func (d *DataBases) SaveUserChatMongo2(uid string, sendTime int64, m *pbMsg.MsgD return nil } +// +//func (d *DataBases) SaveUserChatListMongo2(uid string, sendTime int64, msgList []*pbMsg.MsgDataToDB) error { +// ctx, _ := context.WithTimeout(context.Background(), time.Duration(config.Config.Mongo.DBTimeout)*time.Second) +// c := d.mongoClient.Database(config.Config.Mongo.DBDatabase).Collection(cChat) +// newTime := getCurrentTimestampByMill() +// operationID := "" +// seqUid := "" +// msgListToMongo := make([]MsgInfo, 0) +// +// for _, m := range msgList { +// seqUid = getSeqUid(uid, m.MsgData.Seq) +// var err error +// sMsg := MsgInfo{} +// sMsg.SendTime = sendTime +// if sMsg.Msg, err = proto.Marshal(m.MsgData); err != nil { +// return utils.Wrap(err, "") +// } +// msgListToMongo = append(msgListToMongo, sMsg) +// } +// +// filter := bson.M{"uid": seqUid} +// log.NewDebug(operationID, "filter ", seqUid) +// err := c.FindOneAndUpdate(ctx, filter, bson.M{"$push": bson.M{"msg": bson.M{"$each": msgListToMongo}}}).Err() +// log.NewWarn(operationID, "get mgoSession cost time", getCurrentTimestampByMill()-newTime) +// if err != nil { +// sChat := UserChat{} +// sChat.UID = seqUid +// sChat.Msg = msgListToMongo +// +// if _, err = c.InsertOne(ctx, &sChat); err != nil { +// log.NewError(operationID, "InsertOne failed", filter, err.Error(), sChat) +// return utils.Wrap(err, "") +// } +// } else { +// log.NewDebug(operationID, "FindOneAndUpdate ok", filter) +// } +// +// log.NewDebug(operationID, "find mgo uid cost time", getCurrentTimestampByMill()-newTime) +// return nil +//} + func (d *DataBases) SaveUserChat(uid string, sendTime int64, m *pbMsg.MsgDataToDB) error { var seqUid string newTime := getCurrentTimestampByMill() From 00a2a23190ae688f806a7becc82a1080fb24fae6 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Fri, 20 May 2022 11:20:21 +0800 Subject: [PATCH 11/76] batch to mongo --- internal/msg_transfer/logic/db.go | 6 +++--- pkg/common/db/batch_insert_chat.go | 2 +- pkg/common/db/redisModel.go | 7 +++++++ 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/internal/msg_transfer/logic/db.go b/internal/msg_transfer/logic/db.go index 74106999c..7a2af5404 100644 --- a/internal/msg_transfer/logic/db.go +++ b/internal/msg_transfer/logic/db.go @@ -22,7 +22,7 @@ func saveUserChat(uid string, msg *pbMsg.MsgDataToMQ) error { // return db.DB.SaveUserChatMongo2(uid, pbSaveData.MsgData.SendTime, &pbSaveData) } -func saveUserChatList(uid string, msgList []*pbMsg.MsgDataToMQ, operationID string) error { - log.Info(operationID, utils.GetSelfFuncName(), "args ", uid, len(msgList)) - return db.DB.BatchInsertChat(uid, msgList, operationID) +func saveUserChatList(userID string, msgList []*pbMsg.MsgDataToMQ, operationID string) error { + log.Info(operationID, utils.GetSelfFuncName(), "args ", userID, len(msgList)) + return db.DB.BatchInsertChat(userID, msgList, operationID) } diff --git a/pkg/common/db/batch_insert_chat.go b/pkg/common/db/batch_insert_chat.go index 9c23244f6..f64fc4a18 100644 --- a/pkg/common/db/batch_insert_chat.go +++ b/pkg/common/db/batch_insert_chat.go @@ -74,5 +74,5 @@ func (d *DataBases) BatchInsertChat(userID string, msgList []*pbMsg.MsgDataToMQ, } } log.NewDebug(operationID, "find mgo uid cost time", getCurrentTimestampByMill()-newTime) - return nil + return utils.Wrap(d.SetUserMaxSeq(userID, uint32(currentMaxSeq)), "") } diff --git a/pkg/common/db/redisModel.go b/pkg/common/db/redisModel.go index 9996ea356..01f161f84 100644 --- a/pkg/common/db/redisModel.go +++ b/pkg/common/db/redisModel.go @@ -68,6 +68,13 @@ func (d *DataBases) GetUserMaxSeq(uid string) (uint64, error) { return redis.Uint64(d.Exec("GET", key)) } +//set the largest Seq +func (d *DataBases) SetUserMaxSeq(uid string, maxSeq uint32) error { + key := userIncrSeq + uid + _, err := redis.Uint64(d.Exec("SET", key, maxSeq)) + return err +} + //Set the user's minimum seq func (d *DataBases) SetUserMinSeq(uid string, minSeq uint32) (err error) { key := userMinSeq + uid From 31911a40c0d060f569217992cde268e1f414d6a4 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Fri, 20 May 2022 11:24:10 +0800 Subject: [PATCH 12/76] batch to mongo --- pkg/common/db/batch_insert_chat.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/common/db/batch_insert_chat.go b/pkg/common/db/batch_insert_chat.go index f64fc4a18..c6435525f 100644 --- a/pkg/common/db/batch_insert_chat.go +++ b/pkg/common/db/batch_insert_chat.go @@ -14,6 +14,7 @@ import ( ) func (d *DataBases) BatchInsertChat(userID string, msgList []*pbMsg.MsgDataToMQ, operationID string) error { + newTime := getCurrentTimestampByMill() if len(msgList) > GetSingleGocMsgNum() { return errors.New("too large") } @@ -51,28 +52,27 @@ func (d *DataBases) BatchInsertChat(userID string, msgList []*pbMsg.MsgDataToMQ, } ctx, _ := context.WithTimeout(context.Background(), time.Duration(config.Config.Mongo.DBTimeout)*time.Second) c := d.mongoClient.Database(config.Config.Mongo.DBDatabase).Collection(cChat) - newTime := getCurrentTimestampByMill() if seqUid != "" { filter := bson.M{"uid": seqUid} - log.NewDebug(operationID, "filter ", seqUid) + log.NewDebug(operationID, "filter ", seqUid, "list ", msgListToMongo) err := c.FindOneAndUpdate(ctx, filter, bson.M{"$push": bson.M{"msg": bson.M{"$each": msgListToMongo}}}).Err() if err != nil { log.Error(operationID, "FindOneAndUpdate failed ", err.Error(), filter) return utils.Wrap(err, "") } } - if seqUidNext != "" { filter := bson.M{"uid": seqUidNext} sChat := UserChat{} sChat.UID = seqUidNext sChat.Msg = msgListToMongoNext + log.NewDebug(operationID, "filter ", seqUidNext, "list ", msgListToMongoNext) if _, err = c.InsertOne(ctx, &sChat); err != nil { log.NewError(operationID, "InsertOne failed", filter, err.Error(), sChat) return utils.Wrap(err, "") } } - log.NewDebug(operationID, "find mgo uid cost time", getCurrentTimestampByMill()-newTime) + log.NewWarn(operationID, "batch mgo cost time ", getCurrentTimestampByMill()-newTime, userID, len(msgList)) return utils.Wrap(d.SetUserMaxSeq(userID, uint32(currentMaxSeq)), "") } From b914e8388dcaf29cf0af91b179a3cb807709e67b Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Fri, 20 May 2022 12:11:46 +0800 Subject: [PATCH 13/76] batch to mongo --- pkg/common/db/batch_insert_chat.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/common/db/batch_insert_chat.go b/pkg/common/db/batch_insert_chat.go index c6435525f..82c9638a1 100644 --- a/pkg/common/db/batch_insert_chat.go +++ b/pkg/common/db/batch_insert_chat.go @@ -45,9 +45,11 @@ func (d *DataBases) BatchInsertChat(userID string, msgList []*pbMsg.MsgDataToMQ, msgListToMongo = append(msgListToMongo, sMsg) insertCounter++ seqUid = getSeqUid(userID, uint32(currentMaxSeq)) + log.Debug(operationID, "msgListToMongo ", seqUid, m.MsgData.Seq, m.MsgData.ClientMsgID) } else { msgListToMongoNext = append(msgListToMongoNext, sMsg) seqUidNext = getSeqUid(userID, uint32(currentMaxSeq)) + log.Debug(operationID, "msgListToMongoNext ", seqUidNext, m.MsgData.Seq, m.MsgData.ClientMsgID) } } ctx, _ := context.WithTimeout(context.Background(), time.Duration(config.Config.Mongo.DBTimeout)*time.Second) From 0fbc1095be5f8eca0aba85bbfc6a922dcb81a96b Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Fri, 20 May 2022 12:21:29 +0800 Subject: [PATCH 14/76] batch to mongo --- pkg/common/db/batch_insert_chat.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/common/db/batch_insert_chat.go b/pkg/common/db/batch_insert_chat.go index 82c9638a1..3ce323df9 100644 --- a/pkg/common/db/batch_insert_chat.go +++ b/pkg/common/db/batch_insert_chat.go @@ -33,6 +33,7 @@ func (d *DataBases) BatchInsertChat(userID string, msgList []*pbMsg.MsgDataToMQ, msgListToMongoNext := make([]MsgInfo, 0) seqUid := "" seqUidNext := "" + log.Debug(operationID, "remain ", remain, "insertCounter ", insertCounter, "currentMaxSeq ", currentMaxSeq) for _, m := range msgList { currentMaxSeq++ sMsg := MsgInfo{} @@ -45,11 +46,11 @@ func (d *DataBases) BatchInsertChat(userID string, msgList []*pbMsg.MsgDataToMQ, msgListToMongo = append(msgListToMongo, sMsg) insertCounter++ seqUid = getSeqUid(userID, uint32(currentMaxSeq)) - log.Debug(operationID, "msgListToMongo ", seqUid, m.MsgData.Seq, m.MsgData.ClientMsgID) + log.Debug(operationID, "msgListToMongo ", seqUid, m.MsgData.Seq, m.MsgData.ClientMsgID, insertCounter, remain) } else { msgListToMongoNext = append(msgListToMongoNext, sMsg) seqUidNext = getSeqUid(userID, uint32(currentMaxSeq)) - log.Debug(operationID, "msgListToMongoNext ", seqUidNext, m.MsgData.Seq, m.MsgData.ClientMsgID) + log.Debug(operationID, "msgListToMongoNext ", seqUidNext, m.MsgData.Seq, m.MsgData.ClientMsgID, insertCounter, remain) } } ctx, _ := context.WithTimeout(context.Background(), time.Duration(config.Config.Mongo.DBTimeout)*time.Second) From ea8638c8432ed5e9358ab2d24af9864bbc65c4a2 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Fri, 20 May 2022 13:01:08 +0800 Subject: [PATCH 15/76] batch to mongo --- pkg/common/db/batch_insert_chat.go | 4 ++-- pkg/common/db/redisModel.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/common/db/batch_insert_chat.go b/pkg/common/db/batch_insert_chat.go index 3ce323df9..1cc910e13 100644 --- a/pkg/common/db/batch_insert_chat.go +++ b/pkg/common/db/batch_insert_chat.go @@ -33,7 +33,7 @@ func (d *DataBases) BatchInsertChat(userID string, msgList []*pbMsg.MsgDataToMQ, msgListToMongoNext := make([]MsgInfo, 0) seqUid := "" seqUidNext := "" - log.Debug(operationID, "remain ", remain, "insertCounter ", insertCounter, "currentMaxSeq ", currentMaxSeq) + log.Debug(operationID, "remain ", remain, "insertCounter ", insertCounter, "currentMaxSeq ", currentMaxSeq, userID) for _, m := range msgList { currentMaxSeq++ sMsg := MsgInfo{} @@ -77,5 +77,5 @@ func (d *DataBases) BatchInsertChat(userID string, msgList []*pbMsg.MsgDataToMQ, } } log.NewWarn(operationID, "batch mgo cost time ", getCurrentTimestampByMill()-newTime, userID, len(msgList)) - return utils.Wrap(d.SetUserMaxSeq(userID, uint32(currentMaxSeq)), "") + return utils.Wrap(d.SetUserMaxSeq(userID, uint64(currentMaxSeq)), "") } diff --git a/pkg/common/db/redisModel.go b/pkg/common/db/redisModel.go index 01f161f84..d7165ba94 100644 --- a/pkg/common/db/redisModel.go +++ b/pkg/common/db/redisModel.go @@ -69,7 +69,7 @@ func (d *DataBases) GetUserMaxSeq(uid string) (uint64, error) { } //set the largest Seq -func (d *DataBases) SetUserMaxSeq(uid string, maxSeq uint32) error { +func (d *DataBases) SetUserMaxSeq(uid string, maxSeq uint64) error { key := userIncrSeq + uid _, err := redis.Uint64(d.Exec("SET", key, maxSeq)) return err From 17695927d54a7195ff71bd3a4b97946bf5f057ad Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Fri, 20 May 2022 13:09:28 +0800 Subject: [PATCH 16/76] batch to mongo --- pkg/common/db/batch_insert_chat.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/common/db/batch_insert_chat.go b/pkg/common/db/batch_insert_chat.go index 1cc910e13..e327ba7b5 100644 --- a/pkg/common/db/batch_insert_chat.go +++ b/pkg/common/db/batch_insert_chat.go @@ -27,7 +27,8 @@ func (d *DataBases) BatchInsertChat(userID string, msgList []*pbMsg.MsgDataToMQ, return utils.Wrap(err, "") } - remain := currentMaxSeq % uint64(GetSingleGocMsgNum()) + //4999 + remain := uint64(GetSingleGocMsgNum()) - (currentMaxSeq % uint64(GetSingleGocMsgNum())) insertCounter := uint64(0) msgListToMongo := make([]MsgInfo, 0) msgListToMongoNext := make([]MsgInfo, 0) From 866c7d4f2f261a64b4ebdd8ea1246cd235215662 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Fri, 20 May 2022 13:15:32 +0800 Subject: [PATCH 17/76] batch to mongo --- pkg/common/db/redisModel.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/common/db/redisModel.go b/pkg/common/db/redisModel.go index d7165ba94..46318ac4a 100644 --- a/pkg/common/db/redisModel.go +++ b/pkg/common/db/redisModel.go @@ -71,7 +71,7 @@ func (d *DataBases) GetUserMaxSeq(uid string) (uint64, error) { //set the largest Seq func (d *DataBases) SetUserMaxSeq(uid string, maxSeq uint64) error { key := userIncrSeq + uid - _, err := redis.Uint64(d.Exec("SET", key, maxSeq)) + _, err := d.Exec("SET", key, maxSeq) return err } From 06726758d218d96330a559378b2b0b15e8b0f700 Mon Sep 17 00:00:00 2001 From: Gordon <1432970085@qq.com> Date: Fri, 20 May 2022 13:33:38 +0800 Subject: [PATCH 18/76] concurrent consumption of messages --- internal/msg_transfer/logic/init.go | 3 + .../logic/offline_history_msg_handler.go | 175 ++++++++++---- .../logic/online_history_msg_handler.go | 225 +++++++++++++++--- 3 files changed, 336 insertions(+), 67 deletions(-) diff --git a/internal/msg_transfer/logic/init.go b/internal/msg_transfer/logic/init.go index bef20e8c7..b4cfff6db 100644 --- a/internal/msg_transfer/logic/init.go +++ b/internal/msg_transfer/logic/init.go @@ -13,6 +13,9 @@ import ( const OnlineTopicBusy = 1 const OnlineTopicVacancy = 0 const Msg = 2 +const ConsumerMsgs = 3 +const UserMessages = 4 +const ChannelNum = 10 var ( persistentCH PersistentConsumerHandler diff --git a/internal/msg_transfer/logic/offline_history_msg_handler.go b/internal/msg_transfer/logic/offline_history_msg_handler.go index c5cac9a82..4fdd30577 100644 --- a/internal/msg_transfer/logic/offline_history_msg_handler.go +++ b/internal/msg_transfer/logic/offline_history_msg_handler.go @@ -15,22 +15,28 @@ import ( type OfflineHistoryConsumerHandler struct { msgHandle map[string]fcb + historyConsumerGroup *kfk.MConsumerGroup cmdCh chan Cmd2Value msgCh chan Cmd2Value - historyConsumerGroup *kfk.MConsumerGroup + UserAggregationMsgs map[string][]*pbMsg.MsgDataToMQ + chArrays [ChannelNum]chan Cmd2Value + msgDistributionCh chan Cmd2Value } func (mc *OfflineHistoryConsumerHandler) Init(cmdCh chan Cmd2Value) { mc.msgHandle = make(map[string]fcb) + mc.UserAggregationMsgs = make(map[string][]*pbMsg.MsgDataToMQ, ChannelNum) + mc.msgDistributionCh = make(chan Cmd2Value) //no buffer channel + go mc.MessagesDistributionHandle() mc.cmdCh = cmdCh mc.msgCh = make(chan Cmd2Value, 1000) if config.Config.ReliableStorage { - mc.msgHandle[config.Config.Kafka.Ws2mschatOffline.Topic] = mc.handleChatWs2Mongo + mc.msgHandle[config.Config.Kafka.Ws2mschat.Topic] = mc.handleChatWs2Mongo } else { - mc.msgHandle[config.Config.Kafka.Ws2mschatOffline.Topic] = mc.handleChatWs2MongoLowReliability - for i := 0; i < 10; i++ { - go mc.Run() - + mc.msgHandle[config.Config.Kafka.Ws2mschat.Topic] = mc.handleChatWs2MongoLowReliability + for i := 0; i < ChannelNum; i++ { + mc.chArrays[i] = make(chan Cmd2Value, 1000) + go mc.Run(i) } } mc.historyConsumerGroup = kfk.NewMConsumerGroup(&kfk.MConsumerGroupConfig{KafkaVersion: sarama.V0_10_2_0, @@ -38,31 +44,91 @@ func (mc *OfflineHistoryConsumerHandler) Init(cmdCh chan Cmd2Value) { config.Config.Kafka.Ws2mschatOffline.Addr, config.Config.Kafka.ConsumerGroupID.MsgToMongoOffline) } -func (mc *OfflineHistoryConsumerHandler) Run() { +func (och *OfflineHistoryConsumerHandler) Run(channelID int) { for { select { - case cmd := <-mc.msgCh: + case cmd := <-och.chArrays[channelID]: switch cmd.Cmd { - case Msg: + case UserMessages: msgChannelValue := cmd.Value.(MsgChannelValue) - msg := msgChannelValue.msg - log.Debug(msg.OperationID, "msg arrived channel", msg.String()) - isSenderSync := utils.GetSwitchFromOptions(msg.MsgData.Options, constant.IsSenderSync) - err := saveUserChat(msgChannelValue.userID, &msg) + msgList := msgChannelValue.msgList + storageMsgList := make([]*pbMsg.MsgDataToMQ, 80) + pushMsgList := make([]*pbMsg.MsgDataToMQ, 80) + latestMsgOperationID := msgList[len(msgList)-1].OperationID + log.Debug(latestMsgOperationID, "msg arrived channel", "channel id", channelID, msgList) + for _, v := range msgList { + isHistory := utils.GetSwitchFromOptions(v.MsgData.Options, constant.IsHistory) + isSenderSync := utils.GetSwitchFromOptions(v.MsgData.Options, constant.IsSenderSync) + if isHistory { + storageMsgList = append(storageMsgList, v) + } + if !(!isSenderSync && msgChannelValue.userID == v.MsgData.SendID) { + pushMsgList = append(pushMsgList, v) + } + } + + //switch msgChannelValue.msg.MsgData.SessionType { + //case constant.SingleChatType: + //case constant.GroupChatType: + //case constant.NotificationChatType: + //default: + // log.NewError(msgFromMQ.OperationID, "SessionType error", msgFromMQ.String()) + // return + //} + + err := saveUserChatList(msgChannelValue.userID, storageMsgList, latestMsgOperationID) if err != nil { - singleMsgFailedCount++ - log.NewError(msg.OperationID, "single data insert to mongo err", err.Error(), msg.String()) + singleMsgFailedCount += uint64(len(storageMsgList)) + log.NewError(latestMsgOperationID, "single data insert to mongo err", err.Error(), storageMsgList) } else { singleMsgSuccessCountMutex.Lock() - singleMsgSuccessCount++ + singleMsgSuccessCount += uint64(len(storageMsgList)) singleMsgSuccessCountMutex.Unlock() - if !(!isSenderSync && msgChannelValue.userID == msg.MsgData.SendID) { - go sendMessageToPush(&msg, msgChannelValue.userID) + for _, v := range pushMsgList { + sendMessageToPush(v, msgChannelValue.userID) + } + + } + } + } + } +} +func (och *OfflineHistoryConsumerHandler) MessagesDistributionHandle() { + for { + select { + case cmd := <-och.msgDistributionCh: + switch cmd.Cmd { + case ConsumerMsgs: + consumerMessages := cmd.Value.([]*sarama.ConsumerMessage) + //Aggregation map[userid]message list + for i := 0; i < len(consumerMessages); i++ { + msgFromMQ := pbMsg.MsgDataToMQ{} + err := proto.Unmarshal(consumerMessages[i].Value, &msgFromMQ) + if err != nil { + log.Error("msg_transfer Unmarshal msg err", "", "msg", string(consumerMessages[i].Value), "err", err.Error()) + return + } + if oldM, ok := och.UserAggregationMsgs[string(consumerMessages[i].Key)]; ok { + oldM = append(oldM, &msgFromMQ) + och.UserAggregationMsgs[string(consumerMessages[i].Key)] = oldM + } else { + m := make([]*pbMsg.MsgDataToMQ, 100) + m = append(m, &msgFromMQ) + och.UserAggregationMsgs[string(consumerMessages[i].Key)] = m + } + } + for userID, v := range och.UserAggregationMsgs { + if len(v) >= 0 { + channelID := getHashCode(userID) % ChannelNum + go func(cID uint32, userID string, messages []*pbMsg.MsgDataToMQ) { + och.chArrays[cID] <- Cmd2Value{Cmd: UserMessages, Value: MsgChannelValue{userID: userID, msgList: messages}} + }(channelID, userID, v) } } } } } + } func (mc *OfflineHistoryConsumerHandler) handleChatWs2Mongo(cMsg *sarama.ConsumerMessage, msgKey string, sess sarama.ConsumerGroupSession) { msg := cMsg.Value @@ -157,7 +223,7 @@ func (mc *OfflineHistoryConsumerHandler) handleChatWs2MongoLowReliability(cMsg * sess.MarkMessage(cMsg, "") msgFromMQ.MsgData.Seq = uint32(seq) log.Debug(operationID, "send ch msg is ", msgFromMQ.String()) - mc.msgCh <- Cmd2Value{Cmd: Msg, Value: MsgChannelValue{msgKey, msgFromMQ}} + //mc.msgCh <- Cmd2Value{Cmd: Msg, Value: MsgChannelValue{msgKey, msgFromMQ}} //err := saveUserChat(msgKey, &msgFromMQ) //if err != nil { // singleMsgFailedCount++ @@ -177,27 +243,56 @@ func (mc *OfflineHistoryConsumerHandler) handleChatWs2MongoLowReliability(cMsg * func (OfflineHistoryConsumerHandler) Setup(_ sarama.ConsumerGroupSession) error { return nil } func (OfflineHistoryConsumerHandler) Cleanup(_ sarama.ConsumerGroupSession) error { return nil } -func (mc *OfflineHistoryConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, - claim sarama.ConsumerGroupClaim) error { // a instance in the consumer group - //log.NewDebug("", "offline new session msg come", claim.HighWaterMarkOffset(), claim.Topic(), claim.Partition()) - //for msg := range claim.Messages() { - // log.NewDebug("", "kafka get info to delay mongo", "msgTopic", msg.Topic, "msgPartition", msg.Partition, "offline") - // //mc.msgHandle[msg.Topic](msg.Value, string(msg.Key)) - //} - for msg := range claim.Messages() { - if GetOnlineTopicStatus() == OnlineTopicVacancy { - log.NewDebug("", "vacancy offline kafka get info to mongo", "msgTopic", msg.Topic, "msgPartition", msg.Partition, "msg", string(msg.Value)) - mc.msgHandle[msg.Topic](msg, string(msg.Key), sess) - } else { - select { - case <-mc.cmdCh: - log.NewDebug("", "cmd offline kafka get info to mongo", "msgTopic", msg.Topic, "msgPartition", msg.Partition, "msg", string(msg.Value)) - case <-time.After(time.Millisecond * time.Duration(100)): - log.NewDebug("", "timeout offline kafka get info to mongo", "msgTopic", msg.Topic, "msgPartition", msg.Partition, "msg", string(msg.Value)) - } - mc.msgHandle[msg.Topic](msg, string(msg.Key), sess) - } - } +//func (mc *OfflineHistoryConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, +// claim sarama.ConsumerGroupClaim) error { // a instance in the consumer group +// //log.NewDebug("", "offline new session msg come", claim.HighWaterMarkOffset(), claim.Topic(), claim.Partition()) +// //for msg := range claim.Messages() { +// // log.NewDebug("", "kafka get info to delay mongo", "msgTopic", msg.Topic, "msgPartition", msg.Partition, "offline") +// // //mc.msgHandle[msg.Topic](msg.Value, string(msg.Key)) +// //} +// for msg := range claim.Messages() { +// if GetOnlineTopicStatus() == OnlineTopicVacancy { +// log.NewDebug("", "vacancy offline kafka get info to mongo", "msgTopic", msg.Topic, "msgPartition", msg.Partition, "msg", string(msg.Value)) +// mc.msgHandle[msg.Topic](msg, string(msg.Key), sess) +// } else { +// select { +// case <-mc.cmdCh: +// log.NewDebug("", "cmd offline kafka get info to mongo", "msgTopic", msg.Topic, "msgPartition", msg.Partition, "msg", string(msg.Value)) +// case <-time.After(time.Millisecond * time.Duration(100)): +// log.NewDebug("", "timeout offline kafka get info to mongo", "msgTopic", msg.Topic, "msgPartition", msg.Partition, "msg", string(msg.Value)) +// } +// mc.msgHandle[msg.Topic](msg, string(msg.Key), sess) +// } +// } +// +// return nil +//} +func (och *OfflineHistoryConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, + claim sarama.ConsumerGroupClaim) error { // a instance in the consumer group + log.NewDebug("", "online new session msg come", claim.HighWaterMarkOffset(), claim.Topic(), claim.Partition()) + cMsg := make([]*sarama.ConsumerMessage, 500) + t := time.NewTicker(time.Duration(500) * time.Millisecond) + for msg := range claim.Messages() { + //och.TriggerCmd(OnlineTopicBusy) + cMsg = append(cMsg, msg) + select { + case <-t.C: + if len(cMsg) >= 0 { + och.msgDistributionCh <- Cmd2Value{Cmd: ConsumerMsgs, Value: cMsg} + sess.MarkMessage(msg, "") + cMsg = cMsg[0:0] + } + default: + if len(cMsg) >= 500 { + och.msgDistributionCh <- Cmd2Value{Cmd: ConsumerMsgs, Value: cMsg} + sess.MarkMessage(msg, "") + cMsg = cMsg[0:0] + } + + } + log.NewDebug("", "online kafka get info to mongo", "msgTopic", msg.Topic, "msgPartition", msg.Partition, "online", msg.Offset, claim.HighWaterMarkOffset()) + + } return nil } diff --git a/internal/msg_transfer/logic/online_history_msg_handler.go b/internal/msg_transfer/logic/online_history_msg_handler.go index ea787c107..248e55bd6 100644 --- a/internal/msg_transfer/logic/online_history_msg_handler.go +++ b/internal/msg_transfer/logic/online_history_msg_handler.go @@ -14,13 +14,14 @@ import ( "errors" "github.com/Shopify/sarama" "github.com/golang/protobuf/proto" + "hash/crc32" "strings" "time" ) type MsgChannelValue struct { - userID string - msg pbMsg.MsgDataToMQ + userID string + msgList []*pbMsg.MsgDataToMQ } type fcb func(cMsg *sarama.ConsumerMessage, msgKey string, sess sarama.ConsumerGroupSession) type Cmd2Value struct { @@ -32,19 +33,25 @@ type OnlineHistoryConsumerHandler struct { historyConsumerGroup *kfk.MConsumerGroup cmdCh chan Cmd2Value msgCh chan Cmd2Value + UserAggregationMsgs map[string][]*pbMsg.MsgDataToMQ + chArrays [ChannelNum]chan Cmd2Value + msgDistributionCh chan Cmd2Value } func (och *OnlineHistoryConsumerHandler) Init(cmdCh chan Cmd2Value) { och.msgHandle = make(map[string]fcb) + och.UserAggregationMsgs = make(map[string][]*pbMsg.MsgDataToMQ, ChannelNum) + och.msgDistributionCh = make(chan Cmd2Value) //no buffer channel + go och.MessagesDistributionHandle() och.cmdCh = cmdCh och.msgCh = make(chan Cmd2Value, 1000) if config.Config.ReliableStorage { och.msgHandle[config.Config.Kafka.Ws2mschat.Topic] = och.handleChatWs2Mongo } else { och.msgHandle[config.Config.Kafka.Ws2mschat.Topic] = och.handleChatWs2MongoLowReliability - for i := 0; i < 10; i++ { - go och.Run() - + for i := 0; i < ChannelNum; i++ { + och.chArrays[i] = make(chan Cmd2Value, 1000) + go och.Run(i) } } och.historyConsumerGroup = kfk.NewMConsumerGroup(&kfk.MConsumerGroupConfig{KafkaVersion: sarama.V0_10_2_0, @@ -76,16 +83,29 @@ func sendCmd(ch chan Cmd2Value, value Cmd2Value, timeout int64) error { return errors.New("send cmd timeout") } } -func (och *OnlineHistoryConsumerHandler) Run() { +func (och *OnlineHistoryConsumerHandler) Run(channelID int) { for { select { - case cmd := <-och.msgCh: + case cmd := <-och.chArrays[channelID]: switch cmd.Cmd { - case Msg: + case UserMessages: msgChannelValue := cmd.Value.(MsgChannelValue) - msg := msgChannelValue.msg - log.Debug(msg.OperationID, "msg arrived channel", msg.String()) - isSenderSync := utils.GetSwitchFromOptions(msg.MsgData.Options, constant.IsSenderSync) + msgList := msgChannelValue.msgList + storageMsgList := make([]*pbMsg.MsgDataToMQ, 80) + pushMsgList := make([]*pbMsg.MsgDataToMQ, 80) + latestMsgOperationID := msgList[len(msgList)-1].OperationID + log.Debug(latestMsgOperationID, "msg arrived channel", "channel id", channelID, msgList) + for _, v := range msgList { + isHistory := utils.GetSwitchFromOptions(v.MsgData.Options, constant.IsHistory) + isSenderSync := utils.GetSwitchFromOptions(v.MsgData.Options, constant.IsSenderSync) + if isHistory { + storageMsgList = append(storageMsgList, v) + } + if !(!isSenderSync && msgChannelValue.userID == v.MsgData.SendID) { + pushMsgList = append(pushMsgList, v) + } + } + //switch msgChannelValue.msg.MsgData.SessionType { //case constant.SingleChatType: //case constant.GroupChatType: @@ -95,23 +115,132 @@ func (och *OnlineHistoryConsumerHandler) Run() { // return //} - err := saveUserChat(msgChannelValue.userID, &msg) + err := saveUserChatList(msgChannelValue.userID, storageMsgList, latestMsgOperationID) if err != nil { - singleMsgFailedCount++ - log.NewError(msg.OperationID, "single data insert to mongo err", err.Error(), msg.String()) + singleMsgFailedCount += uint64(len(storageMsgList)) + log.NewError(latestMsgOperationID, "single data insert to mongo err", err.Error(), storageMsgList) } else { singleMsgSuccessCountMutex.Lock() - singleMsgSuccessCount++ + singleMsgSuccessCount += uint64(len(storageMsgList)) singleMsgSuccessCountMutex.Unlock() - if !(!isSenderSync && msgChannelValue.userID == msg.MsgData.SendID) { - go sendMessageToPush(&msg, msgChannelValue.userID) + for _, v := range pushMsgList { + sendMessageToPush(v, msgChannelValue.userID) } + } } } } } -func (och *OnlineHistoryConsumerHandler) handleChatWs2Mongo(cMsg *sarama.ConsumerMessage, msgKey string, sess sarama.ConsumerGroupSession) { + +//func (och *OnlineHistoryConsumerHandler) handleChatWs2Mongo(cMsg *sarama.ConsumerMessage, msgKey string, sess sarama.ConsumerGroupSession) { +// msg := cMsg.Value +// now := time.Now() +// msgFromMQ := pbMsg.MsgDataToMQ{} +// err := proto.Unmarshal(msg, &msgFromMQ) +// if err != nil { +// log.Error("msg_transfer Unmarshal msg err", "", "msg", string(msg), "err", err.Error()) +// return +// } +// operationID := msgFromMQ.OperationID +// log.NewInfo(operationID, "msg come mongo!!!", "", "msg", string(msg)) +// //Control whether to store offline messages (mongo) +// isHistory := utils.GetSwitchFromOptions(msgFromMQ.MsgData.Options, constant.IsHistory) +// //Control whether to store history messages (mysql) +// isPersist := utils.GetSwitchFromOptions(msgFromMQ.MsgData.Options, constant.IsPersistent) +// isSenderSync := utils.GetSwitchFromOptions(msgFromMQ.MsgData.Options, constant.IsSenderSync) +// switch msgFromMQ.MsgData.SessionType { +// case constant.SingleChatType: +// log.NewDebug(msgFromMQ.OperationID, "msg_transfer msg type = SingleChatType", isHistory, isPersist) +// if isHistory { +// err := saveUserChat(msgKey, &msgFromMQ) +// if err != nil { +// singleMsgFailedCount++ +// log.NewError(operationID, "single data insert to mongo err", err.Error(), msgFromMQ.String()) +// return +// } +// singleMsgSuccessCountMutex.Lock() +// singleMsgSuccessCount++ +// singleMsgSuccessCountMutex.Unlock() +// log.NewDebug(msgFromMQ.OperationID, "sendMessageToPush cost time ", time.Since(now)) +// } +// if !isSenderSync && msgKey == msgFromMQ.MsgData.SendID { +// } else { +// go sendMessageToPush(&msgFromMQ, msgKey) +// } +// log.NewDebug(operationID, "saveUserChat cost time ", time.Since(now)) +// case constant.GroupChatType: +// log.NewDebug(msgFromMQ.OperationID, "msg_transfer msg type = GroupChatType", isHistory, isPersist) +// if isHistory { +// err := saveUserChat(msgFromMQ.MsgData.RecvID, &msgFromMQ) +// if err != nil { +// log.NewError(operationID, "group data insert to mongo err", msgFromMQ.String(), msgFromMQ.MsgData.RecvID, err.Error()) +// return +// } +// groupMsgCount++ +// } +// go sendMessageToPush(&msgFromMQ, msgFromMQ.MsgData.RecvID) +// case constant.NotificationChatType: +// log.NewDebug(msgFromMQ.OperationID, "msg_transfer msg type = NotificationChatType", isHistory, isPersist) +// if isHistory { +// err := saveUserChat(msgKey, &msgFromMQ) +// if err != nil { +// log.NewError(operationID, "single data insert to mongo err", err.Error(), msgFromMQ.String()) +// return +// } +// log.NewDebug(msgFromMQ.OperationID, "sendMessageToPush cost time ", time.Since(now)) +// } +// if !isSenderSync && msgKey == msgFromMQ.MsgData.SendID { +// } else { +// go sendMessageToPush(&msgFromMQ, msgKey) +// } +// log.NewDebug(operationID, "saveUserChat cost time ", time.Since(now)) +// default: +// log.NewError(msgFromMQ.OperationID, "SessionType error", msgFromMQ.String()) +// return +// } +// sess.MarkMessage(cMsg, "") +// log.NewDebug(msgFromMQ.OperationID, "msg_transfer handle topic data to database success...", msgFromMQ.String()) +//} + +func (och *OnlineHistoryConsumerHandler) MessagesDistributionHandle() { + for { + select { + case cmd := <-och.msgDistributionCh: + switch cmd.Cmd { + case ConsumerMsgs: + consumerMessages := cmd.Value.([]*sarama.ConsumerMessage) + //Aggregation map[userid]message list + for i := 0; i < len(consumerMessages); i++ { + msgFromMQ := pbMsg.MsgDataToMQ{} + err := proto.Unmarshal(consumerMessages[i].Value, &msgFromMQ) + if err != nil { + log.Error("msg_transfer Unmarshal msg err", "", "msg", string(consumerMessages[i].Value), "err", err.Error()) + return + } + if oldM, ok := och.UserAggregationMsgs[string(consumerMessages[i].Key)]; ok { + oldM = append(oldM, &msgFromMQ) + och.UserAggregationMsgs[string(consumerMessages[i].Key)] = oldM + } else { + m := make([]*pbMsg.MsgDataToMQ, 100) + m = append(m, &msgFromMQ) + och.UserAggregationMsgs[string(consumerMessages[i].Key)] = m + } + } + for userID, v := range och.UserAggregationMsgs { + if len(v) >= 0 { + channelID := getHashCode(userID) % ChannelNum + go func(cID uint32, userID string, messages []*pbMsg.MsgDataToMQ) { + och.chArrays[cID] <- Cmd2Value{Cmd: UserMessages, Value: MsgChannelValue{userID: userID, msgList: messages}} + }(channelID, userID, v) + } + } + } + } + } + +} +func (mc *OnlineHistoryConsumerHandler) handleChatWs2Mongo(cMsg *sarama.ConsumerMessage, msgKey string, sess sarama.ConsumerGroupSession) { msg := cMsg.Value now := time.Now() msgFromMQ := pbMsg.MsgDataToMQ{} @@ -146,7 +275,7 @@ func (och *OnlineHistoryConsumerHandler) handleChatWs2Mongo(cMsg *sarama.Consume } else { go sendMessageToPush(&msgFromMQ, msgKey) } - log.NewDebug(operationID, "saveUserChat cost time ", time.Since(now)) + log.NewDebug(operationID, "saveSingleMsg cost time ", time.Since(now)) case constant.GroupChatType: log.NewDebug(msgFromMQ.OperationID, "msg_transfer msg type = GroupChatType", isHistory, isPersist) if isHistory { @@ -158,6 +287,8 @@ func (och *OnlineHistoryConsumerHandler) handleChatWs2Mongo(cMsg *sarama.Consume groupMsgCount++ } go sendMessageToPush(&msgFromMQ, msgFromMQ.MsgData.RecvID) + log.NewDebug(operationID, "saveGroupMsg cost time ", time.Since(now)) + case constant.NotificationChatType: log.NewDebug(msgFromMQ.OperationID, "msg_transfer msg type = NotificationChatType", isHistory, isPersist) if isHistory { @@ -180,6 +311,7 @@ func (och *OnlineHistoryConsumerHandler) handleChatWs2Mongo(cMsg *sarama.Consume sess.MarkMessage(cMsg, "") log.NewDebug(msgFromMQ.OperationID, "msg_transfer handle topic data to database success...", msgFromMQ.String()) } + func (och *OnlineHistoryConsumerHandler) handleChatWs2MongoLowReliability(cMsg *sarama.ConsumerMessage, msgKey string, sess sarama.ConsumerGroupSession) { msg := cMsg.Value msgFromMQ := pbMsg.MsgDataToMQ{} @@ -202,7 +334,7 @@ func (och *OnlineHistoryConsumerHandler) handleChatWs2MongoLowReliability(cMsg * sess.MarkMessage(cMsg, "") msgFromMQ.MsgData.Seq = uint32(seq) log.Debug(operationID, "send ch msg is ", msgFromMQ.String()) - och.msgCh <- Cmd2Value{Cmd: Msg, Value: MsgChannelValue{msgKey, msgFromMQ}} + //och.msgCh <- Cmd2Value{Cmd: Msg, Value: MsgChannelValue{msgKey, msgFromMQ}} //err := saveUserChat(msgKey, &msgFromMQ) //if err != nil { // singleMsgFailedCount++ @@ -222,19 +354,49 @@ func (och *OnlineHistoryConsumerHandler) handleChatWs2MongoLowReliability(cMsg * func (OnlineHistoryConsumerHandler) Setup(_ sarama.ConsumerGroupSession) error { return nil } func (OnlineHistoryConsumerHandler) Cleanup(_ sarama.ConsumerGroupSession) error { return nil } + +//func (och *OnlineHistoryConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, +// claim sarama.ConsumerGroupClaim) error { // a instance in the consumer group +// log.NewDebug("", "online new session msg come", claim.HighWaterMarkOffset(), claim.Topic(), claim.Partition()) +// for msg := range claim.Messages() { +// SetOnlineTopicStatus(OnlineTopicBusy) +// //och.TriggerCmd(OnlineTopicBusy) +// log.NewDebug("", "online kafka get info to mongo", "msgTopic", msg.Topic, "msgPartition", msg.Partition, "online", msg.Offset, claim.HighWaterMarkOffset()) +// och.msgHandle[msg.Topic](msg, string(msg.Key), sess) +// if claim.HighWaterMarkOffset()-msg.Offset <= 1 { +// log.Debug("", "online msg consume end", claim.HighWaterMarkOffset(), msg.Offset) +// SetOnlineTopicStatus(OnlineTopicVacancy) +// och.TriggerCmd(OnlineTopicVacancy) +// } +// } +// return nil +//} + func (och *OnlineHistoryConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { // a instance in the consumer group log.NewDebug("", "online new session msg come", claim.HighWaterMarkOffset(), claim.Topic(), claim.Partition()) + cMsg := make([]*sarama.ConsumerMessage, 500) + t := time.NewTicker(time.Duration(500) * time.Millisecond) for msg := range claim.Messages() { - SetOnlineTopicStatus(OnlineTopicBusy) //och.TriggerCmd(OnlineTopicBusy) - log.NewDebug("", "online kafka get info to mongo", "msgTopic", msg.Topic, "msgPartition", msg.Partition, "online", msg.Offset, claim.HighWaterMarkOffset()) - och.msgHandle[msg.Topic](msg, string(msg.Key), sess) - if claim.HighWaterMarkOffset()-msg.Offset <= 1 { - log.Debug("", "online msg consume end", claim.HighWaterMarkOffset(), msg.Offset) - SetOnlineTopicStatus(OnlineTopicVacancy) - och.TriggerCmd(OnlineTopicVacancy) + cMsg = append(cMsg, msg) + select { + case <-t.C: + if len(cMsg) >= 0 { + och.msgDistributionCh <- Cmd2Value{Cmd: ConsumerMsgs, Value: cMsg} + sess.MarkMessage(msg, "") + cMsg = cMsg[0:0] + } + default: + if len(cMsg) >= 500 { + och.msgDistributionCh <- Cmd2Value{Cmd: ConsumerMsgs, Value: cMsg} + sess.MarkMessage(msg, "") + cMsg = cMsg[0:0] + } + } + log.NewDebug("", "online kafka get info to mongo", "msgTopic", msg.Topic, "msgPartition", msg.Partition, "online", msg.Offset, claim.HighWaterMarkOffset()) + } return nil } @@ -264,3 +426,12 @@ func sendMessageToPush(message *pbMsg.MsgDataToMQ, pushToUserID string) { } } + +// String hashes a string to a unique hashcode. +// +// crc32 returns a uint32, but for our use we need +// and non negative integer. Here we cast to an integer +// and invert it if the result is negative. +func getHashCode(s string) uint32 { + return crc32.ChecksumIEEE([]byte(s)) +} From f0c6295cff66d4b5f2122d2962f08708f3743e7d Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Fri, 20 May 2022 13:34:34 +0800 Subject: [PATCH 19/76] batch to mongo --- pkg/common/db/batch_insert_chat.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pkg/common/db/batch_insert_chat.go b/pkg/common/db/batch_insert_chat.go index e327ba7b5..b3211e701 100644 --- a/pkg/common/db/batch_insert_chat.go +++ b/pkg/common/db/batch_insert_chat.go @@ -18,16 +18,17 @@ func (d *DataBases) BatchInsertChat(userID string, msgList []*pbMsg.MsgDataToMQ, if len(msgList) > GetSingleGocMsgNum() { return errors.New("too large") } + isInit := false currentMaxSeq, err := d.GetUserMaxSeq(userID) if err == nil { } else if err == redis.ErrNil { + isInit = true currentMaxSeq = 0 } else { return utils.Wrap(err, "") } - //4999 remain := uint64(GetSingleGocMsgNum()) - (currentMaxSeq % uint64(GetSingleGocMsgNum())) insertCounter := uint64(0) msgListToMongo := make([]MsgInfo, 0) @@ -43,6 +44,13 @@ func (d *DataBases) BatchInsertChat(userID string, msgList []*pbMsg.MsgDataToMQ, if sMsg.Msg, err = proto.Marshal(m.MsgData); err != nil { return utils.Wrap(err, "") } + if isInit { + msgListToMongoNext = append(msgListToMongoNext, sMsg) + seqUidNext = getSeqUid(userID, uint32(currentMaxSeq)) + log.Debug(operationID, "msgListToMongoNext ", seqUidNext, m.MsgData.Seq, m.MsgData.ClientMsgID, insertCounter, remain) + continue + } + if insertCounter < remain { msgListToMongo = append(msgListToMongo, sMsg) insertCounter++ From 6f79e63b3fc98e59d4bf80630297448b4f1f5f95 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Fri, 20 May 2022 14:06:31 +0800 Subject: [PATCH 20/76] log --- internal/msg_transfer/logic/online_history_msg_handler.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/msg_transfer/logic/online_history_msg_handler.go b/internal/msg_transfer/logic/online_history_msg_handler.go index 248e55bd6..6bd344737 100644 --- a/internal/msg_transfer/logic/online_history_msg_handler.go +++ b/internal/msg_transfer/logic/online_history_msg_handler.go @@ -378,18 +378,21 @@ func (och *OnlineHistoryConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupS cMsg := make([]*sarama.ConsumerMessage, 500) t := time.NewTicker(time.Duration(500) * time.Millisecond) for msg := range claim.Messages() { + operationID := utils.OperationIDGenerator() //och.TriggerCmd(OnlineTopicBusy) cMsg = append(cMsg, msg) select { case <-t.C: if len(cMsg) >= 0 { och.msgDistributionCh <- Cmd2Value{Cmd: ConsumerMsgs, Value: cMsg} + log.Debug(operationID, "timer send to msgDistributionCh", och.msgDistributionCh, "len: ", len(cMsg)) sess.MarkMessage(msg, "") cMsg = cMsg[0:0] } default: if len(cMsg) >= 500 { och.msgDistributionCh <- Cmd2Value{Cmd: ConsumerMsgs, Value: cMsg} + log.Debug(operationID, "500 send to msgDistributionCh", och.msgDistributionCh, "len: ", len(cMsg)) sess.MarkMessage(msg, "") cMsg = cMsg[0:0] } From 3bf04802e4b1ae6e259e88edbbfc756efc141b68 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Fri, 20 May 2022 14:06:51 +0800 Subject: [PATCH 21/76] log --- internal/msg_transfer/logic/online_history_msg_handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/msg_transfer/logic/online_history_msg_handler.go b/internal/msg_transfer/logic/online_history_msg_handler.go index 6bd344737..5e1e5f935 100644 --- a/internal/msg_transfer/logic/online_history_msg_handler.go +++ b/internal/msg_transfer/logic/online_history_msg_handler.go @@ -376,7 +376,7 @@ func (och *OnlineHistoryConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupS claim sarama.ConsumerGroupClaim) error { // a instance in the consumer group log.NewDebug("", "online new session msg come", claim.HighWaterMarkOffset(), claim.Topic(), claim.Partition()) cMsg := make([]*sarama.ConsumerMessage, 500) - t := time.NewTicker(time.Duration(500) * time.Millisecond) + t := time.NewTicker(time.Duration(10) * time.Millisecond) for msg := range claim.Messages() { operationID := utils.OperationIDGenerator() //och.TriggerCmd(OnlineTopicBusy) From 06aae24dada45bb0dfefeab884cdd81ce7f3640e Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Fri, 20 May 2022 14:12:29 +0800 Subject: [PATCH 22/76] log --- internal/msg_transfer/logic/online_history_msg_handler.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/internal/msg_transfer/logic/online_history_msg_handler.go b/internal/msg_transfer/logic/online_history_msg_handler.go index 5e1e5f935..b6e717af6 100644 --- a/internal/msg_transfer/logic/online_history_msg_handler.go +++ b/internal/msg_transfer/logic/online_history_msg_handler.go @@ -205,19 +205,23 @@ func (och *OnlineHistoryConsumerHandler) Run(channelID int) { func (och *OnlineHistoryConsumerHandler) MessagesDistributionHandle() { for { + operationID := utils.OperationIDGenerator() select { + case cmd := <-och.msgDistributionCh: switch cmd.Cmd { case ConsumerMsgs: consumerMessages := cmd.Value.([]*sarama.ConsumerMessage) //Aggregation map[userid]message list for i := 0; i < len(consumerMessages); i++ { + msgFromMQ := pbMsg.MsgDataToMQ{} err := proto.Unmarshal(consumerMessages[i].Value, &msgFromMQ) if err != nil { - log.Error("msg_transfer Unmarshal msg err", "", "msg", string(consumerMessages[i].Value), "err", err.Error()) + log.Error(operationID, "msg_transfer Unmarshal msg err", "", "msg", string(consumerMessages[i].Value), "err", err.Error()) return } + log.Debug(operationID, "MessagesDistributionHandle ", msgFromMQ.String()) if oldM, ok := och.UserAggregationMsgs[string(consumerMessages[i].Key)]; ok { oldM = append(oldM, &msgFromMQ) och.UserAggregationMsgs[string(consumerMessages[i].Key)] = oldM @@ -230,6 +234,7 @@ func (och *OnlineHistoryConsumerHandler) MessagesDistributionHandle() { for userID, v := range och.UserAggregationMsgs { if len(v) >= 0 { channelID := getHashCode(userID) % ChannelNum + log.Debug(operationID, "UserAggregationMsgs ", len(v), channelID, userID) go func(cID uint32, userID string, messages []*pbMsg.MsgDataToMQ) { och.chArrays[cID] <- Cmd2Value{Cmd: UserMessages, Value: MsgChannelValue{userID: userID, msgList: messages}} }(channelID, userID, v) From b632a991b9b8a588cef2bd4de4a05425c05ede21 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Fri, 20 May 2022 14:18:54 +0800 Subject: [PATCH 23/76] log --- internal/msg_transfer/logic/online_history_msg_handler.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/msg_transfer/logic/online_history_msg_handler.go b/internal/msg_transfer/logic/online_history_msg_handler.go index b6e717af6..b13f02631 100644 --- a/internal/msg_transfer/logic/online_history_msg_handler.go +++ b/internal/msg_transfer/logic/online_history_msg_handler.go @@ -238,6 +238,8 @@ func (och *OnlineHistoryConsumerHandler) MessagesDistributionHandle() { go func(cID uint32, userID string, messages []*pbMsg.MsgDataToMQ) { och.chArrays[cID] <- Cmd2Value{Cmd: UserMessages, Value: MsgChannelValue{userID: userID, msgList: messages}} }(channelID, userID, v) + } else { + log.NewWarn(operationID, "UserAggregationMsgs ", len(v), userID) } } } From a14fd303fcdbd904b8459f22527e8532aae6c488 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Fri, 20 May 2022 14:32:09 +0800 Subject: [PATCH 24/76] log --- internal/msg_transfer/logic/online_history_msg_handler.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/msg_transfer/logic/online_history_msg_handler.go b/internal/msg_transfer/logic/online_history_msg_handler.go index b13f02631..7105c08c2 100644 --- a/internal/msg_transfer/logic/online_history_msg_handler.go +++ b/internal/msg_transfer/logic/online_history_msg_handler.go @@ -213,6 +213,7 @@ func (och *OnlineHistoryConsumerHandler) MessagesDistributionHandle() { case ConsumerMsgs: consumerMessages := cmd.Value.([]*sarama.ConsumerMessage) //Aggregation map[userid]message list + log.Debug(operationID, "consumerMessages len ", len(consumerMessages)) for i := 0; i < len(consumerMessages); i++ { msgFromMQ := pbMsg.MsgDataToMQ{} From a7be7a3e7a6938ea29064370db7baa2d2e3c93c1 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Fri, 20 May 2022 14:38:10 +0800 Subject: [PATCH 25/76] log --- internal/msg_transfer/logic/online_history_msg_handler.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/msg_transfer/logic/online_history_msg_handler.go b/internal/msg_transfer/logic/online_history_msg_handler.go index 7105c08c2..2a84e5a63 100644 --- a/internal/msg_transfer/logic/online_history_msg_handler.go +++ b/internal/msg_transfer/logic/online_history_msg_handler.go @@ -226,12 +226,15 @@ func (och *OnlineHistoryConsumerHandler) MessagesDistributionHandle() { if oldM, ok := och.UserAggregationMsgs[string(consumerMessages[i].Key)]; ok { oldM = append(oldM, &msgFromMQ) och.UserAggregationMsgs[string(consumerMessages[i].Key)] = oldM + log.Debug(operationID, "consumerMessages key ", string(consumerMessages[i].Key), oldM) } else { m := make([]*pbMsg.MsgDataToMQ, 100) m = append(m, &msgFromMQ) och.UserAggregationMsgs[string(consumerMessages[i].Key)] = m + log.Debug(operationID, "consumerMessages key ", string(consumerMessages[i].Key), m) } } + log.Debug(operationID, "UserAggregationMsgs len ", len(och.UserAggregationMsgs)) for userID, v := range och.UserAggregationMsgs { if len(v) >= 0 { channelID := getHashCode(userID) % ChannelNum From 7ea1309146c8338d3f10b1fbe28a96c6083e4d4a Mon Sep 17 00:00:00 2001 From: Gordon <1432970085@qq.com> Date: Fri, 20 May 2022 14:42:49 +0800 Subject: [PATCH 26/76] concurrent consumption of messages --- .../logic/offline_history_msg_handler.go | 48 ++++++++++------ .../logic/online_history_msg_handler.go | 57 ++++++++++++------- 2 files changed, 69 insertions(+), 36 deletions(-) diff --git a/internal/msg_transfer/logic/offline_history_msg_handler.go b/internal/msg_transfer/logic/offline_history_msg_handler.go index 4fdd30577..2f63f2007 100644 --- a/internal/msg_transfer/logic/offline_history_msg_handler.go +++ b/internal/msg_transfer/logic/offline_history_msg_handler.go @@ -18,14 +18,12 @@ type OfflineHistoryConsumerHandler struct { historyConsumerGroup *kfk.MConsumerGroup cmdCh chan Cmd2Value msgCh chan Cmd2Value - UserAggregationMsgs map[string][]*pbMsg.MsgDataToMQ chArrays [ChannelNum]chan Cmd2Value msgDistributionCh chan Cmd2Value } func (mc *OfflineHistoryConsumerHandler) Init(cmdCh chan Cmd2Value) { mc.msgHandle = make(map[string]fcb) - mc.UserAggregationMsgs = make(map[string][]*pbMsg.MsgDataToMQ, ChannelNum) mc.msgDistributionCh = make(chan Cmd2Value) //no buffer channel go mc.MessagesDistributionHandle() mc.cmdCh = cmdCh @@ -52,11 +50,12 @@ func (och *OfflineHistoryConsumerHandler) Run(channelID int) { case UserMessages: msgChannelValue := cmd.Value.(MsgChannelValue) msgList := msgChannelValue.msgList + triggerID := msgChannelValue.triggerID storageMsgList := make([]*pbMsg.MsgDataToMQ, 80) pushMsgList := make([]*pbMsg.MsgDataToMQ, 80) - latestMsgOperationID := msgList[len(msgList)-1].OperationID - log.Debug(latestMsgOperationID, "msg arrived channel", "channel id", channelID, msgList) + log.Debug(triggerID, "msg arrived channel", "channel id", channelID, msgList, msgChannelValue.userID, len(msgList)) for _, v := range msgList { + log.Debug(triggerID, "msg come to storage center", v.String()) isHistory := utils.GetSwitchFromOptions(v.MsgData.Options, constant.IsHistory) isSenderSync := utils.GetSwitchFromOptions(v.MsgData.Options, constant.IsSenderSync) if isHistory { @@ -76,10 +75,10 @@ func (och *OfflineHistoryConsumerHandler) Run(channelID int) { // return //} - err := saveUserChatList(msgChannelValue.userID, storageMsgList, latestMsgOperationID) + err := saveUserChatList(msgChannelValue.userID, storageMsgList, triggerID) if err != nil { singleMsgFailedCount += uint64(len(storageMsgList)) - log.NewError(latestMsgOperationID, "single data insert to mongo err", err.Error(), storageMsgList) + log.NewError(triggerID, "single data insert to mongo err", err.Error(), storageMsgList) } else { singleMsgSuccessCountMutex.Lock() singleMsgSuccessCount += uint64(len(storageMsgList)) @@ -94,34 +93,40 @@ func (och *OfflineHistoryConsumerHandler) Run(channelID int) { } } func (och *OfflineHistoryConsumerHandler) MessagesDistributionHandle() { + UserAggregationMsgs := make(map[string][]*pbMsg.MsgDataToMQ, ChannelNum) for { select { case cmd := <-och.msgDistributionCh: switch cmd.Cmd { case ConsumerMsgs: - consumerMessages := cmd.Value.([]*sarama.ConsumerMessage) + triggerChannelValue := cmd.Value.(TriggerChannelValue) + triggerID := triggerChannelValue.triggerID + consumerMessages := triggerChannelValue.cmsgList //Aggregation map[userid]message list + log.Debug(triggerID, "batch messages come to distribution center", len(consumerMessages)) for i := 0; i < len(consumerMessages); i++ { msgFromMQ := pbMsg.MsgDataToMQ{} err := proto.Unmarshal(consumerMessages[i].Value, &msgFromMQ) if err != nil { - log.Error("msg_transfer Unmarshal msg err", "", "msg", string(consumerMessages[i].Value), "err", err.Error()) + log.Error(triggerID, "msg_transfer Unmarshal msg err", "msg", string(consumerMessages[i].Value), "err", err.Error()) return } - if oldM, ok := och.UserAggregationMsgs[string(consumerMessages[i].Key)]; ok { + log.Debug(triggerID, "single msg come to distribution center", msgFromMQ.String()) + if oldM, ok := UserAggregationMsgs[string(consumerMessages[i].Key)]; ok { oldM = append(oldM, &msgFromMQ) - och.UserAggregationMsgs[string(consumerMessages[i].Key)] = oldM + UserAggregationMsgs[string(consumerMessages[i].Key)] = oldM } else { - m := make([]*pbMsg.MsgDataToMQ, 100) + m := make([]*pbMsg.MsgDataToMQ, 0, 100) m = append(m, &msgFromMQ) - och.UserAggregationMsgs[string(consumerMessages[i].Key)] = m + UserAggregationMsgs[string(consumerMessages[i].Key)] = m } } - for userID, v := range och.UserAggregationMsgs { + log.Debug(triggerID, "generate map list users len", len(UserAggregationMsgs)) + for userID, v := range UserAggregationMsgs { if len(v) >= 0 { channelID := getHashCode(userID) % ChannelNum go func(cID uint32, userID string, messages []*pbMsg.MsgDataToMQ) { - och.chArrays[cID] <- Cmd2Value{Cmd: UserMessages, Value: MsgChannelValue{userID: userID, msgList: messages}} + och.chArrays[cID] <- Cmd2Value{Cmd: UserMessages, Value: MsgChannelValue{userID: userID, msgList: messages, triggerID: triggerID}} }(channelID, userID, v) } } @@ -271,23 +276,32 @@ func (OfflineHistoryConsumerHandler) Cleanup(_ sarama.ConsumerGroupSession) erro func (och *OfflineHistoryConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { // a instance in the consumer group log.NewDebug("", "online new session msg come", claim.HighWaterMarkOffset(), claim.Topic(), claim.Partition()) - cMsg := make([]*sarama.ConsumerMessage, 500) + cMsg := make([]*sarama.ConsumerMessage, 0, 500) t := time.NewTicker(time.Duration(500) * time.Millisecond) + var triggerID string for msg := range claim.Messages() { //och.TriggerCmd(OnlineTopicBusy) cMsg = append(cMsg, msg) select { case <-t.C: if len(cMsg) >= 0 { - och.msgDistributionCh <- Cmd2Value{Cmd: ConsumerMsgs, Value: cMsg} + triggerID = utils.OperationIDGenerator() + log.Debug(triggerID, "timer trigger msg consumer start", len(cMsg)) + och.msgDistributionCh <- Cmd2Value{Cmd: ConsumerMsgs, Value: TriggerChannelValue{ + triggerID: triggerID, cmsgList: cMsg}} sess.MarkMessage(msg, "") cMsg = cMsg[0:0] + log.Debug(triggerID, "timer trigger msg consumer end", len(cMsg)) } default: if len(cMsg) >= 500 { - och.msgDistributionCh <- Cmd2Value{Cmd: ConsumerMsgs, Value: cMsg} + triggerID = utils.OperationIDGenerator() + log.Debug(triggerID, "length trigger msg consumer start", len(cMsg)) + och.msgDistributionCh <- Cmd2Value{Cmd: ConsumerMsgs, Value: TriggerChannelValue{ + triggerID: triggerID, cmsgList: cMsg}} sess.MarkMessage(msg, "") cMsg = cMsg[0:0] + log.Debug(triggerID, "length trigger msg consumer end", len(cMsg)) } } diff --git a/internal/msg_transfer/logic/online_history_msg_handler.go b/internal/msg_transfer/logic/online_history_msg_handler.go index 248e55bd6..b33014bc4 100644 --- a/internal/msg_transfer/logic/online_history_msg_handler.go +++ b/internal/msg_transfer/logic/online_history_msg_handler.go @@ -20,8 +20,13 @@ import ( ) type MsgChannelValue struct { - userID string - msgList []*pbMsg.MsgDataToMQ + userID string + triggerID string + msgList []*pbMsg.MsgDataToMQ +} +type TriggerChannelValue struct { + triggerID string + cmsgList []*sarama.ConsumerMessage } type fcb func(cMsg *sarama.ConsumerMessage, msgKey string, sess sarama.ConsumerGroupSession) type Cmd2Value struct { @@ -33,14 +38,12 @@ type OnlineHistoryConsumerHandler struct { historyConsumerGroup *kfk.MConsumerGroup cmdCh chan Cmd2Value msgCh chan Cmd2Value - UserAggregationMsgs map[string][]*pbMsg.MsgDataToMQ chArrays [ChannelNum]chan Cmd2Value msgDistributionCh chan Cmd2Value } func (och *OnlineHistoryConsumerHandler) Init(cmdCh chan Cmd2Value) { och.msgHandle = make(map[string]fcb) - och.UserAggregationMsgs = make(map[string][]*pbMsg.MsgDataToMQ, ChannelNum) och.msgDistributionCh = make(chan Cmd2Value) //no buffer channel go och.MessagesDistributionHandle() och.cmdCh = cmdCh @@ -91,11 +94,12 @@ func (och *OnlineHistoryConsumerHandler) Run(channelID int) { case UserMessages: msgChannelValue := cmd.Value.(MsgChannelValue) msgList := msgChannelValue.msgList + triggerID := msgChannelValue.triggerID storageMsgList := make([]*pbMsg.MsgDataToMQ, 80) pushMsgList := make([]*pbMsg.MsgDataToMQ, 80) - latestMsgOperationID := msgList[len(msgList)-1].OperationID - log.Debug(latestMsgOperationID, "msg arrived channel", "channel id", channelID, msgList) + log.Debug(triggerID, "msg arrived channel", "channel id", channelID, msgList, msgChannelValue.userID, len(msgList)) for _, v := range msgList { + log.Debug(triggerID, "msg come to storage center", v.String()) isHistory := utils.GetSwitchFromOptions(v.MsgData.Options, constant.IsHistory) isSenderSync := utils.GetSwitchFromOptions(v.MsgData.Options, constant.IsSenderSync) if isHistory { @@ -115,10 +119,10 @@ func (och *OnlineHistoryConsumerHandler) Run(channelID int) { // return //} - err := saveUserChatList(msgChannelValue.userID, storageMsgList, latestMsgOperationID) + err := saveUserChatList(msgChannelValue.userID, storageMsgList, triggerID) if err != nil { singleMsgFailedCount += uint64(len(storageMsgList)) - log.NewError(latestMsgOperationID, "single data insert to mongo err", err.Error(), storageMsgList) + log.NewError(triggerID, "single data insert to mongo err", err.Error(), storageMsgList) } else { singleMsgSuccessCountMutex.Lock() singleMsgSuccessCount += uint64(len(storageMsgList)) @@ -204,34 +208,40 @@ func (och *OnlineHistoryConsumerHandler) Run(channelID int) { //} func (och *OnlineHistoryConsumerHandler) MessagesDistributionHandle() { + UserAggregationMsgs := make(map[string][]*pbMsg.MsgDataToMQ, ChannelNum) for { select { case cmd := <-och.msgDistributionCh: switch cmd.Cmd { case ConsumerMsgs: - consumerMessages := cmd.Value.([]*sarama.ConsumerMessage) + triggerChannelValue := cmd.Value.(TriggerChannelValue) + triggerID := triggerChannelValue.triggerID + consumerMessages := triggerChannelValue.cmsgList //Aggregation map[userid]message list + log.Debug(triggerID, "batch messages come to distribution center", len(consumerMessages)) for i := 0; i < len(consumerMessages); i++ { msgFromMQ := pbMsg.MsgDataToMQ{} err := proto.Unmarshal(consumerMessages[i].Value, &msgFromMQ) if err != nil { - log.Error("msg_transfer Unmarshal msg err", "", "msg", string(consumerMessages[i].Value), "err", err.Error()) + log.Error(triggerID, "msg_transfer Unmarshal msg err", "msg", string(consumerMessages[i].Value), "err", err.Error()) return } - if oldM, ok := och.UserAggregationMsgs[string(consumerMessages[i].Key)]; ok { + log.Debug(triggerID, "single msg come to distribution center", msgFromMQ.String()) + if oldM, ok := UserAggregationMsgs[string(consumerMessages[i].Key)]; ok { oldM = append(oldM, &msgFromMQ) - och.UserAggregationMsgs[string(consumerMessages[i].Key)] = oldM + UserAggregationMsgs[string(consumerMessages[i].Key)] = oldM } else { - m := make([]*pbMsg.MsgDataToMQ, 100) + m := make([]*pbMsg.MsgDataToMQ, 0, 100) m = append(m, &msgFromMQ) - och.UserAggregationMsgs[string(consumerMessages[i].Key)] = m + UserAggregationMsgs[string(consumerMessages[i].Key)] = m } } - for userID, v := range och.UserAggregationMsgs { + log.Debug(triggerID, "generate map list users len", len(UserAggregationMsgs)) + for userID, v := range UserAggregationMsgs { if len(v) >= 0 { channelID := getHashCode(userID) % ChannelNum go func(cID uint32, userID string, messages []*pbMsg.MsgDataToMQ) { - och.chArrays[cID] <- Cmd2Value{Cmd: UserMessages, Value: MsgChannelValue{userID: userID, msgList: messages}} + och.chArrays[cID] <- Cmd2Value{Cmd: UserMessages, Value: MsgChannelValue{userID: userID, msgList: messages, triggerID: triggerID}} }(channelID, userID, v) } } @@ -375,23 +385,32 @@ func (OnlineHistoryConsumerHandler) Cleanup(_ sarama.ConsumerGroupSession) error func (och *OnlineHistoryConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { // a instance in the consumer group log.NewDebug("", "online new session msg come", claim.HighWaterMarkOffset(), claim.Topic(), claim.Partition()) - cMsg := make([]*sarama.ConsumerMessage, 500) + cMsg := make([]*sarama.ConsumerMessage, 0, 500) t := time.NewTicker(time.Duration(500) * time.Millisecond) + var triggerID string for msg := range claim.Messages() { //och.TriggerCmd(OnlineTopicBusy) cMsg = append(cMsg, msg) select { case <-t.C: if len(cMsg) >= 0 { - och.msgDistributionCh <- Cmd2Value{Cmd: ConsumerMsgs, Value: cMsg} + triggerID = utils.OperationIDGenerator() + log.Debug(triggerID, "timer trigger msg consumer start", len(cMsg)) + och.msgDistributionCh <- Cmd2Value{Cmd: ConsumerMsgs, Value: TriggerChannelValue{ + triggerID: triggerID, cmsgList: cMsg}} sess.MarkMessage(msg, "") cMsg = cMsg[0:0] + log.Debug(triggerID, "timer trigger msg consumer end", len(cMsg)) } default: if len(cMsg) >= 500 { - och.msgDistributionCh <- Cmd2Value{Cmd: ConsumerMsgs, Value: cMsg} + triggerID = utils.OperationIDGenerator() + log.Debug(triggerID, "length trigger msg consumer start", len(cMsg)) + och.msgDistributionCh <- Cmd2Value{Cmd: ConsumerMsgs, Value: TriggerChannelValue{ + triggerID: triggerID, cmsgList: cMsg}} sess.MarkMessage(msg, "") cMsg = cMsg[0:0] + log.Debug(triggerID, "length trigger msg consumer end", len(cMsg)) } } From 2b142baf192e1175526188c4c51711b85b6fdd63 Mon Sep 17 00:00:00 2001 From: Gordon <1432970085@qq.com> Date: Fri, 20 May 2022 14:48:06 +0800 Subject: [PATCH 27/76] concurrent consumption of messages --- internal/msg_transfer/logic/init.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/msg_transfer/logic/init.go b/internal/msg_transfer/logic/init.go index b4cfff6db..d36a3d8d5 100644 --- a/internal/msg_transfer/logic/init.go +++ b/internal/msg_transfer/logic/init.go @@ -15,7 +15,7 @@ const OnlineTopicVacancy = 0 const Msg = 2 const ConsumerMsgs = 3 const UserMessages = 4 -const ChannelNum = 10 +const ChannelNum = 11 var ( persistentCH PersistentConsumerHandler From aa733542e37023b24203252239b57dc736948bd8 Mon Sep 17 00:00:00 2001 From: Gordon <1432970085@qq.com> Date: Fri, 20 May 2022 15:01:41 +0800 Subject: [PATCH 28/76] concurrent consumption of messages --- internal/msg_transfer/logic/offline_history_msg_handler.go | 4 ++-- internal/msg_transfer/logic/online_history_msg_handler.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/msg_transfer/logic/offline_history_msg_handler.go b/internal/msg_transfer/logic/offline_history_msg_handler.go index 2f63f2007..69994149a 100644 --- a/internal/msg_transfer/logic/offline_history_msg_handler.go +++ b/internal/msg_transfer/logic/offline_history_msg_handler.go @@ -116,7 +116,7 @@ func (och *OfflineHistoryConsumerHandler) MessagesDistributionHandle() { oldM = append(oldM, &msgFromMQ) UserAggregationMsgs[string(consumerMessages[i].Key)] = oldM } else { - m := make([]*pbMsg.MsgDataToMQ, 0, 100) + m := make([]*pbMsg.MsgDataToMQ, 50, 100) m = append(m, &msgFromMQ) UserAggregationMsgs[string(consumerMessages[i].Key)] = m } @@ -276,7 +276,7 @@ func (OfflineHistoryConsumerHandler) Cleanup(_ sarama.ConsumerGroupSession) erro func (och *OfflineHistoryConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { // a instance in the consumer group log.NewDebug("", "online new session msg come", claim.HighWaterMarkOffset(), claim.Topic(), claim.Partition()) - cMsg := make([]*sarama.ConsumerMessage, 0, 500) + cMsg := make([]*sarama.ConsumerMessage, 200, 500) t := time.NewTicker(time.Duration(500) * time.Millisecond) var triggerID string for msg := range claim.Messages() { diff --git a/internal/msg_transfer/logic/online_history_msg_handler.go b/internal/msg_transfer/logic/online_history_msg_handler.go index b33014bc4..37f7d2c57 100644 --- a/internal/msg_transfer/logic/online_history_msg_handler.go +++ b/internal/msg_transfer/logic/online_history_msg_handler.go @@ -231,7 +231,7 @@ func (och *OnlineHistoryConsumerHandler) MessagesDistributionHandle() { oldM = append(oldM, &msgFromMQ) UserAggregationMsgs[string(consumerMessages[i].Key)] = oldM } else { - m := make([]*pbMsg.MsgDataToMQ, 0, 100) + m := make([]*pbMsg.MsgDataToMQ, 50, 100) m = append(m, &msgFromMQ) UserAggregationMsgs[string(consumerMessages[i].Key)] = m } @@ -385,7 +385,7 @@ func (OnlineHistoryConsumerHandler) Cleanup(_ sarama.ConsumerGroupSession) error func (och *OnlineHistoryConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { // a instance in the consumer group log.NewDebug("", "online new session msg come", claim.HighWaterMarkOffset(), claim.Topic(), claim.Partition()) - cMsg := make([]*sarama.ConsumerMessage, 0, 500) + cMsg := make([]*sarama.ConsumerMessage, 200, 500) t := time.NewTicker(time.Duration(500) * time.Millisecond) var triggerID string for msg := range claim.Messages() { From 222d6cdca1e7b10b53cb31631b42d092bbe0c8a4 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Fri, 20 May 2022 15:21:16 +0800 Subject: [PATCH 29/76] fix bug --- internal/msg_transfer/logic/offline_history_msg_handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/msg_transfer/logic/offline_history_msg_handler.go b/internal/msg_transfer/logic/offline_history_msg_handler.go index 69994149a..d6d25ccd4 100644 --- a/internal/msg_transfer/logic/offline_history_msg_handler.go +++ b/internal/msg_transfer/logic/offline_history_msg_handler.go @@ -276,7 +276,7 @@ func (OfflineHistoryConsumerHandler) Cleanup(_ sarama.ConsumerGroupSession) erro func (och *OfflineHistoryConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { // a instance in the consumer group log.NewDebug("", "online new session msg come", claim.HighWaterMarkOffset(), claim.Topic(), claim.Partition()) - cMsg := make([]*sarama.ConsumerMessage, 200, 500) + cMsg := make([]*sarama.ConsumerMessage, 0, 500) t := time.NewTicker(time.Duration(500) * time.Millisecond) var triggerID string for msg := range claim.Messages() { From 7f806389281e81ce5a3e796a2f8c231ead273032 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Fri, 20 May 2022 15:27:25 +0800 Subject: [PATCH 30/76] fix bug --- internal/msg_transfer/logic/online_history_msg_handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/msg_transfer/logic/online_history_msg_handler.go b/internal/msg_transfer/logic/online_history_msg_handler.go index 37f7d2c57..a1951ee21 100644 --- a/internal/msg_transfer/logic/online_history_msg_handler.go +++ b/internal/msg_transfer/logic/online_history_msg_handler.go @@ -385,7 +385,7 @@ func (OnlineHistoryConsumerHandler) Cleanup(_ sarama.ConsumerGroupSession) error func (och *OnlineHistoryConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { // a instance in the consumer group log.NewDebug("", "online new session msg come", claim.HighWaterMarkOffset(), claim.Topic(), claim.Partition()) - cMsg := make([]*sarama.ConsumerMessage, 200, 500) + cMsg := make([]*sarama.ConsumerMessage, 0, 500) t := time.NewTicker(time.Duration(500) * time.Millisecond) var triggerID string for msg := range claim.Messages() { From a2f7edcbcd67ac56032e622f3095b12546143b80 Mon Sep 17 00:00:00 2001 From: Gordon <1432970085@qq.com> Date: Fri, 20 May 2022 15:28:36 +0800 Subject: [PATCH 31/76] concurrent consumption of messages --- internal/msg_transfer/logic/offline_history_msg_handler.go | 4 ++-- internal/msg_transfer/logic/online_history_msg_handler.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/msg_transfer/logic/offline_history_msg_handler.go b/internal/msg_transfer/logic/offline_history_msg_handler.go index 69994149a..2f63f2007 100644 --- a/internal/msg_transfer/logic/offline_history_msg_handler.go +++ b/internal/msg_transfer/logic/offline_history_msg_handler.go @@ -116,7 +116,7 @@ func (och *OfflineHistoryConsumerHandler) MessagesDistributionHandle() { oldM = append(oldM, &msgFromMQ) UserAggregationMsgs[string(consumerMessages[i].Key)] = oldM } else { - m := make([]*pbMsg.MsgDataToMQ, 50, 100) + m := make([]*pbMsg.MsgDataToMQ, 0, 100) m = append(m, &msgFromMQ) UserAggregationMsgs[string(consumerMessages[i].Key)] = m } @@ -276,7 +276,7 @@ func (OfflineHistoryConsumerHandler) Cleanup(_ sarama.ConsumerGroupSession) erro func (och *OfflineHistoryConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { // a instance in the consumer group log.NewDebug("", "online new session msg come", claim.HighWaterMarkOffset(), claim.Topic(), claim.Partition()) - cMsg := make([]*sarama.ConsumerMessage, 200, 500) + cMsg := make([]*sarama.ConsumerMessage, 0, 500) t := time.NewTicker(time.Duration(500) * time.Millisecond) var triggerID string for msg := range claim.Messages() { diff --git a/internal/msg_transfer/logic/online_history_msg_handler.go b/internal/msg_transfer/logic/online_history_msg_handler.go index 37f7d2c57..b33014bc4 100644 --- a/internal/msg_transfer/logic/online_history_msg_handler.go +++ b/internal/msg_transfer/logic/online_history_msg_handler.go @@ -231,7 +231,7 @@ func (och *OnlineHistoryConsumerHandler) MessagesDistributionHandle() { oldM = append(oldM, &msgFromMQ) UserAggregationMsgs[string(consumerMessages[i].Key)] = oldM } else { - m := make([]*pbMsg.MsgDataToMQ, 50, 100) + m := make([]*pbMsg.MsgDataToMQ, 0, 100) m = append(m, &msgFromMQ) UserAggregationMsgs[string(consumerMessages[i].Key)] = m } @@ -385,7 +385,7 @@ func (OnlineHistoryConsumerHandler) Cleanup(_ sarama.ConsumerGroupSession) error func (och *OnlineHistoryConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { // a instance in the consumer group log.NewDebug("", "online new session msg come", claim.HighWaterMarkOffset(), claim.Topic(), claim.Partition()) - cMsg := make([]*sarama.ConsumerMessage, 200, 500) + cMsg := make([]*sarama.ConsumerMessage, 0, 500) t := time.NewTicker(time.Duration(500) * time.Millisecond) var triggerID string for msg := range claim.Messages() { From e12207fc73e1731f2b5590c5fd9552727a4a5735 Mon Sep 17 00:00:00 2001 From: Gordon <1432970085@qq.com> Date: Fri, 20 May 2022 15:40:40 +0800 Subject: [PATCH 32/76] concurrent consumption of messages --- internal/msg_transfer/logic/online_history_msg_handler.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/msg_transfer/logic/online_history_msg_handler.go b/internal/msg_transfer/logic/online_history_msg_handler.go index b33014bc4..622f7eae6 100644 --- a/internal/msg_transfer/logic/online_history_msg_handler.go +++ b/internal/msg_transfer/logic/online_history_msg_handler.go @@ -239,7 +239,9 @@ func (och *OnlineHistoryConsumerHandler) MessagesDistributionHandle() { log.Debug(triggerID, "generate map list users len", len(UserAggregationMsgs)) for userID, v := range UserAggregationMsgs { if len(v) >= 0 { - channelID := getHashCode(userID) % ChannelNum + hashCode := getHashCode(userID) + channelID := hashCode % ChannelNum + log.Debug(triggerID, "generate channelID", hashCode, channelID, userID) go func(cID uint32, userID string, messages []*pbMsg.MsgDataToMQ) { och.chArrays[cID] <- Cmd2Value{Cmd: UserMessages, Value: MsgChannelValue{userID: userID, msgList: messages, triggerID: triggerID}} }(channelID, userID, v) From 8fe929d8ab8692b082ebe8c3c267df5f6dbf0b0a Mon Sep 17 00:00:00 2001 From: Gordon <1432970085@qq.com> Date: Fri, 20 May 2022 15:47:04 +0800 Subject: [PATCH 33/76] concurrent consumption of messages --- internal/msg_transfer/logic/online_history_msg_handler.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/msg_transfer/logic/online_history_msg_handler.go b/internal/msg_transfer/logic/online_history_msg_handler.go index 622f7eae6..4773e1af3 100644 --- a/internal/msg_transfer/logic/online_history_msg_handler.go +++ b/internal/msg_transfer/logic/online_history_msg_handler.go @@ -242,9 +242,9 @@ func (och *OnlineHistoryConsumerHandler) MessagesDistributionHandle() { hashCode := getHashCode(userID) channelID := hashCode % ChannelNum log.Debug(triggerID, "generate channelID", hashCode, channelID, userID) - go func(cID uint32, userID string, messages []*pbMsg.MsgDataToMQ) { - och.chArrays[cID] <- Cmd2Value{Cmd: UserMessages, Value: MsgChannelValue{userID: userID, msgList: messages, triggerID: triggerID}} - }(channelID, userID, v) + //go func(cID uint32, userID string, messages []*pbMsg.MsgDataToMQ) { + och.chArrays[channelID] <- Cmd2Value{Cmd: UserMessages, Value: MsgChannelValue{userID: userID, msgList: v, triggerID: triggerID}} + //}(channelID, userID, v) } } } From 930923e330f6e4a12a927100935a410ba5482504 Mon Sep 17 00:00:00 2001 From: Gordon <1432970085@qq.com> Date: Fri, 20 May 2022 15:51:37 +0800 Subject: [PATCH 34/76] concurrent consumption of messages --- .../msg_transfer/logic/offline_history_msg_handler.go | 9 +++++---- .../msg_transfer/logic/online_history_msg_handler.go | 9 +++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/internal/msg_transfer/logic/offline_history_msg_handler.go b/internal/msg_transfer/logic/offline_history_msg_handler.go index 2f63f2007..a2a7f7673 100644 --- a/internal/msg_transfer/logic/offline_history_msg_handler.go +++ b/internal/msg_transfer/logic/offline_history_msg_handler.go @@ -28,14 +28,15 @@ func (mc *OfflineHistoryConsumerHandler) Init(cmdCh chan Cmd2Value) { go mc.MessagesDistributionHandle() mc.cmdCh = cmdCh mc.msgCh = make(chan Cmd2Value, 1000) + for i := 0; i < ChannelNum; i++ { + mc.chArrays[i] = make(chan Cmd2Value, 1000) + go mc.Run(i) + } if config.Config.ReliableStorage { mc.msgHandle[config.Config.Kafka.Ws2mschat.Topic] = mc.handleChatWs2Mongo } else { mc.msgHandle[config.Config.Kafka.Ws2mschat.Topic] = mc.handleChatWs2MongoLowReliability - for i := 0; i < ChannelNum; i++ { - mc.chArrays[i] = make(chan Cmd2Value, 1000) - go mc.Run(i) - } + } mc.historyConsumerGroup = kfk.NewMConsumerGroup(&kfk.MConsumerGroupConfig{KafkaVersion: sarama.V0_10_2_0, OffsetsInitial: sarama.OffsetNewest, IsReturnErr: false}, []string{config.Config.Kafka.Ws2mschatOffline.Topic}, diff --git a/internal/msg_transfer/logic/online_history_msg_handler.go b/internal/msg_transfer/logic/online_history_msg_handler.go index 4773e1af3..6efef8124 100644 --- a/internal/msg_transfer/logic/online_history_msg_handler.go +++ b/internal/msg_transfer/logic/online_history_msg_handler.go @@ -48,14 +48,15 @@ func (och *OnlineHistoryConsumerHandler) Init(cmdCh chan Cmd2Value) { go och.MessagesDistributionHandle() och.cmdCh = cmdCh och.msgCh = make(chan Cmd2Value, 1000) + for i := 0; i < ChannelNum; i++ { + och.chArrays[i] = make(chan Cmd2Value, 1000) + go och.Run(i) + } if config.Config.ReliableStorage { och.msgHandle[config.Config.Kafka.Ws2mschat.Topic] = och.handleChatWs2Mongo } else { och.msgHandle[config.Config.Kafka.Ws2mschat.Topic] = och.handleChatWs2MongoLowReliability - for i := 0; i < ChannelNum; i++ { - och.chArrays[i] = make(chan Cmd2Value, 1000) - go och.Run(i) - } + } och.historyConsumerGroup = kfk.NewMConsumerGroup(&kfk.MConsumerGroupConfig{KafkaVersion: sarama.V0_10_2_0, OffsetsInitial: sarama.OffsetNewest, IsReturnErr: false}, []string{config.Config.Kafka.Ws2mschat.Topic}, From 223b2678b4903201011a7d7b2e29e7229e92653a Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Fri, 20 May 2022 16:03:33 +0800 Subject: [PATCH 35/76] fix bug --- internal/msg_transfer/logic/offline_history_msg_handler.go | 4 ++-- internal/msg_transfer/logic/online_history_msg_handler.go | 4 ++-- pkg/common/db/batch_insert_chat.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/msg_transfer/logic/offline_history_msg_handler.go b/internal/msg_transfer/logic/offline_history_msg_handler.go index a2a7f7673..78c867851 100644 --- a/internal/msg_transfer/logic/offline_history_msg_handler.go +++ b/internal/msg_transfer/logic/offline_history_msg_handler.go @@ -52,8 +52,8 @@ func (och *OfflineHistoryConsumerHandler) Run(channelID int) { msgChannelValue := cmd.Value.(MsgChannelValue) msgList := msgChannelValue.msgList triggerID := msgChannelValue.triggerID - storageMsgList := make([]*pbMsg.MsgDataToMQ, 80) - pushMsgList := make([]*pbMsg.MsgDataToMQ, 80) + storageMsgList := make([]*pbMsg.MsgDataToMQ, 0, 80) + pushMsgList := make([]*pbMsg.MsgDataToMQ, 0, 80) log.Debug(triggerID, "msg arrived channel", "channel id", channelID, msgList, msgChannelValue.userID, len(msgList)) for _, v := range msgList { log.Debug(triggerID, "msg come to storage center", v.String()) diff --git a/internal/msg_transfer/logic/online_history_msg_handler.go b/internal/msg_transfer/logic/online_history_msg_handler.go index 6efef8124..173f534b2 100644 --- a/internal/msg_transfer/logic/online_history_msg_handler.go +++ b/internal/msg_transfer/logic/online_history_msg_handler.go @@ -96,8 +96,8 @@ func (och *OnlineHistoryConsumerHandler) Run(channelID int) { msgChannelValue := cmd.Value.(MsgChannelValue) msgList := msgChannelValue.msgList triggerID := msgChannelValue.triggerID - storageMsgList := make([]*pbMsg.MsgDataToMQ, 80) - pushMsgList := make([]*pbMsg.MsgDataToMQ, 80) + storageMsgList := make([]*pbMsg.MsgDataToMQ, 0, 80) + pushMsgList := make([]*pbMsg.MsgDataToMQ, 0, 80) log.Debug(triggerID, "msg arrived channel", "channel id", channelID, msgList, msgChannelValue.userID, len(msgList)) for _, v := range msgList { log.Debug(triggerID, "msg come to storage center", v.String()) diff --git a/pkg/common/db/batch_insert_chat.go b/pkg/common/db/batch_insert_chat.go index b3211e701..e6b7fa7b5 100644 --- a/pkg/common/db/batch_insert_chat.go +++ b/pkg/common/db/batch_insert_chat.go @@ -35,7 +35,7 @@ func (d *DataBases) BatchInsertChat(userID string, msgList []*pbMsg.MsgDataToMQ, msgListToMongoNext := make([]MsgInfo, 0) seqUid := "" seqUidNext := "" - log.Debug(operationID, "remain ", remain, "insertCounter ", insertCounter, "currentMaxSeq ", currentMaxSeq, userID) + log.Debug(operationID, "remain ", remain, "insertCounter ", insertCounter, "currentMaxSeq ", currentMaxSeq, userID, len(msgList)) for _, m := range msgList { currentMaxSeq++ sMsg := MsgInfo{} From 15329a97fe2ed1a3b0c6d1a7011d856b8b0deb61 Mon Sep 17 00:00:00 2001 From: Gordon <1432970085@qq.com> Date: Fri, 20 May 2022 16:07:32 +0800 Subject: [PATCH 36/76] concurrent consumption of messages --- internal/msg_transfer/logic/online_history_msg_handler.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/msg_transfer/logic/online_history_msg_handler.go b/internal/msg_transfer/logic/online_history_msg_handler.go index 6efef8124..297c3ea99 100644 --- a/internal/msg_transfer/logic/online_history_msg_handler.go +++ b/internal/msg_transfer/logic/online_history_msg_handler.go @@ -96,8 +96,8 @@ func (och *OnlineHistoryConsumerHandler) Run(channelID int) { msgChannelValue := cmd.Value.(MsgChannelValue) msgList := msgChannelValue.msgList triggerID := msgChannelValue.triggerID - storageMsgList := make([]*pbMsg.MsgDataToMQ, 80) - pushMsgList := make([]*pbMsg.MsgDataToMQ, 80) + storageMsgList := make([]*pbMsg.MsgDataToMQ, 0, 80) + pushMsgList := make([]*pbMsg.MsgDataToMQ, 0, 80) log.Debug(triggerID, "msg arrived channel", "channel id", channelID, msgList, msgChannelValue.userID, len(msgList)) for _, v := range msgList { log.Debug(triggerID, "msg come to storage center", v.String()) @@ -119,7 +119,7 @@ func (och *OnlineHistoryConsumerHandler) Run(channelID int) { // log.NewError(msgFromMQ.OperationID, "SessionType error", msgFromMQ.String()) // return //} - + log.Debug(triggerID, "msg storage length", len(storageMsgList), "push length", len(pushMsgList)) err := saveUserChatList(msgChannelValue.userID, storageMsgList, triggerID) if err != nil { singleMsgFailedCount += uint64(len(storageMsgList)) @@ -227,7 +227,7 @@ func (och *OnlineHistoryConsumerHandler) MessagesDistributionHandle() { log.Error(triggerID, "msg_transfer Unmarshal msg err", "msg", string(consumerMessages[i].Value), "err", err.Error()) return } - log.Debug(triggerID, "single msg come to distribution center", msgFromMQ.String()) + log.Debug(triggerID, "single msg come to distribution center", msgFromMQ.String(), string(consumerMessages[i].Key)) if oldM, ok := UserAggregationMsgs[string(consumerMessages[i].Key)]; ok { oldM = append(oldM, &msgFromMQ) UserAggregationMsgs[string(consumerMessages[i].Key)] = oldM From abcfaa05e277fbb2659e829f9adf7b3921daada3 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Fri, 20 May 2022 16:12:05 +0800 Subject: [PATCH 37/76] log --- pkg/common/db/batch_insert_chat.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/common/db/batch_insert_chat.go b/pkg/common/db/batch_insert_chat.go index e6b7fa7b5..7d1b61342 100644 --- a/pkg/common/db/batch_insert_chat.go +++ b/pkg/common/db/batch_insert_chat.go @@ -37,6 +37,7 @@ func (d *DataBases) BatchInsertChat(userID string, msgList []*pbMsg.MsgDataToMQ, seqUidNext := "" log.Debug(operationID, "remain ", remain, "insertCounter ", insertCounter, "currentMaxSeq ", currentMaxSeq, userID, len(msgList)) for _, m := range msgList { + log.Debug(operationID, "msg node ", m.String(), m.MsgData.ClientMsgID) currentMaxSeq++ sMsg := MsgInfo{} sMsg.SendTime = m.MsgData.SendTime From 686d77dcc1a1e61f1eca0e992c6226738ad6fea6 Mon Sep 17 00:00:00 2001 From: Gordon <1432970085@qq.com> Date: Fri, 20 May 2022 16:53:35 +0800 Subject: [PATCH 38/76] concurrent consumption of messages --- .../logic/online_history_msg_handler.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/internal/msg_transfer/logic/online_history_msg_handler.go b/internal/msg_transfer/logic/online_history_msg_handler.go index 297c3ea99..36fcb45a0 100644 --- a/internal/msg_transfer/logic/online_history_msg_handler.go +++ b/internal/msg_transfer/logic/online_history_msg_handler.go @@ -397,20 +397,30 @@ func (och *OnlineHistoryConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupS select { case <-t.C: if len(cMsg) >= 0 { + ccMsg := make([]*sarama.ConsumerMessage, 0, 500) + for _, v := range cMsg { + ccMsg = append(ccMsg, v) + + } triggerID = utils.OperationIDGenerator() log.Debug(triggerID, "timer trigger msg consumer start", len(cMsg)) och.msgDistributionCh <- Cmd2Value{Cmd: ConsumerMsgs, Value: TriggerChannelValue{ - triggerID: triggerID, cmsgList: cMsg}} + triggerID: triggerID, cmsgList: ccMsg}} sess.MarkMessage(msg, "") cMsg = cMsg[0:0] log.Debug(triggerID, "timer trigger msg consumer end", len(cMsg)) } default: if len(cMsg) >= 500 { + ccMsg := make([]*sarama.ConsumerMessage, 0, 500) + for _, v := range cMsg { + ccMsg = append(ccMsg, v) + + } triggerID = utils.OperationIDGenerator() log.Debug(triggerID, "length trigger msg consumer start", len(cMsg)) och.msgDistributionCh <- Cmd2Value{Cmd: ConsumerMsgs, Value: TriggerChannelValue{ - triggerID: triggerID, cmsgList: cMsg}} + triggerID: triggerID, cmsgList: ccMsg}} sess.MarkMessage(msg, "") cMsg = cMsg[0:0] log.Debug(triggerID, "length trigger msg consumer end", len(cMsg)) From a12c8f4dbc6e55c4a0c59779d9dc5209a8e71fc0 Mon Sep 17 00:00:00 2001 From: Gordon <1432970085@qq.com> Date: Fri, 20 May 2022 17:06:51 +0800 Subject: [PATCH 39/76] concurrent consumption of messages --- internal/msg_transfer/logic/online_history_msg_handler.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/msg_transfer/logic/online_history_msg_handler.go b/internal/msg_transfer/logic/online_history_msg_handler.go index 36fcb45a0..6e38ac015 100644 --- a/internal/msg_transfer/logic/online_history_msg_handler.go +++ b/internal/msg_transfer/logic/online_history_msg_handler.go @@ -407,7 +407,7 @@ func (och *OnlineHistoryConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupS och.msgDistributionCh <- Cmd2Value{Cmd: ConsumerMsgs, Value: TriggerChannelValue{ triggerID: triggerID, cmsgList: ccMsg}} sess.MarkMessage(msg, "") - cMsg = cMsg[0:0] + cMsg = make([]*sarama.ConsumerMessage, 0, 500) log.Debug(triggerID, "timer trigger msg consumer end", len(cMsg)) } default: @@ -422,7 +422,7 @@ func (och *OnlineHistoryConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupS och.msgDistributionCh <- Cmd2Value{Cmd: ConsumerMsgs, Value: TriggerChannelValue{ triggerID: triggerID, cmsgList: ccMsg}} sess.MarkMessage(msg, "") - cMsg = cMsg[0:0] + cMsg = make([]*sarama.ConsumerMessage, 0, 500) log.Debug(triggerID, "length trigger msg consumer end", len(cMsg)) } From c5f055718ab8d87f063d7e783bd7b5c3b5d036df Mon Sep 17 00:00:00 2001 From: Gordon <1432970085@qq.com> Date: Fri, 20 May 2022 17:32:40 +0800 Subject: [PATCH 40/76] concurrent consumption of messages --- internal/msg_transfer/logic/online_history_msg_handler.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/msg_transfer/logic/online_history_msg_handler.go b/internal/msg_transfer/logic/online_history_msg_handler.go index 6e38ac015..f36d6cfaf 100644 --- a/internal/msg_transfer/logic/online_history_msg_handler.go +++ b/internal/msg_transfer/logic/online_history_msg_handler.go @@ -209,8 +209,8 @@ func (och *OnlineHistoryConsumerHandler) Run(channelID int) { //} func (och *OnlineHistoryConsumerHandler) MessagesDistributionHandle() { - UserAggregationMsgs := make(map[string][]*pbMsg.MsgDataToMQ, ChannelNum) for { + UserAggregationMsgs := make(map[string][]*pbMsg.MsgDataToMQ, ChannelNum) select { case cmd := <-och.msgDistributionCh: switch cmd.Cmd { @@ -250,6 +250,7 @@ func (och *OnlineHistoryConsumerHandler) MessagesDistributionHandle() { } } } + } } @@ -403,7 +404,7 @@ func (och *OnlineHistoryConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupS } triggerID = utils.OperationIDGenerator() - log.Debug(triggerID, "timer trigger msg consumer start", len(cMsg)) + log.Debug(triggerID, "timer trigger msg consumer start", len(ccMsg)) och.msgDistributionCh <- Cmd2Value{Cmd: ConsumerMsgs, Value: TriggerChannelValue{ triggerID: triggerID, cmsgList: ccMsg}} sess.MarkMessage(msg, "") @@ -418,7 +419,7 @@ func (och *OnlineHistoryConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupS } triggerID = utils.OperationIDGenerator() - log.Debug(triggerID, "length trigger msg consumer start", len(cMsg)) + log.Debug(triggerID, "length trigger msg consumer start", len(ccMsg)) och.msgDistributionCh <- Cmd2Value{Cmd: ConsumerMsgs, Value: TriggerChannelValue{ triggerID: triggerID, cmsgList: ccMsg}} sess.MarkMessage(msg, "") From 757829d286697a95c11409f754bfa06ba42dd8c3 Mon Sep 17 00:00:00 2001 From: Gordon <1432970085@qq.com> Date: Fri, 20 May 2022 17:43:25 +0800 Subject: [PATCH 41/76] concurrent consumption of messages --- internal/msg_transfer/logic/online_history_msg_handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/msg_transfer/logic/online_history_msg_handler.go b/internal/msg_transfer/logic/online_history_msg_handler.go index f36d6cfaf..5aa94d607 100644 --- a/internal/msg_transfer/logic/online_history_msg_handler.go +++ b/internal/msg_transfer/logic/online_history_msg_handler.go @@ -390,7 +390,7 @@ func (och *OnlineHistoryConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupS claim sarama.ConsumerGroupClaim) error { // a instance in the consumer group log.NewDebug("", "online new session msg come", claim.HighWaterMarkOffset(), claim.Topic(), claim.Partition()) cMsg := make([]*sarama.ConsumerMessage, 0, 500) - t := time.NewTicker(time.Duration(500) * time.Millisecond) + t := time.NewTicker(time.Duration(10) * time.Millisecond) var triggerID string for msg := range claim.Messages() { //och.TriggerCmd(OnlineTopicBusy) From 624bb0a7e522e3e71d787a14a398928407874903 Mon Sep 17 00:00:00 2001 From: Gordon <1432970085@qq.com> Date: Fri, 20 May 2022 18:46:19 +0800 Subject: [PATCH 42/76] concurrent consumption of messages --- pkg/common/db/batch_insert_chat.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/common/db/batch_insert_chat.go b/pkg/common/db/batch_insert_chat.go index 7d1b61342..eec173897 100644 --- a/pkg/common/db/batch_insert_chat.go +++ b/pkg/common/db/batch_insert_chat.go @@ -28,8 +28,11 @@ func (d *DataBases) BatchInsertChat(userID string, msgList []*pbMsg.MsgDataToMQ, } else { return utils.Wrap(err, "") } - - remain := uint64(GetSingleGocMsgNum()) - (currentMaxSeq % uint64(GetSingleGocMsgNum())) + var remain uint64 + if currentMaxSeq < uint64(GetSingleGocMsgNum()) { + remain = uint64(GetSingleGocMsgNum()-1) - (currentMaxSeq % uint64(GetSingleGocMsgNum())) + } + remain = uint64(GetSingleGocMsgNum()) - (currentMaxSeq % uint64(GetSingleGocMsgNum())) insertCounter := uint64(0) msgListToMongo := make([]MsgInfo, 0) msgListToMongoNext := make([]MsgInfo, 0) From adb6337a27778025d248b3e86ee72aab1bd54e4d Mon Sep 17 00:00:00 2001 From: Gordon <1432970085@qq.com> Date: Fri, 20 May 2022 19:34:45 +0800 Subject: [PATCH 43/76] concurrent consumption of messages --- pkg/common/db/batch_insert_chat.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/common/db/batch_insert_chat.go b/pkg/common/db/batch_insert_chat.go index eec173897..64d2d342b 100644 --- a/pkg/common/db/batch_insert_chat.go +++ b/pkg/common/db/batch_insert_chat.go @@ -31,8 +31,9 @@ func (d *DataBases) BatchInsertChat(userID string, msgList []*pbMsg.MsgDataToMQ, var remain uint64 if currentMaxSeq < uint64(GetSingleGocMsgNum()) { remain = uint64(GetSingleGocMsgNum()-1) - (currentMaxSeq % uint64(GetSingleGocMsgNum())) + } else { + remain = uint64(GetSingleGocMsgNum()) - (currentMaxSeq % uint64(GetSingleGocMsgNum())) } - remain = uint64(GetSingleGocMsgNum()) - (currentMaxSeq % uint64(GetSingleGocMsgNum())) insertCounter := uint64(0) msgListToMongo := make([]MsgInfo, 0) msgListToMongoNext := make([]MsgInfo, 0) @@ -54,7 +55,6 @@ func (d *DataBases) BatchInsertChat(userID string, msgList []*pbMsg.MsgDataToMQ, log.Debug(operationID, "msgListToMongoNext ", seqUidNext, m.MsgData.Seq, m.MsgData.ClientMsgID, insertCounter, remain) continue } - if insertCounter < remain { msgListToMongo = append(msgListToMongo, sMsg) insertCounter++ From eb27db22322aca2fc418d3371d6440f60b959f13 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Fri, 20 May 2022 19:36:06 +0800 Subject: [PATCH 44/76] fix bug --- pkg/common/db/batch_insert_chat.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/common/db/batch_insert_chat.go b/pkg/common/db/batch_insert_chat.go index eec173897..8ef1117a2 100644 --- a/pkg/common/db/batch_insert_chat.go +++ b/pkg/common/db/batch_insert_chat.go @@ -31,8 +31,9 @@ func (d *DataBases) BatchInsertChat(userID string, msgList []*pbMsg.MsgDataToMQ, var remain uint64 if currentMaxSeq < uint64(GetSingleGocMsgNum()) { remain = uint64(GetSingleGocMsgNum()-1) - (currentMaxSeq % uint64(GetSingleGocMsgNum())) + } else { + remain = uint64(GetSingleGocMsgNum()) - (currentMaxSeq % uint64(GetSingleGocMsgNum())) } - remain = uint64(GetSingleGocMsgNum()) - (currentMaxSeq % uint64(GetSingleGocMsgNum())) insertCounter := uint64(0) msgListToMongo := make([]MsgInfo, 0) msgListToMongoNext := make([]MsgInfo, 0) From 9f62de042087ac718729094c0ab28af9854d085b Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Fri, 20 May 2022 19:46:49 +0800 Subject: [PATCH 45/76] fix bug --- pkg/common/db/batch_insert_chat.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/common/db/batch_insert_chat.go b/pkg/common/db/batch_insert_chat.go index 64d2d342b..9fd25f8de 100644 --- a/pkg/common/db/batch_insert_chat.go +++ b/pkg/common/db/batch_insert_chat.go @@ -10,7 +10,6 @@ import ( "github.com/garyburd/redigo/redis" "github.com/golang/protobuf/proto" "go.mongodb.org/mongo-driver/bson" - "time" ) func (d *DataBases) BatchInsertChat(userID string, msgList []*pbMsg.MsgDataToMQ, operationID string) error { @@ -66,7 +65,9 @@ func (d *DataBases) BatchInsertChat(userID string, msgList []*pbMsg.MsgDataToMQ, log.Debug(operationID, "msgListToMongoNext ", seqUidNext, m.MsgData.Seq, m.MsgData.ClientMsgID, insertCounter, remain) } } - ctx, _ := context.WithTimeout(context.Background(), time.Duration(config.Config.Mongo.DBTimeout)*time.Second) + // ctx, _ := context.WithTimeout(context.Background(), time.Duration(config.Config.Mongo.DBTimeout)*time.Second) + + ctx := context.Background() c := d.mongoClient.Database(config.Config.Mongo.DBDatabase).Collection(cChat) if seqUid != "" { From 38317c549794b07d8ff170f94709ca6b502dc6b1 Mon Sep 17 00:00:00 2001 From: Gordon <1432970085@qq.com> Date: Fri, 20 May 2022 20:10:24 +0800 Subject: [PATCH 46/76] concurrent consumption of messages --- pkg/common/db/batch_insert_chat.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/common/db/batch_insert_chat.go b/pkg/common/db/batch_insert_chat.go index 64d2d342b..9b78c8822 100644 --- a/pkg/common/db/batch_insert_chat.go +++ b/pkg/common/db/batch_insert_chat.go @@ -32,7 +32,7 @@ func (d *DataBases) BatchInsertChat(userID string, msgList []*pbMsg.MsgDataToMQ, if currentMaxSeq < uint64(GetSingleGocMsgNum()) { remain = uint64(GetSingleGocMsgNum()-1) - (currentMaxSeq % uint64(GetSingleGocMsgNum())) } else { - remain = uint64(GetSingleGocMsgNum()) - (currentMaxSeq % uint64(GetSingleGocMsgNum())) + remain = uint64(GetSingleGocMsgNum()) - ((currentMaxSeq - 4999) % uint64(GetSingleGocMsgNum())) } insertCounter := uint64(0) msgListToMongo := make([]MsgInfo, 0) From 0265ea45b2a1b428e4d07e6cbea8227804b26527 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Fri, 20 May 2022 20:11:47 +0800 Subject: [PATCH 47/76] fix bug --- config/config.yaml | 4 ++-- pkg/common/db/batch_insert_chat.go | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/config/config.yaml b/config/config.yaml index ee973fe37..1cfee4e1d 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -22,12 +22,12 @@ mongo: dbUri: ""#当dbUri值不为空则直接使用该值 dbAddress: [ 127.0.0.1:37017 ] #mongo地址 目前仅支持单机，默认即可 dbDirect: false - dbTimeout: 10 + dbTimeout: 60 dbDatabase: openIM #mongo db 默认即可 dbSource: admin dbUserName: #mongo用户名，建议先不设置 dbPassword: #mongo密码，建议先不设置 - dbMaxPoolSize: 20 + dbMaxPoolSize: 100 dbRetainChatRecords: 3650 #mongo保存离线消息时间（天），根据需求修改 redis: diff --git a/pkg/common/db/batch_insert_chat.go b/pkg/common/db/batch_insert_chat.go index 9fd25f8de..2f0884441 100644 --- a/pkg/common/db/batch_insert_chat.go +++ b/pkg/common/db/batch_insert_chat.go @@ -31,13 +31,14 @@ func (d *DataBases) BatchInsertChat(userID string, msgList []*pbMsg.MsgDataToMQ, if currentMaxSeq < uint64(GetSingleGocMsgNum()) { remain = uint64(GetSingleGocMsgNum()-1) - (currentMaxSeq % uint64(GetSingleGocMsgNum())) } else { - remain = uint64(GetSingleGocMsgNum()) - (currentMaxSeq % uint64(GetSingleGocMsgNum())) + remain = uint64(GetSingleGocMsgNum()) - ((currentMaxSeq - (uint64(GetSingleGocMsgNum()) - 1)) % uint64(GetSingleGocMsgNum())) } insertCounter := uint64(0) msgListToMongo := make([]MsgInfo, 0) msgListToMongoNext := make([]MsgInfo, 0) seqUid := "" seqUidNext := "" + //1, log.Debug(operationID, "remain ", remain, "insertCounter ", insertCounter, "currentMaxSeq ", currentMaxSeq, userID, len(msgList)) for _, m := range msgList { log.Debug(operationID, "msg node ", m.String(), m.MsgData.ClientMsgID) From 55907a9a05b8f81ca32311c83b6079a43df0a10d Mon Sep 17 00:00:00 2001 From: Gordon <1432970085@qq.com> Date: Fri, 20 May 2022 21:02:18 +0800 Subject: [PATCH 48/76] concurrent consumption of messages --- internal/msg_transfer/logic/online_history_msg_handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/msg_transfer/logic/online_history_msg_handler.go b/internal/msg_transfer/logic/online_history_msg_handler.go index 5aa94d607..9d0fad271 100644 --- a/internal/msg_transfer/logic/online_history_msg_handler.go +++ b/internal/msg_transfer/logic/online_history_msg_handler.go @@ -390,7 +390,7 @@ func (och *OnlineHistoryConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupS claim sarama.ConsumerGroupClaim) error { // a instance in the consumer group log.NewDebug("", "online new session msg come", claim.HighWaterMarkOffset(), claim.Topic(), claim.Partition()) cMsg := make([]*sarama.ConsumerMessage, 0, 500) - t := time.NewTicker(time.Duration(10) * time.Millisecond) + t := time.NewTicker(time.Duration(100) * time.Millisecond) var triggerID string for msg := range claim.Messages() { //och.TriggerCmd(OnlineTopicBusy) From 01b065b3c23d4bd826af944d1ba6018e5bc27d40 Mon Sep 17 00:00:00 2001 From: Gordon <1432970085@qq.com> Date: Fri, 20 May 2022 21:58:23 +0800 Subject: [PATCH 49/76] concurrent consumption of messages --- .../logic/online_history_msg_handler.go | 37 +++++++++---------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/internal/msg_transfer/logic/online_history_msg_handler.go b/internal/msg_transfer/logic/online_history_msg_handler.go index 9d0fad271..c20422551 100644 --- a/internal/msg_transfer/logic/online_history_msg_handler.go +++ b/internal/msg_transfer/logic/online_history_msg_handler.go @@ -392,26 +392,11 @@ func (och *OnlineHistoryConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupS cMsg := make([]*sarama.ConsumerMessage, 0, 500) t := time.NewTicker(time.Duration(100) * time.Millisecond) var triggerID string - for msg := range claim.Messages() { + for { //och.TriggerCmd(OnlineTopicBusy) - cMsg = append(cMsg, msg) select { - case <-t.C: - if len(cMsg) >= 0 { - ccMsg := make([]*sarama.ConsumerMessage, 0, 500) - for _, v := range cMsg { - ccMsg = append(ccMsg, v) - - } - triggerID = utils.OperationIDGenerator() - log.Debug(triggerID, "timer trigger msg consumer start", len(ccMsg)) - och.msgDistributionCh <- Cmd2Value{Cmd: ConsumerMsgs, Value: TriggerChannelValue{ - triggerID: triggerID, cmsgList: ccMsg}} - sess.MarkMessage(msg, "") - cMsg = make([]*sarama.ConsumerMessage, 0, 500) - log.Debug(triggerID, "timer trigger msg consumer end", len(cMsg)) - } - default: + case msg := <-claim.Messages(): + cMsg = append(cMsg, msg) if len(cMsg) >= 500 { ccMsg := make([]*sarama.ConsumerMessage, 0, 500) for _, v := range cMsg { @@ -426,9 +411,23 @@ func (och *OnlineHistoryConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupS cMsg = make([]*sarama.ConsumerMessage, 0, 500) log.Debug(triggerID, "length trigger msg consumer end", len(cMsg)) } + case <-t.C: + if len(cMsg) > 0 { + ccMsg := make([]*sarama.ConsumerMessage, 0, 500) + for _, v := range cMsg { + ccMsg = append(ccMsg, v) + } + triggerID = utils.OperationIDGenerator() + log.Debug(triggerID, "timer trigger msg consumer start", len(ccMsg)) + och.msgDistributionCh <- Cmd2Value{Cmd: ConsumerMsgs, Value: TriggerChannelValue{ + triggerID: triggerID, cmsgList: ccMsg}} + sess.MarkMessage(cMsg[len(cMsg)-1], "") + cMsg = make([]*sarama.ConsumerMessage, 0, 500) + log.Debug(triggerID, "timer trigger msg consumer end", len(cMsg)) + } } - log.NewDebug("", "online kafka get info to mongo", "msgTopic", msg.Topic, "msgPartition", msg.Partition, "online", msg.Offset, claim.HighWaterMarkOffset()) + //log.NewDebug("", "online kafka get info to mongo", "msgTopic", msg.Topic, "msgPartition", msg.Partition, "online", msg.Offset, claim.HighWaterMarkOffset()) } return nil From c2e7863ca0143d052ebed7543bd79aa3db981499 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 21 May 2022 07:32:45 +0800 Subject: [PATCH 50/76] fix bug --- pkg/common/db/batch_insert_chat.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/pkg/common/db/batch_insert_chat.go b/pkg/common/db/batch_insert_chat.go index a7baef303..0354082d0 100644 --- a/pkg/common/db/batch_insert_chat.go +++ b/pkg/common/db/batch_insert_chat.go @@ -28,11 +28,20 @@ func (d *DataBases) BatchInsertChat(userID string, msgList []*pbMsg.MsgDataToMQ, return utils.Wrap(err, "") } var remain uint64 + //if currentMaxSeq < uint64(GetSingleGocMsgNum()) { + // remain = uint64(GetSingleGocMsgNum()-1) - (currentMaxSeq % uint64(GetSingleGocMsgNum())) + //} else { + // remain = uint64(GetSingleGocMsgNum()) - ((currentMaxSeq - (uint64(GetSingleGocMsgNum()) - 1)) % uint64(GetSingleGocMsgNum())) + //} + + blk0 := uint64(GetSingleGocMsgNum() - 1) if currentMaxSeq < uint64(GetSingleGocMsgNum()) { - remain = uint64(GetSingleGocMsgNum()-1) - (currentMaxSeq % uint64(GetSingleGocMsgNum())) + remain = blk0 - currentMaxSeq } else { - remain = uint64(GetSingleGocMsgNum()) - ((currentMaxSeq - (uint64(GetSingleGocMsgNum()) - 1)) % uint64(GetSingleGocMsgNum())) + excludeBlk0 := currentMaxSeq - blk0 + remain = (uint64(GetSingleGocMsgNum()) - (excludeBlk0 % uint64(GetSingleGocMsgNum()))) % uint64(GetSingleGocMsgNum()) } + insertCounter := uint64(0) msgListToMongo := make([]MsgInfo, 0) msgListToMongoNext := make([]MsgInfo, 0) From 79b5551ec1370078ca6434506e9aa34af423ee28 Mon Sep 17 00:00:00 2001 From: Gordon <1432970085@qq.com> Date: Sat, 21 May 2022 09:58:27 +0800 Subject: [PATCH 51/76] log change --- .../msg_transfer/logic/online_history_msg_handler.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/msg_transfer/logic/online_history_msg_handler.go b/internal/msg_transfer/logic/online_history_msg_handler.go index c20422551..7e4026088 100644 --- a/internal/msg_transfer/logic/online_history_msg_handler.go +++ b/internal/msg_transfer/logic/online_history_msg_handler.go @@ -389,7 +389,7 @@ func (OnlineHistoryConsumerHandler) Cleanup(_ sarama.ConsumerGroupSession) error func (och *OnlineHistoryConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { // a instance in the consumer group log.NewDebug("", "online new session msg come", claim.HighWaterMarkOffset(), claim.Topic(), claim.Partition()) - cMsg := make([]*sarama.ConsumerMessage, 0, 500) + cMsg := make([]*sarama.ConsumerMessage, 0, 1000) t := time.NewTicker(time.Duration(100) * time.Millisecond) var triggerID string for { @@ -397,8 +397,8 @@ func (och *OnlineHistoryConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupS select { case msg := <-claim.Messages(): cMsg = append(cMsg, msg) - if len(cMsg) >= 500 { - ccMsg := make([]*sarama.ConsumerMessage, 0, 500) + if len(cMsg) >= 1000 { + ccMsg := make([]*sarama.ConsumerMessage, 0, 1000) for _, v := range cMsg { ccMsg = append(ccMsg, v) @@ -408,12 +408,12 @@ func (och *OnlineHistoryConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupS och.msgDistributionCh <- Cmd2Value{Cmd: ConsumerMsgs, Value: TriggerChannelValue{ triggerID: triggerID, cmsgList: ccMsg}} sess.MarkMessage(msg, "") - cMsg = make([]*sarama.ConsumerMessage, 0, 500) + cMsg = make([]*sarama.ConsumerMessage, 0, 1000) log.Debug(triggerID, "length trigger msg consumer end", len(cMsg)) } case <-t.C: if len(cMsg) > 0 { - ccMsg := make([]*sarama.ConsumerMessage, 0, 500) + ccMsg := make([]*sarama.ConsumerMessage, 0, 1000) for _, v := range cMsg { ccMsg = append(ccMsg, v) } @@ -422,7 +422,7 @@ func (och *OnlineHistoryConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupS och.msgDistributionCh <- Cmd2Value{Cmd: ConsumerMsgs, Value: TriggerChannelValue{ triggerID: triggerID, cmsgList: ccMsg}} sess.MarkMessage(cMsg[len(cMsg)-1], "") - cMsg = make([]*sarama.ConsumerMessage, 0, 500) + cMsg = make([]*sarama.ConsumerMessage, 0, 1000) log.Debug(triggerID, "timer trigger msg consumer end", len(cMsg)) } From 5cdba3c00a8ef05334bb3eb674981cb82af33666 Mon Sep 17 00:00:00 2001 From: Gordon <1432970085@qq.com> Date: Sat, 21 May 2022 10:19:02 +0800 Subject: [PATCH 52/76] log change --- internal/msg_transfer/logic/online_history_msg_handler.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/msg_transfer/logic/online_history_msg_handler.go b/internal/msg_transfer/logic/online_history_msg_handler.go index 7e4026088..f714eaa5c 100644 --- a/internal/msg_transfer/logic/online_history_msg_handler.go +++ b/internal/msg_transfer/logic/online_history_msg_handler.go @@ -388,6 +388,9 @@ func (OnlineHistoryConsumerHandler) Cleanup(_ sarama.ConsumerGroupSession) error func (och *OnlineHistoryConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { // a instance in the consumer group + if sess == nil { + time.Sleep(100 * time.Millisecond) + } log.NewDebug("", "online new session msg come", claim.HighWaterMarkOffset(), claim.Topic(), claim.Partition()) cMsg := make([]*sarama.ConsumerMessage, 0, 1000) t := time.NewTicker(time.Duration(100) * time.Millisecond) From 17b4a0525ddbfed9f7471e5946db199b35d72273 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 21 May 2022 10:59:44 +0800 Subject: [PATCH 53/76] log --- internal/msg_gateway/gate/logic.go | 1 + internal/msg_transfer/logic/online_history_msg_handler.go | 1 + internal/rpc/msg/send_msg.go | 2 ++ pkg/common/db/batch_insert_chat.go | 2 ++ 4 files changed, 6 insertions(+) diff --git a/internal/msg_gateway/gate/logic.go b/internal/msg_gateway/gate/logic.go index 5644f1cb3..884f5f842 100644 --- a/internal/msg_gateway/gate/logic.go +++ b/internal/msg_gateway/gate/logic.go @@ -147,6 +147,7 @@ func (ws *WServer) sendMsgReq(conn *UserConn, m *Req) { sendMsgAllCount++ sendMsgAllCountLock.Unlock() log.NewInfo(m.OperationID, "Ws call success to sendMsgReq start", m.MsgIncr, m.ReqIdentifier, m.SendID, m.Data) + nReply := new(pbChat.SendMsgResp) isPass, errCode, errMsg, pData := ws.argsValidate(m, constant.WSSendMsg) if isPass { diff --git a/internal/msg_transfer/logic/online_history_msg_handler.go b/internal/msg_transfer/logic/online_history_msg_handler.go index c20422551..7e5053df1 100644 --- a/internal/msg_transfer/logic/online_history_msg_handler.go +++ b/internal/msg_transfer/logic/online_history_msg_handler.go @@ -105,6 +105,7 @@ func (och *OnlineHistoryConsumerHandler) Run(channelID int) { isSenderSync := utils.GetSwitchFromOptions(v.MsgData.Options, constant.IsSenderSync) if isHistory { storageMsgList = append(storageMsgList, v) + log.NewWarn(triggerID, "storageMsgList to mongodb client msgID: ", v.MsgData.ClientMsgID) } if !(!isSenderSync && msgChannelValue.userID == v.MsgData.SendID) { pushMsgList = append(pushMsgList, v) diff --git a/internal/rpc/msg/send_msg.go b/internal/rpc/msg/send_msg.go index 16e4c5ab6..e04610c7b 100644 --- a/internal/rpc/msg/send_msg.go +++ b/internal/rpc/msg/send_msg.go @@ -372,6 +372,8 @@ func (rpc *rpcChat) sendMsgToKafka(m *pbChat.MsgDataToMQ, key string, status str pid, offset, err := rpc.onlineProducer.SendMessage(m, key) if err != nil { log.Error(m.OperationID, "kafka send failed", "send data", m.String(), "pid", pid, "offset", offset, "err", err.Error(), "key", key, status) + } else { + log.NewWarn(m.OperationID, "sendMsgToKafka client msgID ", m.MsgData.ClientMsgID) } return err case constant.OfflineStatus: diff --git a/pkg/common/db/batch_insert_chat.go b/pkg/common/db/batch_insert_chat.go index 0354082d0..b5fca3502 100644 --- a/pkg/common/db/batch_insert_chat.go +++ b/pkg/common/db/batch_insert_chat.go @@ -48,6 +48,8 @@ func (d *DataBases) BatchInsertChat(userID string, msgList []*pbMsg.MsgDataToMQ, seqUid := "" seqUidNext := "" log.Debug(operationID, "remain ", remain, "insertCounter ", insertCounter, "currentMaxSeq ", currentMaxSeq, userID, len(msgList)) + //4998 remain ==1 + //4999 for _, m := range msgList { log.Debug(operationID, "msg node ", m.String(), m.MsgData.ClientMsgID) currentMaxSeq++ From 7d1843f48513b246fcf4e081a5e3057ccaad4704 Mon Sep 17 00:00:00 2001 From: Gordon <1432970085@qq.com> Date: Sat, 21 May 2022 14:19:45 +0800 Subject: [PATCH 54/76] log change --- .../msg_transfer/logic/online_history_msg_handler.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/internal/msg_transfer/logic/online_history_msg_handler.go b/internal/msg_transfer/logic/online_history_msg_handler.go index 007896bd4..905d39b10 100644 --- a/internal/msg_transfer/logic/online_history_msg_handler.go +++ b/internal/msg_transfer/logic/online_history_msg_handler.go @@ -129,9 +129,15 @@ func (och *OnlineHistoryConsumerHandler) Run(channelID int) { singleMsgSuccessCountMutex.Lock() singleMsgSuccessCount += uint64(len(storageMsgList)) singleMsgSuccessCountMutex.Unlock() - for _, v := range pushMsgList { - sendMessageToPush(v, msgChannelValue.userID) - } + //go func(push,storage []*pbMsg.MsgDataToMQ) { + // for _, v := range storage { + // sendMessageToPush(v, msgChannelValue.userID) + // } + // for _, x := range utils.DifferenceString() { + // sendMessageToPush(v, msgChannelValue.userID) + // } + // + //}(pushMsgList,storageMsgList) } } From 5b9e5f904b1a277f3e22f98eb94fa41ebec10568 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 21 May 2022 15:01:01 +0800 Subject: [PATCH 55/76] log --- .../logic/online_history_msg_handler.go | 13 ++++++++++--- internal/rpc/msg/send_msg.go | 2 +- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/internal/msg_transfer/logic/online_history_msg_handler.go b/internal/msg_transfer/logic/online_history_msg_handler.go index 007896bd4..4f0c9ba8f 100644 --- a/internal/msg_transfer/logic/online_history_msg_handler.go +++ b/internal/msg_transfer/logic/online_history_msg_handler.go @@ -105,7 +105,7 @@ func (och *OnlineHistoryConsumerHandler) Run(channelID int) { isSenderSync := utils.GetSwitchFromOptions(v.MsgData.Options, constant.IsSenderSync) if isHistory { storageMsgList = append(storageMsgList, v) - log.NewWarn(triggerID, "storageMsgList to mongodb client msgID: ", v.MsgData.ClientMsgID) + // log.NewWarn(triggerID, "storageMsgList to mongodb client msgID: ", v.MsgData.ClientMsgID) } if !(!isSenderSync && msgChannelValue.userID == v.MsgData.SendID) { pushMsgList = append(pushMsgList, v) @@ -389,9 +389,16 @@ func (OnlineHistoryConsumerHandler) Cleanup(_ sarama.ConsumerGroupSession) error func (och *OnlineHistoryConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { // a instance in the consumer group - if sess == nil { - time.Sleep(100 * time.Millisecond) + for { + if sess == nil { + log.Error("sess == nil") + time.Sleep(100 * time.Millisecond) + } else { + log.NewWarn("sess == ", sess) + break + } } + log.NewDebug("", "online new session msg come", claim.HighWaterMarkOffset(), claim.Topic(), claim.Partition()) cMsg := make([]*sarama.ConsumerMessage, 0, 1000) t := time.NewTicker(time.Duration(100) * time.Millisecond) diff --git a/internal/rpc/msg/send_msg.go b/internal/rpc/msg/send_msg.go index e04610c7b..587f56998 100644 --- a/internal/rpc/msg/send_msg.go +++ b/internal/rpc/msg/send_msg.go @@ -373,7 +373,7 @@ func (rpc *rpcChat) sendMsgToKafka(m *pbChat.MsgDataToMQ, key string, status str if err != nil { log.Error(m.OperationID, "kafka send failed", "send data", m.String(), "pid", pid, "offset", offset, "err", err.Error(), "key", key, status) } else { - log.NewWarn(m.OperationID, "sendMsgToKafka client msgID ", m.MsgData.ClientMsgID) + // log.NewWarn(m.OperationID, "sendMsgToKafka client msgID ", m.MsgData.ClientMsgID) } return err case constant.OfflineStatus: From e2eb546069e431831151cd1c0b95bf1a1e5073ba Mon Sep 17 00:00:00 2001 From: Gordon <1432970085@qq.com> Date: Sat, 21 May 2022 15:03:26 +0800 Subject: [PATCH 56/76] log change --- .../logic/online_history_msg_handler.go | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/internal/msg_transfer/logic/online_history_msg_handler.go b/internal/msg_transfer/logic/online_history_msg_handler.go index 905d39b10..8568f32b4 100644 --- a/internal/msg_transfer/logic/online_history_msg_handler.go +++ b/internal/msg_transfer/logic/online_history_msg_handler.go @@ -97,7 +97,7 @@ func (och *OnlineHistoryConsumerHandler) Run(channelID int) { msgList := msgChannelValue.msgList triggerID := msgChannelValue.triggerID storageMsgList := make([]*pbMsg.MsgDataToMQ, 0, 80) - pushMsgList := make([]*pbMsg.MsgDataToMQ, 0, 80) + noStoragepushMsgList := make([]*pbMsg.MsgDataToMQ, 0, 80) log.Debug(triggerID, "msg arrived channel", "channel id", channelID, msgList, msgChannelValue.userID, len(msgList)) for _, v := range msgList { log.Debug(triggerID, "msg come to storage center", v.String()) @@ -106,10 +106,12 @@ func (och *OnlineHistoryConsumerHandler) Run(channelID int) { if isHistory { storageMsgList = append(storageMsgList, v) log.NewWarn(triggerID, "storageMsgList to mongodb client msgID: ", v.MsgData.ClientMsgID) + } else { + if !(!isSenderSync && msgChannelValue.userID == v.MsgData.SendID) { + noStoragepushMsgList = append(noStoragepushMsgList, v) + } } - if !(!isSenderSync && msgChannelValue.userID == v.MsgData.SendID) { - pushMsgList = append(pushMsgList, v) - } + } //switch msgChannelValue.msg.MsgData.SessionType { @@ -120,7 +122,7 @@ func (och *OnlineHistoryConsumerHandler) Run(channelID int) { // log.NewError(msgFromMQ.OperationID, "SessionType error", msgFromMQ.String()) // return //} - log.Debug(triggerID, "msg storage length", len(storageMsgList), "push length", len(pushMsgList)) + log.Debug(triggerID, "msg storage length", len(storageMsgList), "push length", len(noStoragepushMsgList)) err := saveUserChatList(msgChannelValue.userID, storageMsgList, triggerID) if err != nil { singleMsgFailedCount += uint64(len(storageMsgList)) @@ -129,15 +131,15 @@ func (och *OnlineHistoryConsumerHandler) Run(channelID int) { singleMsgSuccessCountMutex.Lock() singleMsgSuccessCount += uint64(len(storageMsgList)) singleMsgSuccessCountMutex.Unlock() - //go func(push,storage []*pbMsg.MsgDataToMQ) { - // for _, v := range storage { - // sendMessageToPush(v, msgChannelValue.userID) - // } - // for _, x := range utils.DifferenceString() { - // sendMessageToPush(v, msgChannelValue.userID) - // } - // - //}(pushMsgList,storageMsgList) + go func(push, storage []*pbMsg.MsgDataToMQ) { + for _, v := range storage { + sendMessageToPush(v, msgChannelValue.userID) + } + for _, x := range push { + sendMessageToPush(x, msgChannelValue.userID) + } + + }(noStoragepushMsgList, storageMsgList) } } From 31e0a8a88b57a0e54c1675deb804a5c6cd6e10ac Mon Sep 17 00:00:00 2001 From: Gordon <1432970085@qq.com> Date: Sat, 21 May 2022 15:04:42 +0800 Subject: [PATCH 57/76] log change --- internal/msg_transfer/logic/online_history_msg_handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/msg_transfer/logic/online_history_msg_handler.go b/internal/msg_transfer/logic/online_history_msg_handler.go index 8568f32b4..67e299dcc 100644 --- a/internal/msg_transfer/logic/online_history_msg_handler.go +++ b/internal/msg_transfer/logic/online_history_msg_handler.go @@ -105,7 +105,7 @@ func (och *OnlineHistoryConsumerHandler) Run(channelID int) { isSenderSync := utils.GetSwitchFromOptions(v.MsgData.Options, constant.IsSenderSync) if isHistory { storageMsgList = append(storageMsgList, v) - log.NewWarn(triggerID, "storageMsgList to mongodb client msgID: ", v.MsgData.ClientMsgID) + //log.NewWarn(triggerID, "storageMsgList to mongodb client msgID: ", v.MsgData.ClientMsgID) } else { if !(!isSenderSync && msgChannelValue.userID == v.MsgData.SendID) { noStoragepushMsgList = append(noStoragepushMsgList, v) From b864f9c6a9c4c7bad2610661a38c5de62117a0b6 Mon Sep 17 00:00:00 2001 From: Gordon <1432970085@qq.com> Date: Sat, 21 May 2022 19:17:31 +0800 Subject: [PATCH 58/76] message --- internal/msg_transfer/logic/init.go | 4 ++-- internal/rpc/msg/send_msg.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/msg_transfer/logic/init.go b/internal/msg_transfer/logic/init.go index d36a3d8d5..53fcea474 100644 --- a/internal/msg_transfer/logic/init.go +++ b/internal/msg_transfer/logic/init.go @@ -15,7 +15,7 @@ const OnlineTopicVacancy = 0 const Msg = 2 const ConsumerMsgs = 3 const UserMessages = 4 -const ChannelNum = 11 +const ChannelNum = 100 var ( persistentCH PersistentConsumerHandler @@ -52,7 +52,7 @@ func Run() { fmt.Println("not start mysql consumer") } go historyCH.historyConsumerGroup.RegisterHandleAndConsumer(&historyCH) - go offlineHistoryCH.historyConsumerGroup.RegisterHandleAndConsumer(&offlineHistoryCH) + //go offlineHistoryCH.historyConsumerGroup.RegisterHandleAndConsumer(&offlineHistoryCH) } func SetOnlineTopicStatus(status int) { w.Lock() diff --git a/internal/rpc/msg/send_msg.go b/internal/rpc/msg/send_msg.go index 587f56998..7e4ab3953 100644 --- a/internal/rpc/msg/send_msg.go +++ b/internal/rpc/msg/send_msg.go @@ -377,7 +377,7 @@ func (rpc *rpcChat) sendMsgToKafka(m *pbChat.MsgDataToMQ, key string, status str } return err case constant.OfflineStatus: - pid, offset, err := rpc.offlineProducer.SendMessage(m, key) + pid, offset, err := rpc.onlineProducer.SendMessage(m, key) if err != nil { log.Error(m.OperationID, "kafka send failed", "send data", m.String(), "pid", pid, "offset", offset, "err", err.Error(), "key", key, status) } From 821ad4de2ec3d6665c65232808552f8b9762645d Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sun, 22 May 2022 15:09:21 +0800 Subject: [PATCH 59/76] log --- internal/rpc/msg/send_msg.go | 5 ++++- pkg/common/db/mongoModel.go | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/internal/rpc/msg/send_msg.go b/internal/rpc/msg/send_msg.go index 587f56998..a31c03611 100644 --- a/internal/rpc/msg/send_msg.go +++ b/internal/rpc/msg/send_msg.go @@ -142,7 +142,8 @@ func (rpc *rpcChat) encapsulateMsgData(msg *sdk_ws.MsgData) { } func (rpc *rpcChat) SendMsg(_ context.Context, pb *pbChat.SendMsgReq) (*pbChat.SendMsgResp, error) { replay := pbChat.SendMsgResp{} - log.NewDebug(pb.OperationID, "rpc sendMsg come here", pb.String()) + newTime := db.GetCurrentTimestampByMill() + log.NewWarn(pb.OperationID, "rpc sendMsg come here", pb.String(), pb.MsgData.ClientMsgID) flag, errCode, errMsg := userRelationshipVerification(pb) if !flag { return returnMsg(&replay, pb, errCode, errMsg, "", 0) @@ -360,6 +361,8 @@ func (rpc *rpcChat) SendMsg(_ context.Context, pb *pbChat.SendMsgReq) (*pbChat.S return returnMsg(&replay, pb, 201, "kafka send msg err", "", 0) } } + + log.NewWarn(pb.OperationID, "send msg cost time ", db.GetCurrentTimestampByMill()-newTime, pb.MsgData.ClientMsgID) return returnMsg(&replay, pb, 0, "", msgToMQSingle.MsgData.ServerMsgID, msgToMQSingle.MsgData.SendTime) default: return returnMsg(&replay, pb, 203, "unkonwn sessionType", "", 0) diff --git a/pkg/common/db/mongoModel.go b/pkg/common/db/mongoModel.go index 96206c6f2..80e9d3400 100644 --- a/pkg/common/db/mongoModel.go +++ b/pkg/common/db/mongoModel.go @@ -884,6 +884,9 @@ func generateWorkMomentCommentID(workMomentID string) string { func getCurrentTimestampByMill() int64 { return time.Now().UnixNano() / 1e6 } +func GetCurrentTimestampByMill() int64 { + return time.Now().UnixNano() / 1e6 +} func getSeqUid(uid string, seq uint32) string { seqSuffix := seq / singleGocMsgNum From 9f295d8399165e76cf0baa68aaeb4f5f23a2f42a Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sun, 22 May 2022 15:15:55 +0800 Subject: [PATCH 60/76] log --- internal/rpc/msg/send_msg.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/rpc/msg/send_msg.go b/internal/rpc/msg/send_msg.go index 7e0a2227a..4367a6eb7 100644 --- a/internal/rpc/msg/send_msg.go +++ b/internal/rpc/msg/send_msg.go @@ -342,6 +342,7 @@ func (rpc *rpcChat) SendMsg(_ context.Context, pb *pbChat.SendMsgReq) (*pbChat.S } }() } + log.NewWarn(pb.OperationID, "send msg cost time ", db.GetCurrentTimestampByMill()-newTime, pb.MsgData.ClientMsgID) return returnMsg(&replay, pb, 0, "", msgToMQSingle.MsgData.ServerMsgID, msgToMQSingle.MsgData.SendTime) } From 2fc6b0a8cb34bbbe60667c775d81393915776c55 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sun, 22 May 2022 15:57:02 +0800 Subject: [PATCH 61/76] log --- internal/rpc/msg/send_msg.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/internal/rpc/msg/send_msg.go b/internal/rpc/msg/send_msg.go index 4367a6eb7..2df926486 100644 --- a/internal/rpc/msg/send_msg.go +++ b/internal/rpc/msg/send_msg.go @@ -266,6 +266,9 @@ func (rpc *rpcChat) SendMsg(_ context.Context, pb *pbChat.SendMsgReq) (*pbChat.S m[constant.OnlineStatus] = memberUserIDList } + log.NewWarn(pb.OperationID, "send msg cost time1 ", db.GetCurrentTimestampByMill()-newTime, pb.MsgData.ClientMsgID) + newTime = db.GetCurrentTimestampByMill() + //split parallel send var wg sync.WaitGroup var sendTag bool @@ -284,6 +287,8 @@ func (rpc *rpcChat) SendMsg(_ context.Context, pb *pbChat.SendMsgReq) (*pbChat.S wg.Add(1) go rpc.sendMsgToGroup(addUidList, *pb, constant.OnlineStatus, &sendTag, &wg) wg.Wait() + log.NewWarn(pb.OperationID, "send msg cost time2 ", db.GetCurrentTimestampByMill()-newTime, pb.MsgData.ClientMsgID) + newTime = db.GetCurrentTimestampByMill() // callback if err := callbackAfterSendGroupMsg(pb); err != nil { log.NewError(pb.OperationID, utils.GetSelfFuncName(), "callbackAfterSendGroupMsg failed", err.Error()) @@ -342,7 +347,7 @@ func (rpc *rpcChat) SendMsg(_ context.Context, pb *pbChat.SendMsgReq) (*pbChat.S } }() } - log.NewWarn(pb.OperationID, "send msg cost time ", db.GetCurrentTimestampByMill()-newTime, pb.MsgData.ClientMsgID) + log.NewWarn(pb.OperationID, "send msg cost time3 ", db.GetCurrentTimestampByMill()-newTime, pb.MsgData.ClientMsgID) return returnMsg(&replay, pb, 0, "", msgToMQSingle.MsgData.ServerMsgID, msgToMQSingle.MsgData.SendTime) } From d276be9594e5af47f6b888c9407a02638e3f3636 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sun, 22 May 2022 18:28:51 +0800 Subject: [PATCH 62/76] Reduce memory copies --- internal/rpc/msg/send_msg.go | 67 ++++++++++++++++++++++++++++++++++-- pkg/utils/map.go | 1 + 2 files changed, 66 insertions(+), 2 deletions(-) diff --git a/internal/rpc/msg/send_msg.go b/internal/rpc/msg/send_msg.go index 2df926486..cbb301956 100644 --- a/internal/rpc/msg/send_msg.go +++ b/internal/rpc/msg/send_msg.go @@ -277,11 +277,15 @@ func (rpc *rpcChat) SendMsg(_ context.Context, pb *pbChat.SendMsgReq) (*pbChat.S remain := len(v) % split for i := 0; i < len(v)/split; i++ { wg.Add(1) - go rpc.sendMsgToGroup(v[i*split:(i+1)*split], *pb, k, &sendTag, &wg) + tmp := valueCopy(pb) + // go rpc.sendMsgToGroup(v[i*split:(i+1)*split], *pb, k, &sendTag, &wg) + go rpc.sendMsgToGroupOptimization(v[i*split:(i+1)*split], tmp, k, &sendTag, &wg) } if remain > 0 { wg.Add(1) - go rpc.sendMsgToGroup(v[split*(len(v)/split):], *pb, k, &sendTag, &wg) + tmp := valueCopy(pb) + // go rpc.sendMsgToGroup(v[split*(len(v)/split):], *pb, k, &sendTag, &wg) + go rpc.sendMsgToGroupOptimization(v[split*(len(v)/split):], tmp, k, &sendTag, &wg) } } wg.Add(1) @@ -431,6 +435,29 @@ func modifyMessageByUserMessageReceiveOpt(userID, sourceID string, sessionType i return true } +func modifyMessageByUserMessageReceiveOptoptimization(userID, sourceID string, sessionType int, operationID string, options *map[string]bool) bool { + conversationID := utils.GetConversationIDBySessionType(sourceID, sessionType) + opt, err := db.DB.GetSingleConversationRecvMsgOpt(userID, conversationID) + if err != nil && err != redis.ErrNil { + log.NewError(operationID, "GetSingleConversationMsgOpt from redis err", userID, conversationID, err.Error()) + return true + } + + switch opt { + case constant.ReceiveMessage: + return true + case constant.NotReceiveMessage: + return false + case constant.ReceiveNotNotifyMessage: + if *options == nil { + *options = make(map[string]bool, 10) + } + utils.SetSwitchFromOptions(*options, constant.IsOfflinePush, false) + return true + } + return true +} + type NotificationMsg struct { SendID string RecvID string @@ -748,6 +775,23 @@ func getOnlineAndOfflineUserIDList(memberList []string, m map[string][]string, o m[constant.OfflineStatus] = offlUserIDList } +func valueCopy(pb *pbChat.SendMsgReq) *pbChat.SendMsgReq { + offlinePushInfo := sdk_ws.OfflinePushInfo{} + if pb.MsgData.OfflinePushInfo != nil { + offlinePushInfo = *pb.MsgData.OfflinePushInfo + } + msgData := sdk_ws.MsgData{} + msgData = *pb.MsgData + msgData.OfflinePushInfo = &offlinePushInfo + + options := make(map[string]bool, 10) + for key, value := range pb.MsgData.Options { + options[key] = value + } + msgData.Options = options + return &pbChat.SendMsgReq{Token: "", OperationID: pb.OperationID, MsgData: &msgData} +} + func (rpc *rpcChat) sendMsgToGroup(list []string, pb pbChat.SendMsgReq, status string, sendTag *bool, wg *sync.WaitGroup) { // log.Debug(pb.OperationID, "split userID ", list) offlinePushInfo := sdk_ws.OfflinePushInfo{} @@ -783,3 +827,22 @@ func (rpc *rpcChat) sendMsgToGroup(list []string, pb pbChat.SendMsgReq, status s } wg.Done() } + +func (rpc *rpcChat) sendMsgToGroupOptimization(list []string, groupPB *pbChat.SendMsgReq, status string, sendTag *bool, wg *sync.WaitGroup) { + msgToMQGroup := pbChat.MsgDataToMQ{Token: groupPB.Token, OperationID: groupPB.OperationID, MsgData: groupPB.MsgData} + for _, v := range list { + groupPB.MsgData.RecvID = v + isSend := modifyMessageByUserMessageReceiveOpt(v, groupPB.MsgData.GroupID, constant.GroupChatType, groupPB) + if isSend { + err := rpc.sendMsgToKafka(&msgToMQGroup, v, status) + if err != nil { + log.NewError(msgToMQGroup.OperationID, "kafka send msg err:UserId", v, msgToMQGroup.String()) + } else { + *sendTag = true + } + } else { + log.Debug(groupPB.OperationID, "not sendMsgToKafka, ", v) + } + } + wg.Done() +} diff --git a/pkg/utils/map.go b/pkg/utils/map.go index 66ca27471..7a5fb2d6b 100644 --- a/pkg/utils/map.go +++ b/pkg/utils/map.go @@ -124,6 +124,7 @@ func GetSwitchFromOptions(Options map[string]bool, key string) (result bool) { } return false } + func SetSwitchFromOptions(options map[string]bool, key string, value bool) { if options == nil { options = make(map[string]bool, 5) From 883e7f96ee0435232e00ee9cc90d90ef119ddd53 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sun, 22 May 2022 18:42:08 +0800 Subject: [PATCH 63/76] log --- internal/rpc/msg/send_msg.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/rpc/msg/send_msg.go b/internal/rpc/msg/send_msg.go index cbb301956..77d6bf5b5 100644 --- a/internal/rpc/msg/send_msg.go +++ b/internal/rpc/msg/send_msg.go @@ -288,6 +288,7 @@ func (rpc *rpcChat) SendMsg(_ context.Context, pb *pbChat.SendMsgReq) (*pbChat.S go rpc.sendMsgToGroupOptimization(v[split*(len(v)/split):], tmp, k, &sendTag, &wg) } } + log.NewWarn(pb.OperationID, "send msg cost time22 ", db.GetCurrentTimestampByMill()-newTime, pb.MsgData.ClientMsgID, "uidList : ", len(addUidList)) wg.Add(1) go rpc.sendMsgToGroup(addUidList, *pb, constant.OnlineStatus, &sendTag, &wg) wg.Wait() From 04a9788bc0712e953da5a497933f52dd12a10b23 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sun, 22 May 2022 18:47:16 +0800 Subject: [PATCH 64/76] log --- internal/rpc/msg/send_msg.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/rpc/msg/send_msg.go b/internal/rpc/msg/send_msg.go index 77d6bf5b5..09509f3f7 100644 --- a/internal/rpc/msg/send_msg.go +++ b/internal/rpc/msg/send_msg.go @@ -272,7 +272,7 @@ func (rpc *rpcChat) SendMsg(_ context.Context, pb *pbChat.SendMsgReq) (*pbChat.S //split parallel send var wg sync.WaitGroup var sendTag bool - var split = 50 + var split = 20 for k, v := range m { remain := len(v) % split for i := 0; i < len(v)/split; i++ { From 8c119a76d1943f2ea6f60915d7c716ef2403fac7 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sun, 22 May 2022 21:08:57 +0800 Subject: [PATCH 65/76] sess == nil --- .../msg_transfer/logic/online_history_msg_handler.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/internal/msg_transfer/logic/online_history_msg_handler.go b/internal/msg_transfer/logic/online_history_msg_handler.go index 67e299dcc..040096529 100644 --- a/internal/msg_transfer/logic/online_history_msg_handler.go +++ b/internal/msg_transfer/logic/online_history_msg_handler.go @@ -397,9 +397,15 @@ func (OnlineHistoryConsumerHandler) Cleanup(_ sarama.ConsumerGroupSession) error func (och *OnlineHistoryConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { // a instance in the consumer group - if sess == nil { - time.Sleep(100 * time.Millisecond) + for { + if sess == nil { + log.NewWarn("", " sess == nil, waiting ") + time.Sleep(100 * time.Millisecond) + } else { + break + } } + log.NewDebug("", "online new session msg come", claim.HighWaterMarkOffset(), claim.Topic(), claim.Partition()) cMsg := make([]*sarama.ConsumerMessage, 0, 1000) t := time.NewTicker(time.Duration(100) * time.Millisecond) From b09e2c11eb54e69c14dd31319d1a4a7497945158 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sun, 22 May 2022 21:10:18 +0800 Subject: [PATCH 66/76] sess == nil --- internal/msg_transfer/logic/online_history_msg_handler.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/msg_transfer/logic/online_history_msg_handler.go b/internal/msg_transfer/logic/online_history_msg_handler.go index 040096529..7039b3079 100644 --- a/internal/msg_transfer/logic/online_history_msg_handler.go +++ b/internal/msg_transfer/logic/online_history_msg_handler.go @@ -419,7 +419,6 @@ func (och *OnlineHistoryConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupS ccMsg := make([]*sarama.ConsumerMessage, 0, 1000) for _, v := range cMsg { ccMsg = append(ccMsg, v) - } triggerID = utils.OperationIDGenerator() log.Debug(triggerID, "length trigger msg consumer start", len(ccMsg)) From 5c0b06f27d6cf9bd61c21c8ac08c244cc7f0208b Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sun, 22 May 2022 21:27:29 +0800 Subject: [PATCH 67/76] log --- internal/msg_transfer/logic/online_history_msg_handler.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/msg_transfer/logic/online_history_msg_handler.go b/internal/msg_transfer/logic/online_history_msg_handler.go index 7039b3079..12b02b50b 100644 --- a/internal/msg_transfer/logic/online_history_msg_handler.go +++ b/internal/msg_transfer/logic/online_history_msg_handler.go @@ -397,6 +397,7 @@ func (OnlineHistoryConsumerHandler) Cleanup(_ sarama.ConsumerGroupSession) error func (och *OnlineHistoryConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { // a instance in the consumer group + for { if sess == nil { log.NewWarn("", " sess == nil, waiting ") @@ -414,6 +415,7 @@ func (och *OnlineHistoryConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupS //och.TriggerCmd(OnlineTopicBusy) select { case msg := <-claim.Messages(): + log.NewDebug("", "claim.Messages ", msg, *msg) cMsg = append(cMsg, msg) if len(cMsg) >= 1000 { ccMsg := make([]*sarama.ConsumerMessage, 0, 1000) From 64d83d90487499836c378f2057611ecbafae2dc6 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sun, 22 May 2022 21:31:10 +0800 Subject: [PATCH 68/76] log --- internal/msg_transfer/logic/online_history_msg_handler.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/msg_transfer/logic/online_history_msg_handler.go b/internal/msg_transfer/logic/online_history_msg_handler.go index 12b02b50b..e41c61611 100644 --- a/internal/msg_transfer/logic/online_history_msg_handler.go +++ b/internal/msg_transfer/logic/online_history_msg_handler.go @@ -415,7 +415,11 @@ func (och *OnlineHistoryConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupS //och.TriggerCmd(OnlineTopicBusy) select { case msg := <-claim.Messages(): - log.NewDebug("", "claim.Messages ", msg, *msg) + if msg == nil { + log.NewDebug("", "claim.Messages ", msg) + continue + } + cMsg = append(cMsg, msg) if len(cMsg) >= 1000 { ccMsg := make([]*sarama.ConsumerMessage, 0, 1000) From 24073b20ab6a055b4847fb0749aeb57cf93ff9db Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sun, 22 May 2022 21:37:38 +0800 Subject: [PATCH 69/76] log --- internal/msg_transfer/logic/online_history_msg_handler.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/internal/msg_transfer/logic/online_history_msg_handler.go b/internal/msg_transfer/logic/online_history_msg_handler.go index e41c61611..12b02b50b 100644 --- a/internal/msg_transfer/logic/online_history_msg_handler.go +++ b/internal/msg_transfer/logic/online_history_msg_handler.go @@ -415,11 +415,7 @@ func (och *OnlineHistoryConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupS //och.TriggerCmd(OnlineTopicBusy) select { case msg := <-claim.Messages(): - if msg == nil { - log.NewDebug("", "claim.Messages ", msg) - continue - } - + log.NewDebug("", "claim.Messages ", msg, *msg) cMsg = append(cMsg, msg) if len(cMsg) >= 1000 { ccMsg := make([]*sarama.ConsumerMessage, 0, 1000) From bef0e40db8756bcff497f855e7e93db200aac7f3 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sun, 22 May 2022 21:48:01 +0800 Subject: [PATCH 70/76] log --- internal/msg_transfer/logic/online_history_msg_handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/msg_transfer/logic/online_history_msg_handler.go b/internal/msg_transfer/logic/online_history_msg_handler.go index 12b02b50b..8fee2a69f 100644 --- a/internal/msg_transfer/logic/online_history_msg_handler.go +++ b/internal/msg_transfer/logic/online_history_msg_handler.go @@ -415,7 +415,7 @@ func (och *OnlineHistoryConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupS //och.TriggerCmd(OnlineTopicBusy) select { case msg := <-claim.Messages(): - log.NewDebug("", "claim.Messages ", msg, *msg) + log.NewDebug("", "claim.Messages ", msg) cMsg = append(cMsg, msg) if len(cMsg) >= 1000 { ccMsg := make([]*sarama.ConsumerMessage, 0, 1000) From f5c7197dda3df9faa3bd7160a60629c881d10b79 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Mon, 23 May 2022 08:42:50 +0800 Subject: [PATCH 71/76] log --- internal/rpc/msg/send_msg.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/rpc/msg/send_msg.go b/internal/rpc/msg/send_msg.go index 09509f3f7..f7bb18db3 100644 --- a/internal/rpc/msg/send_msg.go +++ b/internal/rpc/msg/send_msg.go @@ -266,7 +266,7 @@ func (rpc *rpcChat) SendMsg(_ context.Context, pb *pbChat.SendMsgReq) (*pbChat.S m[constant.OnlineStatus] = memberUserIDList } - log.NewWarn(pb.OperationID, "send msg cost time1 ", db.GetCurrentTimestampByMill()-newTime, pb.MsgData.ClientMsgID) + log.Debug(pb.OperationID, "send msg cost time1 ", db.GetCurrentTimestampByMill()-newTime, pb.MsgData.ClientMsgID) newTime = db.GetCurrentTimestampByMill() //split parallel send @@ -288,11 +288,11 @@ func (rpc *rpcChat) SendMsg(_ context.Context, pb *pbChat.SendMsgReq) (*pbChat.S go rpc.sendMsgToGroupOptimization(v[split*(len(v)/split):], tmp, k, &sendTag, &wg) } } - log.NewWarn(pb.OperationID, "send msg cost time22 ", db.GetCurrentTimestampByMill()-newTime, pb.MsgData.ClientMsgID, "uidList : ", len(addUidList)) + log.Debug(pb.OperationID, "send msg cost time22 ", db.GetCurrentTimestampByMill()-newTime, pb.MsgData.ClientMsgID, "uidList : ", len(addUidList)) wg.Add(1) go rpc.sendMsgToGroup(addUidList, *pb, constant.OnlineStatus, &sendTag, &wg) wg.Wait() - log.NewWarn(pb.OperationID, "send msg cost time2 ", db.GetCurrentTimestampByMill()-newTime, pb.MsgData.ClientMsgID) + log.Debug(pb.OperationID, "send msg cost time2 ", db.GetCurrentTimestampByMill()-newTime, pb.MsgData.ClientMsgID) newTime = db.GetCurrentTimestampByMill() // callback if err := callbackAfterSendGroupMsg(pb); err != nil { @@ -352,7 +352,7 @@ func (rpc *rpcChat) SendMsg(_ context.Context, pb *pbChat.SendMsgReq) (*pbChat.S } }() } - log.NewWarn(pb.OperationID, "send msg cost time3 ", db.GetCurrentTimestampByMill()-newTime, pb.MsgData.ClientMsgID) + log.Debug(pb.OperationID, "send msg cost time3 ", db.GetCurrentTimestampByMill()-newTime, pb.MsgData.ClientMsgID) return returnMsg(&replay, pb, 0, "", msgToMQSingle.MsgData.ServerMsgID, msgToMQSingle.MsgData.SendTime) } @@ -373,7 +373,7 @@ func (rpc *rpcChat) SendMsg(_ context.Context, pb *pbChat.SendMsgReq) (*pbChat.S } } - log.NewWarn(pb.OperationID, "send msg cost time ", db.GetCurrentTimestampByMill()-newTime, pb.MsgData.ClientMsgID) + log.Debug(pb.OperationID, "send msg cost time ", db.GetCurrentTimestampByMill()-newTime, pb.MsgData.ClientMsgID) return returnMsg(&replay, pb, 0, "", msgToMQSingle.MsgData.ServerMsgID, msgToMQSingle.MsgData.SendTime) default: return returnMsg(&replay, pb, 203, "unkonwn sessionType", "", 0) From 1f20024b91fd0d5181923d81dc38e5867958f827 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Mon, 23 May 2022 08:44:20 +0800 Subject: [PATCH 72/76] log --- internal/rpc/msg/send_msg.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/rpc/msg/send_msg.go b/internal/rpc/msg/send_msg.go index f7bb18db3..b52e02aba 100644 --- a/internal/rpc/msg/send_msg.go +++ b/internal/rpc/msg/send_msg.go @@ -376,7 +376,7 @@ func (rpc *rpcChat) SendMsg(_ context.Context, pb *pbChat.SendMsgReq) (*pbChat.S log.Debug(pb.OperationID, "send msg cost time ", db.GetCurrentTimestampByMill()-newTime, pb.MsgData.ClientMsgID) return returnMsg(&replay, pb, 0, "", msgToMQSingle.MsgData.ServerMsgID, msgToMQSingle.MsgData.SendTime) default: - return returnMsg(&replay, pb, 203, "unkonwn sessionType", "", 0) + return returnMsg(&replay, pb, 203, "unknown sessionType", "", 0) } } From f0dadcef7039e16ffa08fded19ad2c08cfc01352 Mon Sep 17 00:00:00 2001 From: Gordon <1432970085@qq.com> Date: Mon, 23 May 2022 09:41:34 +0800 Subject: [PATCH 73/76] message handle --- .../logic/online_history_msg_handler.go | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/internal/msg_transfer/logic/online_history_msg_handler.go b/internal/msg_transfer/logic/online_history_msg_handler.go index 8fee2a69f..9ea433faa 100644 --- a/internal/msg_transfer/logic/online_history_msg_handler.go +++ b/internal/msg_transfer/logic/online_history_msg_handler.go @@ -416,19 +416,21 @@ func (och *OnlineHistoryConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupS select { case msg := <-claim.Messages(): log.NewDebug("", "claim.Messages ", msg) - cMsg = append(cMsg, msg) - if len(cMsg) >= 1000 { - ccMsg := make([]*sarama.ConsumerMessage, 0, 1000) - for _, v := range cMsg { - ccMsg = append(ccMsg, v) + if msg != nil { + cMsg = append(cMsg, msg) + if len(cMsg) >= 1000 { + ccMsg := make([]*sarama.ConsumerMessage, 0, 1000) + for _, v := range cMsg { + ccMsg = append(ccMsg, v) + } + triggerID = utils.OperationIDGenerator() + log.Debug(triggerID, "length trigger msg consumer start", len(ccMsg)) + och.msgDistributionCh <- Cmd2Value{Cmd: ConsumerMsgs, Value: TriggerChannelValue{ + triggerID: triggerID, cmsgList: ccMsg}} + sess.MarkMessage(msg, "") + cMsg = make([]*sarama.ConsumerMessage, 0, 1000) + log.Debug(triggerID, "length trigger msg consumer end", len(cMsg)) } - triggerID = utils.OperationIDGenerator() - log.Debug(triggerID, "length trigger msg consumer start", len(ccMsg)) - och.msgDistributionCh <- Cmd2Value{Cmd: ConsumerMsgs, Value: TriggerChannelValue{ - triggerID: triggerID, cmsgList: ccMsg}} - sess.MarkMessage(msg, "") - cMsg = make([]*sarama.ConsumerMessage, 0, 1000) - log.Debug(triggerID, "length trigger msg consumer end", len(cMsg)) } case <-t.C: if len(cMsg) > 0 { From 767131a117fe1d83577a51ffcbac93a12aa4f0b5 Mon Sep 17 00:00:00 2001 From: Gordon <1432970085@qq.com> Date: Mon, 23 May 2022 09:54:43 +0800 Subject: [PATCH 74/76] message handle --- internal/rpc/msg/send_msg.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/rpc/msg/send_msg.go b/internal/rpc/msg/send_msg.go index b52e02aba..4fc66bbaf 100644 --- a/internal/rpc/msg/send_msg.go +++ b/internal/rpc/msg/send_msg.go @@ -790,7 +790,7 @@ func valueCopy(pb *pbChat.SendMsgReq) *pbChat.SendMsgReq { options[key] = value } msgData.Options = options - return &pbChat.SendMsgReq{Token: "", OperationID: pb.OperationID, MsgData: &msgData} + return &pbChat.SendMsgReq{Token: pb.Token, OperationID: pb.OperationID, MsgData: &msgData} } func (rpc *rpcChat) sendMsgToGroup(list []string, pb pbChat.SendMsgReq, status string, sendTag *bool, wg *sync.WaitGroup) { From 50bb13b7cdb99a0c830e920613be5448e4d2fc14 Mon Sep 17 00:00:00 2001 From: Gordon <1432970085@qq.com> Date: Mon, 23 May 2022 10:55:33 +0800 Subject: [PATCH 75/76] message handle --- .../logic/online_history_msg_handler.go | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/internal/msg_transfer/logic/online_history_msg_handler.go b/internal/msg_transfer/logic/online_history_msg_handler.go index 9ea433faa..8e4e3c2a2 100644 --- a/internal/msg_transfer/logic/online_history_msg_handler.go +++ b/internal/msg_transfer/logic/online_history_msg_handler.go @@ -416,22 +416,21 @@ func (och *OnlineHistoryConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupS select { case msg := <-claim.Messages(): log.NewDebug("", "claim.Messages ", msg) - if msg != nil { - cMsg = append(cMsg, msg) - if len(cMsg) >= 1000 { - ccMsg := make([]*sarama.ConsumerMessage, 0, 1000) - for _, v := range cMsg { - ccMsg = append(ccMsg, v) - } - triggerID = utils.OperationIDGenerator() - log.Debug(triggerID, "length trigger msg consumer start", len(ccMsg)) - och.msgDistributionCh <- Cmd2Value{Cmd: ConsumerMsgs, Value: TriggerChannelValue{ - triggerID: triggerID, cmsgList: ccMsg}} - sess.MarkMessage(msg, "") - cMsg = make([]*sarama.ConsumerMessage, 0, 1000) - log.Debug(triggerID, "length trigger msg consumer end", len(cMsg)) + cMsg = append(cMsg, msg) + if len(cMsg) >= 1000 { + ccMsg := make([]*sarama.ConsumerMessage, 0, 1000) + for _, v := range cMsg { + ccMsg = append(ccMsg, v) } + triggerID = utils.OperationIDGenerator() + log.Debug(triggerID, "length trigger msg consumer start", len(ccMsg)) + och.msgDistributionCh <- Cmd2Value{Cmd: ConsumerMsgs, Value: TriggerChannelValue{ + triggerID: triggerID, cmsgList: ccMsg}} + sess.MarkMessage(msg, "") + cMsg = make([]*sarama.ConsumerMessage, 0, 1000) + log.Debug(triggerID, "length trigger msg consumer end", len(cMsg)) } + case <-t.C: if len(cMsg) > 0 { ccMsg := make([]*sarama.ConsumerMessage, 0, 1000) From e385f054df002ab0f6e9de48216aeac4e9a18997 Mon Sep 17 00:00:00 2001 From: Gordon <1432970085@qq.com> Date: Mon, 23 May 2022 11:57:41 +0800 Subject: [PATCH 76/76] message handle --- internal/msg_transfer/logic/init.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/msg_transfer/logic/init.go b/internal/msg_transfer/logic/init.go index 53fcea474..f3f1b138e 100644 --- a/internal/msg_transfer/logic/init.go +++ b/internal/msg_transfer/logic/init.go @@ -39,7 +39,7 @@ func Init() { historyCH.Init(cmdCh) onlineTopicStatus = OnlineTopicVacancy log.Debug("come msg transfer ts", config.Config.Kafka.ConsumerGroupID.MsgToMongoOffline, config.Config.Kafka.Ws2mschatOffline.Topic) - offlineHistoryCH.Init(cmdCh) + //offlineHistoryCH.Init(cmdCh) statistics.NewStatistics(&singleMsgSuccessCount, config.Config.ModuleName.MsgTransferName, fmt.Sprintf("%d second singleMsgCount insert to mongo", constant.StatisticsTimeInterval), constant.StatisticsTimeInterval) statistics.NewStatistics(&groupMsgCount, config.Config.ModuleName.MsgTransferName, fmt.Sprintf("%d second groupMsgCount insert to mongo", constant.StatisticsTimeInterval), constant.StatisticsTimeInterval) producer = kafka.NewKafkaProducer(config.Config.Kafka.Ms2pschat.Addr, config.Config.Kafka.Ms2pschat.Topic)