blackteay/open-im-server
1
0
Fork 0
mirror of https://github.com/openimsdk/open-im-server.git synced 2025-04-24 02:16:16 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'tuoyun' of github.com:OpenIMSDK/Open-IM-Server into tuoyun

Browse Source
This commit is contained in:
wangchuxiao 2022-05-26 10:08:22 +08:00
commit 9beff11125
1 changed files with 59 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

98
internal/msg_transfer/logic/online_history_msg_handler.go
View File

@ -17,6 +17,7 @@ import (
"github.com/golang/protobuf/proto" "github.com/golang/protobuf/proto"
"hash/crc32" "hash/crc32"
"strings" "strings"
"sync"
"time" "time"
) )
@ -50,11 +51,11 @@ func (och *OnlineHistoryConsumerHandler) Init(cmdCh chan Cmd2Value) {
go och.MessagesDistributionHandle() go och.MessagesDistributionHandle()
och.cmdCh = cmdCh och.cmdCh = cmdCh
for i := 0; i < ChannelNum; i++ { for i := 0; i < ChannelNum; i++ {
och.chArrays[i] = make(chan Cmd2Value, 100) och.chArrays[i] = make(chan Cmd2Value, 50)
go och.Run(i) go och.Run(i)
} }
for i := 0; i < ChannelNum; i++ { for i := 0; i < ChannelNum; i++ {
och.chMongoArrays[i] = make(chan Cmd2Value, 1000) och.chMongoArrays[i] = make(chan Cmd2Value, 10000)
go och.MongoMessageRun(i) go och.MongoMessageRun(i)
} }
if config.Config.ReliableStorage { if config.Config.ReliableStorage {
@ -153,7 +154,6 @@ func (och *OnlineHistoryConsumerHandler) Run(channelID int) {
} }
} }
func (och *OnlineHistoryConsumerHandler) SendMessageToMongoCH(userID string, triggerID string, messages []*pbMsg.MsgDataToMQ, lastSeq uint64) { func (och *OnlineHistoryConsumerHandler) SendMessageToMongoCH(userID string, triggerID string, messages []*pbMsg.MsgDataToMQ, lastSeq uint64) {
return
hashCode := getHashCode(userID) hashCode := getHashCode(userID)
channelID := hashCode % ChannelNum channelID := hashCode % ChannelNum
log.Debug(triggerID, "generate channelID", hashCode, channelID, userID) log.Debug(triggerID, "generate channelID", hashCode, channelID, userID)
@ -389,11 +389,62 @@ func (och *OnlineHistoryConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupS
break break
} }
} }
rwLock := new(sync.RWMutex)
log.NewDebug("", "online new session msg come", claim.HighWaterMarkOffset(), claim.Topic(), claim.Partition()) log.NewDebug("", "online new session msg come", claim.HighWaterMarkOffset(), claim.Topic(), claim.Partition())
cMsg := make([]*sarama.ConsumerMessage, 0, 1000) cMsg := make([]*sarama.ConsumerMessage, 0, 1000)
t := time.NewTicker(time.Duration(100) * time.Millisecond) t := time.NewTicker(time.Duration(100) * time.Millisecond)
var triggerID string var triggerID string
go func() {
for {
select {
//case :
// triggerID = utils.OperationIDGenerator()
//
// log.NewDebug(triggerID, "claim.Messages ", msg)
// cMsg = append(cMsg, msg)
// if len(cMsg) >= 1000 {
// ccMsg := make([]*sarama.ConsumerMessage, 0, 1000)
// for _, v := range cMsg {
// ccMsg = append(ccMsg, v)
// }
// log.Debug(triggerID, "length trigger msg consumer start", len(ccMsg))
// och.msgDistributionCh <- Cmd2Value{Cmd: ConsumerMsgs, Value: TriggerChannelValue{
// triggerID: triggerID, cmsgList: ccMsg}}
// sess.MarkMessage(msg, "")
// cMsg = make([]*sarama.ConsumerMessage, 0, 1000)
// log.Debug(triggerID, "length trigger msg consumer end", len(cMsg))
// }
case <-t.C:
if len(cMsg) > 0 {
rwLock.Lock()
ccMsg := make([]*sarama.ConsumerMessage, 0, 1000)
for _, v := range cMsg {
ccMsg = append(ccMsg, v)
}
cMsg = make([]*sarama.ConsumerMessage, 0, 1000)
rwLock.Unlock()
split := 1000
triggerID = utils.OperationIDGenerator()
log.NewWarn(triggerID, "timer trigger msg consumer start", len(ccMsg))
for i := 0; i < len(ccMsg)/split; i++ {
//log.Debug()
och.msgDistributionCh <- Cmd2Value{Cmd: ConsumerMsgs, Value: TriggerChannelValue{
triggerID: triggerID, cmsgList: ccMsg[i*split : (i+1)*split]}}
}
if (len(ccMsg) % split) > 0 {
och.msgDistributionCh <- Cmd2Value{Cmd: ConsumerMsgs, Value: TriggerChannelValue{
triggerID: triggerID, cmsgList: ccMsg[split*(len(ccMsg)/split):]}}
}
//sess.MarkMessage(ccMsg[len(cMsg)-1], "")
log.NewWarn(triggerID, "timer trigger msg consumer end", len(cMsg))
}
}
}
}()
for msg := range claim.Messages() { for msg := range claim.Messages() {
//msgFromMQ := pbMsg.MsgDataToMQ{} //msgFromMQ := pbMsg.MsgDataToMQ{}
//err := proto.Unmarshal(msg.Value, &msgFromMQ) //err := proto.Unmarshal(msg.Value, &msgFromMQ)
@ -407,47 +458,16 @@ func (och *OnlineHistoryConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupS
////go func(cID uint32, userID string, messages []*pbMsg.MsgDataToMQ) { ////go func(cID uint32, userID string, messages []*pbMsg.MsgDataToMQ) {
//och.chArrays[channelID] <- Cmd2Value{Cmd: UserMessages, Value: MsgChannelValue{userID: userID, msgList: []*pbMsg.MsgDataToMQ{&msgFromMQ}, triggerID: msgFromMQ.OperationID}} //och.chArrays[channelID] <- Cmd2Value{Cmd: UserMessages, Value: MsgChannelValue{userID: userID, msgList: []*pbMsg.MsgDataToMQ{&msgFromMQ}, triggerID: msgFromMQ.OperationID}}
//sess.MarkMessage(msg, "") //sess.MarkMessage(msg, "")
rwLock.Lock()
cMsg = append(cMsg, msg) cMsg = append(cMsg, msg)
rwLock.Unlock()
sess.MarkMessage(msg, "")
//och.TriggerCmd(OnlineTopicBusy) //och.TriggerCmd(OnlineTopicBusy)
select {
//case :
// triggerID = utils.OperationIDGenerator()
//
// log.NewDebug(triggerID, "claim.Messages ", msg)
// cMsg = append(cMsg, msg)
// if len(cMsg) >= 1000 {
// ccMsg := make([]*sarama.ConsumerMessage, 0, 1000)
// for _, v := range cMsg {
// ccMsg = append(ccMsg, v)
// }
// log.Debug(triggerID, "length trigger msg consumer start", len(ccMsg))
// och.msgDistributionCh <- Cmd2Value{Cmd: ConsumerMsgs, Value: TriggerChannelValue{
// triggerID: triggerID, cmsgList: ccMsg}}
// sess.MarkMessage(msg, "")
// cMsg = make([]*sarama.ConsumerMessage, 0, 1000)
// log.Debug(triggerID, "length trigger msg consumer end", len(cMsg))
// }
case <-t.C:
if len(cMsg) > 0 {
ccMsg := make([]*sarama.ConsumerMessage, 0, 1000)
for _, v := range cMsg {
ccMsg = append(ccMsg, v)
}
triggerID = utils.OperationIDGenerator()
log.Debug(triggerID, "timer trigger msg consumer start", len(ccMsg))
och.msgDistributionCh <- Cmd2Value{Cmd: ConsumerMsgs, Value: TriggerChannelValue{
triggerID: triggerID, cmsgList: ccMsg}}
sess.MarkMessage(ccMsg[len(cMsg)-1], "")
cMsg = make([]*sarama.ConsumerMessage, 0, 1000)
log.Debug(triggerID, "timer trigger msg consumer end", len(cMsg))
}
default:
}
//log.NewDebug("", "online kafka get info to mongo", "msgTopic", msg.Topic, "msgPartition", msg.Partition, "online", msg.Offset, claim.HighWaterMarkOffset()) //log.NewDebug("", "online kafka get info to mongo", "msgTopic", msg.Topic, "msgPartition", msg.Partition, "online", msg.Offset, claim.HighWaterMarkOffset())
} }
return nil return nil
} }