mirror of
https://github.com/openimsdk/open-im-server.git
synced 2025-04-27 20:30:40 +08:00
msg update
This commit is contained in:
parent
b07bb1765e
commit
11ebe4e234
@ -1,6 +1,7 @@
|
|||||||
package msgtransfer
|
package msgtransfer
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
|
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
|
||||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/constant"
|
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/constant"
|
||||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/controller"
|
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/controller"
|
||||||
@ -22,13 +23,13 @@ const ChannelNum = 100
|
|||||||
|
|
||||||
type MsgChannelValue struct {
|
type MsgChannelValue struct {
|
||||||
aggregationID string //maybe userID or super groupID
|
aggregationID string //maybe userID or super groupID
|
||||||
triggerID string
|
ctx context.Context
|
||||||
msgList []*pbMsg.MsgDataToMQ
|
ctxMsgList []*ContextMsg
|
||||||
lastSeq uint64
|
lastSeq uint64
|
||||||
}
|
}
|
||||||
|
|
||||||
type TriggerChannelValue struct {
|
type TriggerChannelValue struct {
|
||||||
triggerID string
|
ctx context.Context
|
||||||
cMsgList []*sarama.ConsumerMessage
|
cMsgList []*sarama.ConsumerMessage
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -36,6 +37,10 @@ type Cmd2Value struct {
|
|||||||
Cmd int
|
Cmd int
|
||||||
Value interface{}
|
Value interface{}
|
||||||
}
|
}
|
||||||
|
type ContextMsg struct {
|
||||||
|
message *pbMsg.MsgDataToMQ
|
||||||
|
ctx context.Context
|
||||||
|
}
|
||||||
|
|
||||||
type OnlineHistoryRedisConsumerHandler struct {
|
type OnlineHistoryRedisConsumerHandler struct {
|
||||||
historyConsumerGroup *kafka.MConsumerGroup
|
historyConsumerGroup *kafka.MConsumerGroup
|
||||||
@ -80,38 +85,39 @@ func (och *OnlineHistoryRedisConsumerHandler) Run(channelID int) {
|
|||||||
switch cmd.Cmd {
|
switch cmd.Cmd {
|
||||||
case AggregationMessages:
|
case AggregationMessages:
|
||||||
msgChannelValue := cmd.Value.(MsgChannelValue)
|
msgChannelValue := cmd.Value.(MsgChannelValue)
|
||||||
msgList := msgChannelValue.msgList
|
ctxMsgList := msgChannelValue.ctxMsgList
|
||||||
triggerID := msgChannelValue.triggerID
|
ctx := msgChannelValue.ctx
|
||||||
storageMsgList := make([]*pbMsg.MsgDataToMQ, 0, 80)
|
storageMsgList := make([]*pbMsg.MsgDataToMQ, 0, 80)
|
||||||
notStoragePushMsgList := make([]*pbMsg.MsgDataToMQ, 0, 80)
|
storagePushMsgList := make([]*ContextMsg, 0, 80)
|
||||||
log.Debug(triggerID, "msg arrived channel", "channel id", channelID, msgList, msgChannelValue.aggregationID, len(msgList))
|
notStoragePushMsgList := make([]*ContextMsg, 0, 80)
|
||||||
|
log.ZDebug(ctx, "msg arrived channel", "channel id", channelID, "msgList length", len(ctxMsgList), "aggregationID", msgChannelValue.aggregationID)
|
||||||
var modifyMsgList []*pbMsg.MsgDataToMQ
|
var modifyMsgList []*pbMsg.MsgDataToMQ
|
||||||
ctx := mcontext.NewCtx("redis consumer")
|
//ctx := mcontext.NewCtx("redis consumer")
|
||||||
mcontext.SetOperationID(ctx, triggerID)
|
//mcontext.SetOperationID(ctx, triggerID)
|
||||||
for _, v := range msgList {
|
for _, v := range ctxMsgList {
|
||||||
log.Debug(triggerID, "msg come to storage center", v.String())
|
log.ZDebug(ctx, "msg come to storage center", v.message.String())
|
||||||
isHistory := utils.GetSwitchFromOptions(v.MsgData.Options, constant.IsHistory)
|
isHistory := utils.GetSwitchFromOptions(v.message.MsgData.Options, constant.IsHistory)
|
||||||
isSenderSync := utils.GetSwitchFromOptions(v.MsgData.Options, constant.IsSenderSync)
|
isSenderSync := utils.GetSwitchFromOptions(v.message.MsgData.Options, constant.IsSenderSync)
|
||||||
if isHistory {
|
if isHistory {
|
||||||
storageMsgList = append(storageMsgList, v)
|
storageMsgList = append(storageMsgList, v.message)
|
||||||
//log.NewWarn(triggerID, "storageMsgList to mongodb client msgID: ", v.MsgData.ClientMsgID)
|
storagePushMsgList = append(storagePushMsgList, v)
|
||||||
} else {
|
} else {
|
||||||
if !(!isSenderSync && msgChannelValue.aggregationID == v.MsgData.SendID) {
|
if !(!isSenderSync && msgChannelValue.aggregationID == v.message.MsgData.SendID) {
|
||||||
notStoragePushMsgList = append(notStoragePushMsgList, v)
|
notStoragePushMsgList = append(notStoragePushMsgList, v)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if v.MsgData.ContentType == constant.ReactionMessageModifier || v.MsgData.ContentType == constant.ReactionMessageDeleter {
|
if v.message.MsgData.ContentType == constant.ReactionMessageModifier || v.message.MsgData.ContentType == constant.ReactionMessageDeleter {
|
||||||
modifyMsgList = append(modifyMsgList, v)
|
modifyMsgList = append(modifyMsgList, v.message)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(modifyMsgList) > 0 {
|
if len(modifyMsgList) > 0 {
|
||||||
och.msgDatabase.MsgToModifyMQ(ctx, msgChannelValue.aggregationID, triggerID, modifyMsgList)
|
och.msgDatabase.MsgToModifyMQ(ctx, msgChannelValue.aggregationID, "", modifyMsgList)
|
||||||
}
|
}
|
||||||
log.Debug(triggerID, "msg storage length", len(storageMsgList), "push length", len(notStoragePushMsgList))
|
log.ZDebug(ctx, "msg storage length", len(storageMsgList), "push length", len(notStoragePushMsgList))
|
||||||
if len(storageMsgList) > 0 {
|
if len(storageMsgList) > 0 {
|
||||||
lastSeq, err := och.msgDatabase.BatchInsertChat2Cache(ctx, msgChannelValue.aggregationID, storageMsgList)
|
lastSeq, err := och.msgDatabase.BatchInsertChat2Cache(ctx, msgChannelValue.aggregationID, storageMsgList)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.NewError(triggerID, "single data insert to redis err", err.Error(), storageMsgList)
|
log.ZError(ctx, "batch data insert to redis err", err, "storageMsgList", storageMsgList)
|
||||||
och.singleMsgFailedCountMutex.Lock()
|
och.singleMsgFailedCountMutex.Lock()
|
||||||
och.singleMsgFailedCount += uint64(len(storageMsgList))
|
och.singleMsgFailedCount += uint64(len(storageMsgList))
|
||||||
och.singleMsgFailedCountMutex.Unlock()
|
och.singleMsgFailedCountMutex.Unlock()
|
||||||
@ -119,18 +125,20 @@ func (och *OnlineHistoryRedisConsumerHandler) Run(channelID int) {
|
|||||||
och.singleMsgSuccessCountMutex.Lock()
|
och.singleMsgSuccessCountMutex.Lock()
|
||||||
och.singleMsgSuccessCount += uint64(len(storageMsgList))
|
och.singleMsgSuccessCount += uint64(len(storageMsgList))
|
||||||
och.singleMsgSuccessCountMutex.Unlock()
|
och.singleMsgSuccessCountMutex.Unlock()
|
||||||
och.msgDatabase.MsgToMongoMQ(ctx, msgChannelValue.aggregationID, triggerID, storageMsgList, lastSeq)
|
och.msgDatabase.MsgToMongoMQ(ctx, msgChannelValue.aggregationID, "", storageMsgList, lastSeq)
|
||||||
for _, v := range storageMsgList {
|
for _, v := range storagePushMsgList {
|
||||||
och.msgDatabase.MsgToPushMQ(ctx, msgChannelValue.aggregationID, v)
|
och.msgDatabase.MsgToPushMQ(v.ctx, msgChannelValue.aggregationID, v.message)
|
||||||
}
|
}
|
||||||
for _, v := range notStoragePushMsgList {
|
for _, v := range notStoragePushMsgList {
|
||||||
och.msgDatabase.MsgToPushMQ(ctx, msgChannelValue.aggregationID, v)
|
och.msgDatabase.MsgToPushMQ(v.ctx, msgChannelValue.aggregationID, v.message)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
for _, v := range notStoragePushMsgList {
|
for _, v := range notStoragePushMsgList {
|
||||||
och.msgDatabase.MsgToPushMQ(ctx, msgChannelValue.aggregationID, v)
|
p, o, err := och.msgDatabase.MsgToPushMQ(v.ctx, msgChannelValue.aggregationID, v.message)
|
||||||
|
if err != nil {
|
||||||
|
log.ZError(v.ctx, "kafka send failed", err, "msg", v.message.String(), "pid", p, "offset", o)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -140,40 +148,43 @@ func (och *OnlineHistoryRedisConsumerHandler) Run(channelID int) {
|
|||||||
|
|
||||||
func (och *OnlineHistoryRedisConsumerHandler) MessagesDistributionHandle() {
|
func (och *OnlineHistoryRedisConsumerHandler) MessagesDistributionHandle() {
|
||||||
for {
|
for {
|
||||||
aggregationMsgs := make(map[string][]*pbMsg.MsgDataToMQ, ChannelNum)
|
aggregationMsgs := make(map[string][]*ContextMsg, ChannelNum)
|
||||||
select {
|
select {
|
||||||
case cmd := <-och.msgDistributionCh:
|
case cmd := <-och.msgDistributionCh:
|
||||||
switch cmd.Cmd {
|
switch cmd.Cmd {
|
||||||
case ConsumerMsgs:
|
case ConsumerMsgs:
|
||||||
triggerChannelValue := cmd.Value.(TriggerChannelValue)
|
triggerChannelValue := cmd.Value.(TriggerChannelValue)
|
||||||
triggerID := triggerChannelValue.triggerID
|
ctx := triggerChannelValue.ctx
|
||||||
consumerMessages := triggerChannelValue.cMsgList
|
consumerMessages := triggerChannelValue.cMsgList
|
||||||
//Aggregation map[userid]message list
|
//Aggregation map[userid]message list
|
||||||
log.Debug(triggerID, "batch messages come to distribution center", len(consumerMessages))
|
log.ZDebug(ctx, "batch messages come to distribution center", len(consumerMessages))
|
||||||
for i := 0; i < len(consumerMessages); i++ {
|
for i := 0; i < len(consumerMessages); i++ {
|
||||||
|
ctxMsg := &ContextMsg{}
|
||||||
msgFromMQ := pbMsg.MsgDataToMQ{}
|
msgFromMQ := pbMsg.MsgDataToMQ{}
|
||||||
err := proto.Unmarshal(consumerMessages[i].Value, &msgFromMQ)
|
err := proto.Unmarshal(consumerMessages[i].Value, &msgFromMQ)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error(triggerID, "msg_transfer Unmarshal msg err", "msg", string(consumerMessages[i].Value), "err", err.Error())
|
log.ZError(ctx, "msg_transfer Unmarshal msg err", err, string(consumerMessages[i].Value))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
log.Debug(triggerID, "single msg come to distribution center", msgFromMQ.String(), string(consumerMessages[i].Key))
|
ctxMsg.ctx = kafka.GetContextWithMQHeader(consumerMessages[i].Headers)
|
||||||
|
ctxMsg.message = &msgFromMQ
|
||||||
|
log.ZDebug(ctx, "single msg come to distribution center", msgFromMQ.String(), string(consumerMessages[i].Key))
|
||||||
if oldM, ok := aggregationMsgs[string(consumerMessages[i].Key)]; ok {
|
if oldM, ok := aggregationMsgs[string(consumerMessages[i].Key)]; ok {
|
||||||
oldM = append(oldM, &msgFromMQ)
|
oldM = append(oldM, ctxMsg)
|
||||||
aggregationMsgs[string(consumerMessages[i].Key)] = oldM
|
aggregationMsgs[string(consumerMessages[i].Key)] = oldM
|
||||||
} else {
|
} else {
|
||||||
m := make([]*pbMsg.MsgDataToMQ, 0, 100)
|
m := make([]*ContextMsg, 0, 100)
|
||||||
m = append(m, &msgFromMQ)
|
m = append(m, ctxMsg)
|
||||||
aggregationMsgs[string(consumerMessages[i].Key)] = m
|
aggregationMsgs[string(consumerMessages[i].Key)] = m
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
log.Debug(triggerID, "generate map list users len", len(aggregationMsgs))
|
log.ZDebug(ctx, "generate map list users len", len(aggregationMsgs))
|
||||||
for aggregationID, v := range aggregationMsgs {
|
for aggregationID, v := range aggregationMsgs {
|
||||||
if len(v) >= 0 {
|
if len(v) >= 0 {
|
||||||
hashCode := utils.GetHashCode(aggregationID)
|
hashCode := utils.GetHashCode(aggregationID)
|
||||||
channelID := hashCode % ChannelNum
|
channelID := hashCode % ChannelNum
|
||||||
log.Debug(triggerID, "generate channelID", hashCode, channelID, aggregationID)
|
log.ZDebug(ctx, "generate channelID", "hashCode", hashCode, "channelID", channelID, "aggregationID", aggregationID)
|
||||||
och.chArrays[channelID] <- Cmd2Value{Cmd: AggregationMessages, Value: MsgChannelValue{aggregationID: aggregationID, msgList: v, triggerID: triggerID}}
|
och.chArrays[channelID] <- Cmd2Value{Cmd: AggregationMessages, Value: MsgChannelValue{aggregationID: aggregationID, ctxMsgList: v, ctx: ctx}}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -194,10 +205,9 @@ func (och *OnlineHistoryRedisConsumerHandler) ConsumeClaim(sess sarama.ConsumerG
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
rwLock := new(sync.RWMutex)
|
rwLock := new(sync.RWMutex)
|
||||||
log.NewDebug("", "online new session msg come", claim.HighWaterMarkOffset(), claim.Topic(), claim.Partition())
|
log.ZDebug(context.Background(), "online new session msg come", claim.HighWaterMarkOffset(), claim.Topic(), claim.Partition())
|
||||||
cMsg := make([]*sarama.ConsumerMessage, 0, 1000)
|
cMsg := make([]*sarama.ConsumerMessage, 0, 1000)
|
||||||
t := time.NewTicker(time.Duration(100) * time.Millisecond)
|
t := time.NewTicker(time.Duration(100) * time.Millisecond)
|
||||||
var triggerID string
|
|
||||||
go func() {
|
go func() {
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
@ -211,18 +221,18 @@ func (och *OnlineHistoryRedisConsumerHandler) ConsumeClaim(sess sarama.ConsumerG
|
|||||||
cMsg = make([]*sarama.ConsumerMessage, 0, 1000)
|
cMsg = make([]*sarama.ConsumerMessage, 0, 1000)
|
||||||
rwLock.Unlock()
|
rwLock.Unlock()
|
||||||
split := 1000
|
split := 1000
|
||||||
triggerID = utils.OperationIDGenerator()
|
ctx := mcontext.WithTriggerIDContext(context.Background(), utils.OperationIDGenerator())
|
||||||
log.Debug(triggerID, "timer trigger msg consumer start", len(ccMsg))
|
log.ZDebug(ctx, "timer trigger msg consumer start", len(ccMsg))
|
||||||
for i := 0; i < len(ccMsg)/split; i++ {
|
for i := 0; i < len(ccMsg)/split; i++ {
|
||||||
//log.Debug()
|
//log.Debug()
|
||||||
och.msgDistributionCh <- Cmd2Value{Cmd: ConsumerMsgs, Value: TriggerChannelValue{
|
och.msgDistributionCh <- Cmd2Value{Cmd: ConsumerMsgs, Value: TriggerChannelValue{
|
||||||
triggerID: triggerID, cMsgList: ccMsg[i*split : (i+1)*split]}}
|
ctx: ctx, cMsgList: ccMsg[i*split : (i+1)*split]}}
|
||||||
}
|
}
|
||||||
if (len(ccMsg) % split) > 0 {
|
if (len(ccMsg) % split) > 0 {
|
||||||
och.msgDistributionCh <- Cmd2Value{Cmd: ConsumerMsgs, Value: TriggerChannelValue{
|
och.msgDistributionCh <- Cmd2Value{Cmd: ConsumerMsgs, Value: TriggerChannelValue{
|
||||||
triggerID: triggerID, cMsgList: ccMsg[split*(len(ccMsg)/split):]}}
|
ctx: ctx, cMsgList: ccMsg[split*(len(ccMsg)/split):]}}
|
||||||
}
|
}
|
||||||
log.Debug(triggerID, "timer trigger msg consumer end", len(cMsg))
|
log.ZDebug(ctx, "timer trigger msg consumer end", len(ccMsg))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -6,7 +6,6 @@ import (
|
|||||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/constant"
|
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/constant"
|
||||||
kfk "github.com/OpenIMSDK/Open-IM-Server/pkg/common/kafka"
|
kfk "github.com/OpenIMSDK/Open-IM-Server/pkg/common/kafka"
|
||||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/log"
|
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/log"
|
||||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/mcontext"
|
|
||||||
pbChat "github.com/OpenIMSDK/Open-IM-Server/pkg/proto/msg"
|
pbChat "github.com/OpenIMSDK/Open-IM-Server/pkg/proto/msg"
|
||||||
pbPush "github.com/OpenIMSDK/Open-IM-Server/pkg/proto/push"
|
pbPush "github.com/OpenIMSDK/Open-IM-Server/pkg/proto/push"
|
||||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
|
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
|
||||||
@ -29,10 +28,10 @@ func NewConsumerHandler(pusher *Pusher) *ConsumerHandler {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (c *ConsumerHandler) handleMs2PsChat(ctx context.Context, msg []byte) {
|
func (c *ConsumerHandler) handleMs2PsChat(ctx context.Context, msg []byte) {
|
||||||
log.NewDebug("", "msg come from kafka And push!!!", "msg", string(msg))
|
log.ZDebug(ctx, "msg come from kafka And push!!!", "msg", string(msg))
|
||||||
msgFromMQ := pbChat.PushMsgDataToMQ{}
|
msgFromMQ := pbChat.PushMsgDataToMQ{}
|
||||||
if err := proto.Unmarshal(msg, &msgFromMQ); err != nil {
|
if err := proto.Unmarshal(msg, &msgFromMQ); err != nil {
|
||||||
log.Error("", "push Unmarshal msg err", "msg", string(msg), "err", err.Error())
|
log.ZError(ctx, "push Unmarshal msg err", err, "msg", string(msg))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
pbData := &pbPush.PushMsgReq{
|
pbData := &pbPush.PushMsgReq{
|
||||||
@ -44,7 +43,6 @@ func (c *ConsumerHandler) handleMs2PsChat(ctx context.Context, msg []byte) {
|
|||||||
if nowSec-sec > 10 {
|
if nowSec-sec > 10 {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
mcontext.SetOperationID(ctx, "")
|
|
||||||
var err error
|
var err error
|
||||||
switch msgFromMQ.MsgData.SessionType {
|
switch msgFromMQ.MsgData.SessionType {
|
||||||
case constant.SuperGroupChatType:
|
case constant.SuperGroupChatType:
|
||||||
@ -53,7 +51,7 @@ func (c *ConsumerHandler) handleMs2PsChat(ctx context.Context, msg []byte) {
|
|||||||
err = c.pusher.MsgToUser(ctx, pbData.SourceID, pbData.MsgData)
|
err = c.pusher.MsgToUser(ctx, pbData.SourceID, pbData.MsgData)
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.NewError("", "push failed", *pbData)
|
log.ZError(ctx, "push failed", err, "msg", pbData.String())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
func (ConsumerHandler) Setup(_ sarama.ConsumerGroupSession) error { return nil }
|
func (ConsumerHandler) Setup(_ sarama.ConsumerGroupSession) error { return nil }
|
||||||
|
@ -277,6 +277,7 @@ const OpUserPlatform = "platform"
|
|||||||
const Token = "token"
|
const Token = "token"
|
||||||
const RpcCustomHeader = "customHeader" // rpc中间件自定义ctx参数
|
const RpcCustomHeader = "customHeader" // rpc中间件自定义ctx参数
|
||||||
const CheckKey = "CheckKey"
|
const CheckKey = "CheckKey"
|
||||||
|
const TriggerID = "triggerID"
|
||||||
|
|
||||||
const (
|
const (
|
||||||
UnreliableNotification = 1
|
UnreliableNotification = 1
|
||||||
|
@ -77,7 +77,7 @@ type MsgDatabase interface {
|
|||||||
|
|
||||||
MsgToMQ(ctx context.Context, key string, msg2mq *pbMsg.MsgDataToMQ) error
|
MsgToMQ(ctx context.Context, key string, msg2mq *pbMsg.MsgDataToMQ) error
|
||||||
MsgToModifyMQ(ctx context.Context, aggregationID string, triggerID string, messages []*pbMsg.MsgDataToMQ) error
|
MsgToModifyMQ(ctx context.Context, aggregationID string, triggerID string, messages []*pbMsg.MsgDataToMQ) error
|
||||||
MsgToPushMQ(ctx context.Context, sourceID string, msg2mq *pbMsg.MsgDataToMQ) error
|
MsgToPushMQ(ctx context.Context, sourceID string, msg2mq *pbMsg.MsgDataToMQ) (int32, int64, error)
|
||||||
MsgToMongoMQ(ctx context.Context, aggregationID string, triggerID string, messages []*pbMsg.MsgDataToMQ, lastSeq int64) error
|
MsgToMongoMQ(ctx context.Context, aggregationID string, triggerID string, messages []*pbMsg.MsgDataToMQ, lastSeq int64) error
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -193,10 +193,9 @@ func (db *msgDatabase) MsgToModifyMQ(ctx context.Context, aggregationID string,
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (db *msgDatabase) MsgToPushMQ(ctx context.Context, key string, msg2mq *pbMsg.MsgDataToMQ) error {
|
func (db *msgDatabase) MsgToPushMQ(ctx context.Context, key string, msg2mq *pbMsg.MsgDataToMQ) (int32, int64, error) {
|
||||||
mqPushMsg := pbMsg.PushMsgDataToMQ{MsgData: msg2mq.MsgData, SourceID: key}
|
mqPushMsg := pbMsg.PushMsgDataToMQ{MsgData: msg2mq.MsgData, SourceID: key}
|
||||||
_, _, err := db.producerToPush.SendMessage(ctx, key, &mqPushMsg)
|
return db.producerToPush.SendMessage(ctx, key, &mqPushMsg)
|
||||||
return err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (db *msgDatabase) MsgToMongoMQ(ctx context.Context, aggregationID string, triggerID string, messages []*pbMsg.MsgDataToMQ, lastSeq int64) error {
|
func (db *msgDatabase) MsgToMongoMQ(ctx context.Context, aggregationID string, triggerID string, messages []*pbMsg.MsgDataToMQ, lastSeq int64) error {
|
||||||
|
@ -47,7 +47,24 @@ func NewKafkaProducer(addr []string, topic string) *Producer {
|
|||||||
p.producer = producer
|
p.producer = producer
|
||||||
return &p
|
return &p
|
||||||
}
|
}
|
||||||
|
func GetMQHeaderWithContext(ctx context.Context) ([]sarama.RecordHeader, error) {
|
||||||
|
operationID, opUserID, platform, connID, err := mcontext.GetMustCtxInfo(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return []sarama.RecordHeader{
|
||||||
|
{Key: []byte(constant.OperationID), Value: []byte(operationID)},
|
||||||
|
{Key: []byte(constant.OpUserID), Value: []byte(opUserID)},
|
||||||
|
{Key: []byte(constant.OpUserPlatform), Value: []byte(platform)},
|
||||||
|
{Key: []byte(constant.ConnID), Value: []byte(connID)}}, err
|
||||||
|
}
|
||||||
|
func GetContextWithMQHeader(header []*sarama.RecordHeader) context.Context {
|
||||||
|
var values []string
|
||||||
|
for _, recordHeader := range header {
|
||||||
|
values = append(values, string(recordHeader.Value))
|
||||||
|
}
|
||||||
|
return mcontext.WithMustInfoCtx(values)
|
||||||
|
}
|
||||||
func (p *Producer) SendMessage(ctx context.Context, key string, m proto.Message) (int32, int64, error) {
|
func (p *Producer) SendMessage(ctx context.Context, key string, m proto.Message) (int32, int64, error) {
|
||||||
log.ZDebug(ctx, "SendMessage", "key ", key, "msg", m.String())
|
log.ZDebug(ctx, "SendMessage", "key ", key, "msg", m.String())
|
||||||
kMsg := &sarama.ProducerMessage{}
|
kMsg := &sarama.ProducerMessage{}
|
||||||
@ -65,15 +82,11 @@ func (p *Producer) SendMessage(ctx context.Context, key string, m proto.Message)
|
|||||||
return 0, 0, utils.Wrap(emptyMsg, "")
|
return 0, 0, utils.Wrap(emptyMsg, "")
|
||||||
}
|
}
|
||||||
kMsg.Metadata = ctx
|
kMsg.Metadata = ctx
|
||||||
operationID, opUserID, platform, connID, err := mcontext.GetMustCtxInfo(ctx)
|
header, err := GetMQHeaderWithContext(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, 0, utils.Wrap(err, "")
|
return 0, 0, utils.Wrap(err, "")
|
||||||
}
|
}
|
||||||
kMsg.Headers = []sarama.RecordHeader{
|
kMsg.Headers = header
|
||||||
{Key: []byte(constant.OperationID), Value: []byte(operationID)},
|
|
||||||
{Key: []byte(constant.OpUserID), Value: []byte(opUserID)},
|
|
||||||
{Key: []byte(constant.OpUserPlatform), Value: []byte(platform)},
|
|
||||||
{Key: []byte(constant.ConnID), Value: []byte(connID)}}
|
|
||||||
partition, offset, err := p.producer.SendMessage(kMsg)
|
partition, offset, err := p.producer.SendMessage(kMsg)
|
||||||
log.ZDebug(ctx, "ByteEncoder SendMessage end", "key ", kMsg.Key, "key length", kMsg.Value.Length())
|
log.ZDebug(ctx, "ByteEncoder SendMessage end", "key ", kMsg.Key, "key length", kMsg.Value.Length())
|
||||||
if err == nil {
|
if err == nil {
|
||||||
|
@ -147,14 +147,18 @@ func (l *ZapLogger) kvAppend(ctx context.Context, keysAndValues []interface{}) [
|
|||||||
operationID := mcontext.GetOperationID(ctx)
|
operationID := mcontext.GetOperationID(ctx)
|
||||||
opUserID := mcontext.GetOpUserID(ctx)
|
opUserID := mcontext.GetOpUserID(ctx)
|
||||||
connID := mcontext.GetConnID(ctx)
|
connID := mcontext.GetConnID(ctx)
|
||||||
|
triggerID := mcontext.GetTriggerID(ctx)
|
||||||
if opUserID != "" {
|
if opUserID != "" {
|
||||||
keysAndValues = append([]interface{}{constant.OpUserID, mcontext.GetOpUserID(ctx)}, keysAndValues...)
|
keysAndValues = append([]interface{}{constant.OpUserID, opUserID}, keysAndValues...)
|
||||||
}
|
}
|
||||||
if operationID != "" {
|
if operationID != "" {
|
||||||
keysAndValues = append([]interface{}{constant.OperationID, mcontext.GetOperationID(ctx)}, keysAndValues...)
|
keysAndValues = append([]interface{}{constant.OperationID, operationID}, keysAndValues...)
|
||||||
}
|
}
|
||||||
if connID != "" {
|
if connID != "" {
|
||||||
keysAndValues = append([]interface{}{constant.ConnID, mcontext.GetConnID(ctx)}, keysAndValues...)
|
keysAndValues = append([]interface{}{constant.ConnID, connID}, keysAndValues...)
|
||||||
|
}
|
||||||
|
if triggerID != "" {
|
||||||
|
keysAndValues = append([]interface{}{constant.TriggerID, triggerID}, keysAndValues...)
|
||||||
}
|
}
|
||||||
return keysAndValues
|
return keysAndValues
|
||||||
}
|
}
|
||||||
|
@ -6,6 +6,17 @@ import (
|
|||||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/errs"
|
"github.com/OpenIMSDK/Open-IM-Server/pkg/errs"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var mapper = []string{constant.OperationID, constant.OpUserID, constant.OpUserPlatform, constant.ConnID}
|
||||||
|
|
||||||
|
func WithOpUserIDContext(ctx context.Context, opUserID string) context.Context {
|
||||||
|
return context.WithValue(ctx, constant.OpUserID, opUserID)
|
||||||
|
}
|
||||||
|
func WithOpUserPlatformContext(ctx context.Context, platform string) context.Context {
|
||||||
|
return context.WithValue(ctx, constant.OpUserPlatform, platform)
|
||||||
|
}
|
||||||
|
func WithTriggerIDContext(ctx context.Context, triggerID string) context.Context {
|
||||||
|
return context.WithValue(ctx, constant.TriggerID, triggerID)
|
||||||
|
}
|
||||||
func NewCtx(operationID string) context.Context {
|
func NewCtx(operationID string) context.Context {
|
||||||
c := context.Background()
|
c := context.Background()
|
||||||
ctx := context.WithValue(c, constant.OperationID, operationID)
|
ctx := context.WithValue(c, constant.OperationID, operationID)
|
||||||
@ -34,6 +45,34 @@ func GetOperationID(ctx context.Context) string {
|
|||||||
}
|
}
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
func GetOpUserID(ctx context.Context) string {
|
||||||
|
if ctx.Value(constant.OpUserID) != "" {
|
||||||
|
s, ok := ctx.Value(constant.OpUserID).(string)
|
||||||
|
if ok {
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
func GetConnID(ctx context.Context) string {
|
||||||
|
if ctx.Value(constant.ConnID) != "" {
|
||||||
|
s, ok := ctx.Value(constant.ConnID).(string)
|
||||||
|
if ok {
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func GetTriggerID(ctx context.Context) string {
|
||||||
|
if ctx.Value(constant.TriggerID) != "" {
|
||||||
|
s, ok := ctx.Value(constant.TriggerID).(string)
|
||||||
|
if ok {
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
func GetMustCtxInfo(ctx context.Context) (operationID, opUserID, platform, connID string, err error) {
|
func GetMustCtxInfo(ctx context.Context) (operationID, opUserID, platform, connID string, err error) {
|
||||||
operationID, ok := ctx.Value(constant.OperationID).(string)
|
operationID, ok := ctx.Value(constant.OperationID).(string)
|
||||||
if !ok {
|
if !ok {
|
||||||
@ -54,23 +93,12 @@ func GetMustCtxInfo(ctx context.Context) (operationID, opUserID, platform, connI
|
|||||||
return
|
return
|
||||||
|
|
||||||
}
|
}
|
||||||
|
func WithMustInfoCtx(values []string) context.Context {
|
||||||
|
ctx := context.Background()
|
||||||
|
for i, v := range values {
|
||||||
|
ctx = context.WithValue(ctx, mapper[i], v)
|
||||||
|
|
||||||
func GetOpUserID(ctx context.Context) string {
|
|
||||||
if ctx.Value(constant.OpUserID) != "" {
|
|
||||||
s, ok := ctx.Value(constant.OpUserID).(string)
|
|
||||||
if ok {
|
|
||||||
return s
|
|
||||||
}
|
}
|
||||||
}
|
return ctx
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
func GetConnID(ctx context.Context) string {
|
|
||||||
if ctx.Value(constant.ConnID) != "" {
|
|
||||||
s, ok := ctx.Value(constant.ConnID).(string)
|
|
||||||
if ok {
|
|
||||||
return s
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return ""
|
|
||||||
}
|
}
|
||||||
|
Loading…
x
Reference in New Issue
Block a user