blackteay/open-im-server
1
0
Fork 0
mirror of https://github.com/openimsdk/open-im-server.git synced 2025-04-06 04:15:46 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: change consumer group receive messages avoid kafka consumer group always rebalanced. (#1942)

Browse Source
This commit is contained in:
OpenIM-Gordon 2024-02-27 16:44:51 +08:00 committed by GitHub
parent 6c4334b730
commit 8e6c1d74c7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 10 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
internal/msgtransfer/init.go
View File

@ -20,8 +20,6 @@ import (
"fmt" "fmt"
"github.com/OpenIMSDK/tools/errs" "github.com/OpenIMSDK/tools/errs"
"github.com/OpenIMSDK/tools/log"
util "github.com/openimsdk/open-im-server/v3/pkg/util/genutil" util "github.com/openimsdk/open-im-server/v3/pkg/util/genutil"
"net/http" "net/http"
@ -118,11 +116,8 @@ func (m *MsgTransfer) Start(prometheusPort int) error {
netErr error netErr error
) )
onError := func(ctx context.Context, err error, errInfo string) { go m.historyCH.historyConsumerGroup.RegisterHandleAndConsumer(m.ctx, m.historyCH)
log.ZWarn(ctx, errInfo, err) go m.historyMongoCH.historyConsumerGroup.RegisterHandleAndConsumer(m.ctx, m.historyMongoCH)
}
go m.historyCH.historyConsumerGroup.RegisterHandleAndConsumer(m.ctx, m.historyCH, onError)
go m.historyMongoCH.historyConsumerGroup.RegisterHandleAndConsumer(m.ctx, m.historyMongoCH, onError)
if config.Config.Prometheus.Enable { if config.Config.Prometheus.Enable {
go func() { go func() {

1
internal/msgtransfer/online_history_msg_handler.go
View File

@ -438,6 +438,7 @@ func (och *OnlineHistoryRedisConsumerHandler) ConsumeClaim(
wg = sync.WaitGroup{} wg = sync.WaitGroup{}
running = new(atomic.Bool) running = new(atomic.Bool)
) )
running.Store(true)
wg.Add(1) wg.Add(1)
go func() { go func() {

7
internal/push/consumer_init.go
View File

@ -16,8 +16,6 @@ package push
import ( import (
"context" "context"
"github.com/OpenIMSDK/tools/log"
) )
type Consumer struct { type Consumer struct {
@ -36,9 +34,6 @@ func NewConsumer(pusher *Pusher) (*Consumer, error) {
} }
func (c *Consumer) Start() { func (c *Consumer) Start() {
onError := func(ctx context.Context, err error, errInfo string) { go c.pushCh.pushConsumerGroup.RegisterHandleAndConsumer(context.Background(), &c.pushCh)
log.ZWarn(ctx, errInfo, err)
}
go c.pushCh.pushConsumerGroup.RegisterHandleAndConsumer(context.Background(), &c.pushCh, onError)
} }

13
pkg/common/kafka/consumer_group.go
View File

@ -17,8 +17,6 @@ package kafka
import ( import (
"context" "context"
"errors" "errors"
"fmt"
"github.com/IBM/sarama" "github.com/IBM/sarama"
"github.com/OpenIMSDK/tools/errs" "github.com/OpenIMSDK/tools/errs"
"github.com/OpenIMSDK/tools/log" "github.com/OpenIMSDK/tools/log"
@ -67,17 +65,18 @@ func (mc *MConsumerGroup) GetContextFromMsg(cMsg *sarama.ConsumerMessage) contex
return GetContextWithMQHeader(cMsg.Headers) return GetContextWithMQHeader(cMsg.Headers)
} }
func (mc *MConsumerGroup) RegisterHandleAndConsumer(ctx context.Context, handler sarama.ConsumerGroupHandler, onError func(context.Context, error, string)) { func (mc *MConsumerGroup) RegisterHandleAndConsumer(ctx context.Context, handler sarama.ConsumerGroupHandler) {
log.ZDebug(ctx, "register consumer group", "groupID", mc.groupID) log.ZDebug(ctx, "register consumer group", "groupID", mc.groupID)
for { for {
err := mc.ConsumerGroup.Consume(ctx, mc.topics, handler) err := mc.ConsumerGroup.Consume(ctx, mc.topics, handler)
if errors.Is(err, sarama.ErrClosedConsumerGroup) || errors.Is(err, context.Canceled) { if errors.Is(err, sarama.ErrClosedConsumerGroup) {
return
}
if errors.Is(err, context.Canceled) {
return return
} }
if err != nil { if err != nil {
errInfo := fmt.Sprintf("consume err: %v, topic: %v, groupID: %s", err, strings.Join(mc.topics, ", "), mc.groupID) log.ZWarn(ctx, "consume err", err, "topic", mc.topics, "groupID", mc.groupID)
onError(ctx, err, errInfo) // 调用回调函数处理错误
return
} }
} }
} }