fix(msg): keep read sequences monotonic across devices

This commit is contained in:
john 2026-10-19 15:35:57 +08:00
parent a5b427a3d6
commit e3bd4ea48d
13 changed files with 745 additions and 110 deletions

View File

@ -0,0 +1,48 @@
# Rolling out monotonic read sequences
Read progress is shared by a user across devices. Redis updates, cache fills, and
MongoDB writes must retain the greatest confirmed `read_seq` for each
`(user_id, conversation_id)`. Read-receipt notifications continue to use the
existing asynchronous queue and MQ pipeline.
## MongoDB preflight
The server creates a unique compound index named `user_conversation_unique` on
`seq_user`. The explicit name allows it to coexist with the previous non-unique
index on MongoDB 5.0 and later (the documented deployment uses MongoDB 6.0).
The old index is not automatically dropped.
Before deploying, check for duplicate rows in the intended OpenIM database:
```javascript
db.seq_user.aggregate([
{ $group: {
_id: { user_id: "$user_id", conversation_id: "$conversation_id" },
count: { $sum: 1 }
} },
{ $match: { count: { $gt: 1 } } },
{ $limit: 20 }
])
```
Resolve duplicates through a reviewed data migration before rollout, preserving
the greatest confirmed `read_seq` and reviewing the independent `min_seq` and
`max_seq` bounds. Coordinate the migration with writers. The application does not
delete or merge historical records automatically; index creation fails explicitly
if duplicates remain. Check index build time and permissions for the deployment.
If an equivalent unique index has already been installed manually, reconcile its
name with `user_conversation_unique` before rollout.
## Redis compatibility and limits
Read-sequence keys keep the existing Redis hash `value` field and 30-day write
TTL. Their dedicated monotonic fill path replaces generic RocksCache fills;
updating a value also invalidates an older RocksCache fill owner. Complete the
rollout across all writers, since old processes can still perform unconditional
writes. Other cached fields retain their existing cache behavior.
Atomic maximum updates tolerate concurrent devices, duplicate receipts, and
out-of-order persistence. They do not repair a previously lost persistence event
or guarantee delivery through an in-memory queue. Reliable retry/compensation and
reconciliation of existing Redis/MongoDB differences remain separate work. Do not
mark all messages as read to repair a discrepancy; use confirmed read progress.

3
go.mod
View File

@ -4,6 +4,7 @@ go 1.25.0
require (
firebase.google.com/go/v4 v4.14.1
github.com/alicebob/miniredis/v2 v2.34.0
github.com/dtm-labs/rockscache v0.1.1
github.com/gin-gonic/gin v1.9.1
github.com/go-playground/validator/v10 v10.20.0
@ -56,6 +57,7 @@ require (
cloud.google.com/go/longrunning v0.5.5 // indirect
cloud.google.com/go/storage v1.40.0 // indirect
github.com/MicahParks/keyfunc v1.9.0 // indirect
github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302 // indirect
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible // indirect
github.com/aws/aws-sdk-go-v2 v1.32.5 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1 // indirect
@ -182,6 +184,7 @@ require (
github.com/xdg-go/scram v1.1.2 // indirect
github.com/xdg-go/stringprep v1.0.4 // indirect
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
github.com/yuin/gopher-lua v1.1.1 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
go.etcd.io/etcd/api/v3 v3.5.13 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.13 // indirect

6
go.sum
View File

@ -19,6 +19,10 @@ github.com/IBM/sarama v1.43.0/go.mod h1:zlE6HEbC/SMQ9mhEYaF7nNLYOUyrs0obySKCckWP
github.com/MicahParks/keyfunc v1.9.0 h1:lhKd5xrFHLNOWrDc4Tyb/Q1AJ4LCzQ48GVJyVIID3+o=
github.com/MicahParks/keyfunc v1.9.0/go.mod h1:IdnCilugA0O/99dW+/MkvlyrsX8+L8+x95xuVNtM5jw=
github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM=
github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302 h1:uvdUDbHQHO85qeSydJtItA4T55Pw6BtAejd0APRJOCE=
github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
github.com/alicebob/miniredis/v2 v2.34.0 h1:mBFWMaJSNL9RwdGRyEDoAAv8OQc5UlEhLDQggTglU/0=
github.com/alicebob/miniredis/v2 v2.34.0/go.mod h1:kWShP4b58T1CW0Y5dViCd5ztzrDqRWqM3nksiyXk5s8=
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g=
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
github.com/aws/aws-sdk-go-v2 v1.32.5 h1:U8vdWJuY7ruAkzaOdD7guwJjD06YSKmnKCJs7s3IkIo=
@ -478,6 +482,8 @@ github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7Jul
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
go.etcd.io/etcd/api/v3 v3.5.13 h1:8WXU2/NBge6AUF1K1gOexB6e07NgsN1hXK0rSTtgSp4=

View File

@ -96,10 +96,11 @@ func (m *msgServer) SetConversationHasReadSeq(ctx context.Context, req *msg.SetC
if req.HasReadSeq > maxSeq {
return nil, errs.ErrArgs.WrapMsg("hasReadSeq must not be bigger than maxSeq")
}
if err := m.MsgDatabase.SetHasReadSeq(ctx, req.UserID, req.ConversationID, req.HasReadSeq); err != nil {
hasReadSeq, err := m.MsgDatabase.SetHasReadSeq(ctx, req.UserID, req.ConversationID, req.HasReadSeq)
if err != nil {
return nil, err
}
m.sendMarkAsReadNotification(ctx, req.ConversationID, constant.SingleChatType, req.UserID, req.UserID, nil, req.HasReadSeq)
m.sendMarkAsReadNotification(ctx, req.ConversationID, constant.SingleChatType, req.UserID, req.UserID, nil, hasReadSeq)
return &msg.SetConversationHasReadSeqResp{}, nil
}
@ -122,16 +123,10 @@ func (m *msgServer) MarkMsgsAsRead(ctx context.Context, req *msg.MarkMsgsAsReadR
if err := m.MsgDatabase.MarkSingleChatMsgsAsRead(ctx, req.UserID, req.ConversationID, req.Seqs); err != nil {
return nil, err
}
currentHasReadSeq, err := m.MsgDatabase.GetHasReadSeq(ctx, req.UserID, req.ConversationID)
if err != nil && !errors.Is(err, redis.Nil) {
hasReadSeq, err = m.MsgDatabase.SetHasReadSeq(ctx, req.UserID, req.ConversationID, hasReadSeq)
if err != nil {
return nil, err
}
if hasReadSeq > currentHasReadSeq {
err = m.MsgDatabase.SetHasReadSeq(ctx, req.UserID, req.ConversationID, hasReadSeq)
if err != nil {
return nil, err
}
}
reqCallback := &cbapi.CallbackSingleMsgReadReq{
ConversationID: conversation.ConversationID,
@ -176,23 +171,17 @@ func (m *msgServer) MarkConversationAsRead(ctx context.Context, req *msg.MarkCon
return nil, err
}
}
if req.HasReadSeq > hasReadSeq {
err = m.MsgDatabase.SetHasReadSeq(ctx, req.UserID, req.ConversationID, req.HasReadSeq)
if err != nil {
return nil, err
}
hasReadSeq = req.HasReadSeq
hasReadSeq, err = m.MsgDatabase.SetHasReadSeq(ctx, req.UserID, req.ConversationID, req.HasReadSeq)
if err != nil {
return nil, err
}
m.sendMarkAsReadNotification(ctx, req.ConversationID, conversation.ConversationType, req.UserID,
m.conversationAndGetRecvID(conversation, req.UserID), seqs, hasReadSeq)
} else if conversation.ConversationType == constant.ReadGroupChatType ||
conversation.ConversationType == constant.NotificationChatType {
if req.HasReadSeq > hasReadSeq {
err = m.MsgDatabase.SetHasReadSeq(ctx, req.UserID, req.ConversationID, req.HasReadSeq)
if err != nil {
return nil, err
}
hasReadSeq = req.HasReadSeq
hasReadSeq, err = m.MsgDatabase.SetHasReadSeq(ctx, req.UserID, req.ConversationID, req.HasReadSeq)
if err != nil {
return nil, err
}
m.sendMarkAsReadNotification(ctx, req.ConversationID, constant.SingleChatType, req.UserID,
req.UserID, seqs, hasReadSeq)

View File

@ -0,0 +1,107 @@
package msg
import (
"context"
"encoding/json"
"fmt"
"testing"
"time"
"github.com/openimsdk/open-im-server/v3/pkg/common/config"
"github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller"
"github.com/openimsdk/open-im-server/v3/pkg/common/webhook"
"github.com/openimsdk/open-im-server/v3/pkg/notification"
"github.com/openimsdk/open-im-server/v3/pkg/rpccache"
"github.com/openimsdk/open-im-server/v3/pkg/rpcli"
"github.com/openimsdk/protocol/constant"
"github.com/openimsdk/protocol/conversation"
pbmsg "github.com/openimsdk/protocol/msg"
"github.com/openimsdk/protocol/sdkws"
"github.com/openimsdk/tools/mcontext"
"github.com/openimsdk/tools/mq/memamq"
"github.com/stretchr/testify/require"
"google.golang.org/grpc"
)
type receiptDatabase struct {
controller.CommonMsgDatabase
calls int
initial int64
}
func (*receiptDatabase) GetMaxSeq(context.Context, string) (int64, error) { return 346, nil }
func (db *receiptDatabase) GetHasReadSeq(context.Context, string, string) (int64, error) {
return db.initial, nil
}
func (db *receiptDatabase) SetHasReadSeq(context.Context, string, string, int64) (int64, error) {
db.calls++
return 346, nil // Another device advanced after the RPC's initial read.
}
func (*receiptDatabase) MarkSingleChatMsgsAsRead(context.Context, string, string, []int64) error {
return nil
}
type receiptConversationClient struct {
conversation.ConversationClient
sessionType int32
}
func (c *receiptConversationClient) GetConversation(_ context.Context, req *conversation.GetConversationReq, _ ...grpc.CallOption) (*conversation.GetConversationResp, error) {
return &conversation.GetConversationResp{Conversation: &conversation.Conversation{
ConversationID: req.ConversationID, OwnerUserID: req.OwnerUserID,
ConversationType: c.sessionType, UserID: "peer", GroupID: "group",
}}, nil
}
func TestReadSeqReceiptsUseEffectiveProgress(t *testing.T) {
for _, method := range []string{"set", "messages", "single conversation", "group conversation"} {
for _, requested := range []int64{200, 346} {
for _, initial := range []int64{100, 346} {
t.Run(fmt.Sprintf("%s/initial=%d/requested=%d", method, initial, requested), func(t *testing.T) {
ctx := mcontext.WithOpUserIDContext(context.Background(), "user")
db := &receiptDatabase{initial: initial}
queue := memamq.NewMemoryQueue(1, 8)
defer queue.Stop()
sent := make(chan *sdkws.MsgData, 1)
notifier := notification.NewNotificationSender(&config.Notification{}, notification.WithQueue(queue),
notification.WithLocalSendMsg(func(_ context.Context, req *pbmsg.SendMsgReq) (*pbmsg.SendMsgResp, error) {
sent <- req.MsgData
return &pbmsg.SendMsgResp{}, nil
}))
sessionType := int32(constant.SingleChatType)
if method == "group conversation" {
sessionType = constant.ReadGroupChatType
}
client := &rpcli.ConversationClient{ConversationClient: &receiptConversationClient{sessionType: sessionType}}
server := &msgServer{MsgDatabase: db, notificationSender: notifier, config: &Config{},
ConversationLocalCache: rpccache.NewConversationLocalCache(client, &config.LocalCache{}, nil),
webhookClient: webhook.NewWebhookClient("", queue),
}
var err error
switch method {
case "set":
_, err = server.SetConversationHasReadSeq(ctx, &pbmsg.SetConversationHasReadSeqReq{UserID: "user", ConversationID: "group", HasReadSeq: requested})
case "messages":
_, err = server.MarkMsgsAsRead(ctx, &pbmsg.MarkMsgsAsReadReq{UserID: "user", ConversationID: "group", Seqs: []int64{requested}})
default:
_, err = server.MarkConversationAsRead(ctx, &pbmsg.MarkConversationAsReadReq{UserID: "user", ConversationID: "group", HasReadSeq: requested})
}
require.NoError(t, err)
require.Equal(t, 1, db.calls, "equal reports must still allow persistence/notification repair")
select {
case message := <-sent:
require.EqualValues(t, constant.HasReadReceipt, message.ContentType)
var elem sdkws.NotificationElem
require.NoError(t, json.Unmarshal(message.Content, &elem))
var tips sdkws.MarkAsReadTips
require.NoError(t, json.Unmarshal([]byte(elem.Detail), &tips))
require.EqualValues(t, 346, tips.HasReadSeq)
require.Equal(t, "group", tips.ConversationID)
case <-time.After(3 * time.Second):
t.Fatal("read receipt was not emitted")
}
})
}
}
}
}

View File

@ -65,26 +65,110 @@ func (s *seqUserCacheRedis) SetUserMinSeq(ctx context.Context, conversationID st
return s.SetUserMinSeqs(ctx, userID, map[string]int64{conversationID: seq})
}
// Read sequences use their own monotonic fill path. A generic RocksCache
// refill can otherwise overwrite a concurrent read receipt with an older DB value.
func (s *seqUserCacheRedis) GetUserReadSeq(ctx context.Context, conversationID string, userID string) (int64, error) {
return getCache(ctx, s.rocks, s.getSeqUserReadSeqKey(conversationID, userID), s.readExpireTime, func(ctx context.Context) (int64, error) {
if s.rocks.Disable() {
return s.mgo.GetUserReadSeq(ctx, conversationID, userID)
})
}
key := s.getSeqUserReadSeqKey(conversationID, userID)
value, err := s.rocks.GetRedis().HGet(ctx, key, "value").Result()
if err == nil && value != "" {
return strconv.ParseInt(value, 10, 64)
}
if err != nil && err != redis.Nil {
return 0, errs.Wrap(err)
}
seq, err := s.mgo.GetUserReadSeq(ctx, conversationID, userID)
if err != nil {
return 0, err
}
return s.mergeReadSeq(ctx, key, seq)
}
func (s *seqUserCacheRedis) SetUserReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error {
if s.rocks.GetRedis() == nil {
return s.SetUserReadSeqToDB(ctx, conversationID, userID, seq)
// SetUserReadSeq returns the effective sequence, including any concurrent advance.
func (s *seqUserCacheRedis) SetUserReadSeq(ctx context.Context, conversationID string, userID string, seq int64) (int64, error) {
if seq < 0 {
return 0, errs.ErrArgs.WrapMsg("read seq must not be negative")
}
dbSeq, err := s.GetUserReadSeq(ctx, conversationID, userID)
if s.rocks.Disable() {
if err := s.SetUserReadSeqToDB(ctx, conversationID, userID, seq); err != nil {
return 0, err
}
return s.mgo.GetUserReadSeq(ctx, conversationID, userID)
}
// Preserve the MongoDB baseline on a cold cache, as well as the value read
// before a key expires between this read and the atomic update.
current, err := s.GetUserReadSeq(ctx, conversationID, userID)
if err != nil {
return err
return 0, err
}
if dbSeq < seq {
if err := s.rocks.GetClient().RawSet(ctx, s.getSeqUserReadSeqKey(conversationID, userID), strconv.Itoa(int(seq)), s.readExpireTime); err != nil {
return errs.Wrap(err)
return s.mergeReadSeq(ctx, s.getSeqUserReadSeqKey(conversationID, userID), max(current, seq))
}
// Compare decimal strings rather than Lua doubles to preserve the full int64 range.
// Removing the refill owner also fences any RocksCache fill started by an older
// server instance before this write during a rolling upgrade.
var mergeReadSeqScript = redis.NewScript(`
local incoming = ARGV[1]
local current = redis.call('HGET', KEYS[1], 'value')
if current and current ~= '' then
if not string.match(current, '^%d+$') then
return redis.error_reply('invalid cached read sequence')
end
if #current > #incoming or (#current == #incoming and current > incoming) then
incoming = current
end
end
redis.call('HSET', KEYS[1], 'value', incoming)
redis.call('HDEL', KEYS[1], 'lockUntil', 'lockOwner')
redis.call('PEXPIRE', KEYS[1], ARGV[2])
return incoming
`)
func (s *seqUserCacheRedis) mergeReadSeq(ctx context.Context, key string, seq int64) (int64, error) {
if seq < 0 {
return 0, errs.ErrArgs.WrapMsg("read seq must not be negative")
}
value, err := mergeReadSeqScript.Run(ctx, s.rocks.GetRedis(), []string{key},
strconv.FormatInt(seq, 10), s.readExpireTime.Milliseconds()).Text()
if err != nil {
return 0, errs.Wrap(err)
}
return strconv.ParseInt(value, 10, 64)
}
func (s *seqUserCacheRedis) mergeReadSeqs(ctx context.Context, userID string, seqs map[string]int64) (map[string]int64, error) {
pipe := s.rocks.GetRedis().Pipeline()
commands := make(map[string]*redis.Cmd, len(seqs))
for conversationID, seq := range seqs {
if seq < 0 {
return nil, errs.ErrArgs.WrapMsg("read seq must not be negative")
}
// Each script touches a single key, so ClusterClient can route the
// pipeline across slots without a cross-slot transaction.
commands[conversationID] = mergeReadSeqScript.Eval(ctx, pipe,
[]string{s.getSeqUserReadSeqKey(conversationID, userID)},
strconv.FormatInt(seq, 10), s.readExpireTime.Milliseconds())
}
if len(commands) > 0 {
if _, err := pipe.Exec(ctx); err != nil {
return nil, errs.Wrap(err)
}
}
return nil
result := make(map[string]int64, len(commands))
for conversationID, cmd := range commands {
value, err := cmd.Text()
if err != nil {
return nil, errs.Wrap(err)
}
seq, err := strconv.ParseInt(value, 10, 64)
if err != nil {
return nil, errs.Wrap(err)
}
result[conversationID] = seq
}
return result, nil
}
func (s *seqUserCacheRedis) SetUserReadSeqToDB(ctx context.Context, conversationID string, userID string, seq int64) error {
@ -102,83 +186,84 @@ func (s *seqUserCacheRedis) SetUserMinSeqs(ctx context.Context, userID string, s
return DeleteCacheBySlot(ctx, s.rocks, keys)
}
func (s *seqUserCacheRedis) setUserRedisReadSeqs(ctx context.Context, userID string, seqs map[string]int64) error {
keys := make([]string, 0, len(seqs))
keySeq := make(map[string]int64)
func (s *seqUserCacheRedis) SetUserReadSeqs(ctx context.Context, userID string, seqs map[string]int64) error {
conversationIDs := make([]string, 0, len(seqs))
for conversationID, seq := range seqs {
key := s.getSeqUserReadSeqKey(conversationID, userID)
keys = append(keys, key)
keySeq[key] = seq
if seq < 0 {
return errs.ErrArgs.WrapMsg("read seq must not be negative")
}
conversationIDs = append(conversationIDs, conversationID)
}
slotKeys, err := groupKeysBySlot(ctx, s.rocks.GetRedis(), keys)
if s.rocks.Disable() {
for conversationID, seq := range seqs {
if err := s.SetUserReadSeqToDB(ctx, conversationID, userID, seq); err != nil {
return err
}
}
return nil
}
current, err := s.GetUserReadSeqs(ctx, userID, conversationIDs)
if err != nil {
return err
}
for _, keys := range slotKeys {
pipe := s.rocks.GetRedis().Pipeline()
for _, key := range keys {
pipe.HSet(ctx, key, "value", strconv.FormatInt(keySeq[key], 10))
pipe.Expire(ctx, key, s.readExpireTime)
}
if _, err := pipe.Exec(ctx); err != nil {
return err
}
updates := make(map[string]int64, len(seqs))
for conversationID, seq := range seqs {
updates[conversationID] = max(current[conversationID], seq)
}
return nil
}
func (s *seqUserCacheRedis) SetUserReadSeqs(ctx context.Context, userID string, seqs map[string]int64) error {
if len(seqs) == 0 {
return nil
}
if err := s.setUserRedisReadSeqs(ctx, userID, seqs); err != nil {
return err
}
return nil
_, err = s.mergeReadSeqs(ctx, userID, updates)
return err
}
func (s *seqUserCacheRedis) GetUserReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) {
res, err := batchGetCache2(ctx, s.rocks, s.readExpireTime, conversationIDs, func(conversationID string) string {
return s.getSeqUserReadSeqKey(conversationID, userID)
}, func(v *readSeqModel) string {
return v.ConversationID
}, func(ctx context.Context, conversationIDs []string) ([]*readSeqModel, error) {
seqs, err := s.mgo.GetUserReadSeqs(ctx, userID, conversationIDs)
if s.rocks.Disable() {
return s.mgo.GetUserReadSeqs(ctx, userID, conversationIDs)
}
pipe := s.rocks.GetRedis().Pipeline()
commands := make(map[string]*redis.StringCmd, len(conversationIDs))
for _, conversationID := range conversationIDs {
if _, ok := commands[conversationID]; !ok {
commands[conversationID] = pipe.HGet(ctx, s.getSeqUserReadSeqKey(conversationID, userID), "value")
}
}
if len(commands) > 0 {
if _, err := pipe.Exec(ctx); err != nil && err != redis.Nil {
return nil, errs.Wrap(err)
}
}
result := make(map[string]int64, len(commands))
missing := make([]string, 0)
for conversationID, cmd := range commands {
value, err := cmd.Result()
if err == redis.Nil || (err == nil && value == "") {
missing = append(missing, conversationID)
continue
}
if err != nil {
return nil, err
return nil, errs.Wrap(err)
}
res := make([]*readSeqModel, 0, len(seqs))
for conversationID, seq := range seqs {
res = append(res, &readSeqModel{ConversationID: conversationID, Seq: seq})
seq, err := strconv.ParseInt(value, 10, 64)
if err != nil {
return nil, errs.Wrap(err)
}
return res, nil
})
result[conversationID] = seq
}
if len(missing) == 0 {
return result, nil
}
stored, err := s.mgo.GetUserReadSeqs(ctx, userID, missing)
if err != nil {
return nil, err
}
data := make(map[string]int64)
for _, v := range res {
data[v.ConversationID] = v.Seq
fills := make(map[string]int64, len(missing))
for _, conversationID := range missing {
fills[conversationID] = stored[conversationID]
}
return data, nil
}
var _ BatchCacheCallback[string] = (*readSeqModel)(nil)
type readSeqModel struct {
ConversationID string
Seq int64
}
func (r *readSeqModel) BatchCache(conversationID string) {
r.ConversationID = conversationID
}
func (r *readSeqModel) UnmarshalJSON(bytes []byte) (err error) {
r.Seq, err = strconv.ParseInt(string(bytes), 10, 64)
return
}
func (r *readSeqModel) MarshalJSON() ([]byte, error) {
return []byte(strconv.FormatInt(r.Seq, 10)), nil
filled, err := s.mergeReadSeqs(ctx, userID, fills)
if err != nil {
return nil, err
}
for conversationID, seq := range filled {
result[conversationID] = seq
}
return result, nil
}

View File

@ -0,0 +1,274 @@
package redis
import (
"context"
"fmt"
"strconv"
"sync"
"testing"
"time"
"github.com/alicebob/miniredis/v2"
"github.com/openimsdk/open-im-server/v3/pkg/common/storage/database"
"github.com/redis/go-redis/v9"
"github.com/stretchr/testify/require"
)
type readSeqStore struct {
database.SeqUser
read func(context.Context, string, string) (int64, error)
readBatch func(context.Context, string, []string) (map[string]int64, error)
write func(context.Context, string, string, int64) error
}
func (s *readSeqStore) SetUserReadSeq(ctx context.Context, conv, user string, seq int64) error {
return s.write(ctx, conv, user, seq)
}
func (s *readSeqStore) GetUserReadSeq(ctx context.Context, conv, user string) (int64, error) {
if s.read != nil {
return s.read(ctx, conv, user)
}
return 100, nil
}
func (s *readSeqStore) GetUserReadSeqs(ctx context.Context, user string, convs []string) (map[string]int64, error) {
if s.readBatch != nil {
return s.readBatch(ctx, user, convs)
}
result := make(map[string]int64, len(convs))
for _, conv := range convs {
result[conv] = 100
}
return result, nil
}
func readSeqTestCache(t *testing.T, store database.SeqUser) (*seqUserCacheRedis, *redis.Client, *miniredis.Miniredis) {
t.Helper()
server := miniredis.RunT(t)
rdb := redis.NewClient(&redis.Options{Addr: server.Addr()})
t.Cleanup(func() { _ = rdb.Close() })
return NewSeqUserCacheRedis(rdb, store).(*seqUserCacheRedis), rdb, server
}
// Delay the lower writer at the Redis command boundary, after its cache read.
// This covers both the old HSET and the atomic-script implementation.
type pauseReadSeqWrite struct {
started chan struct{}
resume chan struct{}
once sync.Once
}
func (h *pauseReadSeqWrite) DialHook(next redis.DialHook) redis.DialHook { return next }
func (h *pauseReadSeqWrite) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook {
return next
}
func (h *pauseReadSeqWrite) ProcessHook(next redis.ProcessHook) redis.ProcessHook {
return func(ctx context.Context, cmd redis.Cmder) error {
if cmd.Name() == "hset" || cmd.Name() == "evalsha" || cmd.Name() == "eval" {
for _, arg := range cmd.Args() {
if fmt.Sprint(arg) == "200" {
h.once.Do(func() {
close(h.started)
select {
case <-h.resume:
case <-ctx.Done():
}
})
break
}
}
}
return next(ctx, cmd)
}
}
// Keep a compile-time check that this hook remains usable by go-redis.
var _ redis.Hook = (*pauseReadSeqWrite)(nil)
func TestReadSeqConcurrentWritersNeverRegress(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
s, rdb, _ := readSeqTestCache(t, &readSeqStore{})
key := s.getSeqUserReadSeqKey("group", "user")
require.NoError(t, rdb.HSet(ctx, key, "value", "100").Err())
hook := &pauseReadSeqWrite{started: make(chan struct{}), resume: make(chan struct{})}
rdb.AddHook(hook)
done := make(chan error, 1)
go func() { _, err := s.SetUserReadSeq(ctx, "group", "user", 200); done <- err }()
select {
case <-hook.started:
case <-ctx.Done():
t.Fatal("lower writer never reached Redis")
}
highSeq, highErr := s.SetUserReadSeq(ctx, "group", "user", 346)
close(hook.resume)
require.NoError(t, highErr)
require.EqualValues(t, 346, highSeq)
require.NoError(t, <-done)
require.Equal(t, "346", rdb.HGet(ctx, key, "value").Val())
}
func TestReadSeqRefillDoesNotOverwriteNewerProgress(t *testing.T) {
for _, batch := range []bool{false, true} {
t.Run(fmt.Sprintf("batch=%v", batch), func(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
started, resume := make(chan struct{}), make(chan struct{})
var once sync.Once
wait := func() {
once.Do(func() {
close(started)
select {
case <-resume:
case <-ctx.Done():
}
})
}
store := &readSeqStore{}
if batch {
store.readBatch = func(context.Context, string, []string) (map[string]int64, error) {
wait()
return map[string]int64{"group": 100}, nil
}
} else {
store.read = func(context.Context, string, string) (int64, error) { wait(); return 100, nil }
}
s, rdb, _ := readSeqTestCache(t, store)
type result struct {
seq int64
err error
}
done := make(chan result, 1)
go func() {
if batch {
values, err := s.GetUserReadSeqs(ctx, "user", []string{"group"})
done <- result{values["group"], err}
} else {
seq, err := s.GetUserReadSeq(ctx, "group", "user")
done <- result{seq, err}
}
}()
select {
case <-started:
case <-ctx.Done():
t.Fatal("refill never reached MongoDB")
}
// Seed the new baseline as another request completes its refill, then
// advance it through the real writer while the older query is pending.
key := s.getSeqUserReadSeqKey("group", "user")
require.NoError(t, rdb.HSet(ctx, key, "value", "100").Err())
require.NoError(t, rdb.HDel(ctx, key, "lockUntil", "lockOwner").Err())
writeErr := s.SetUserReadSeqs(ctx, "user", map[string]int64{"group": 346})
close(resume)
require.NoError(t, writeErr)
got := <-done
require.NoError(t, got.err)
require.EqualValues(t, 346, got.seq, "the response must include the concurrent advancement")
require.Equal(t, "346", rdb.HGet(ctx, key, "value").Val())
})
}
}
func TestReadSeqBatchPreservesMongoBaseline(t *testing.T) {
store := &readSeqStore{readBatch: func(context.Context, string, []string) (map[string]int64, error) {
return map[string]int64{"group": 346}, nil
}}
s, rdb, _ := readSeqTestCache(t, store)
ctx := context.Background()
require.NoError(t, s.SetUserReadSeqs(ctx, "user", map[string]int64{"group": 200}))
require.Equal(t, "346", rdb.HGet(ctx, s.getSeqUserReadSeqKey("group", "user"), "value").Val())
}
func TestReadSeqBatchNeverRegresses(t *testing.T) {
s, rdb, _ := readSeqTestCache(t, &readSeqStore{})
ctx := context.Background()
key := s.getSeqUserReadSeqKey("group", "user")
require.NoError(t, rdb.HSet(ctx, key, "value", "346").Err())
require.NoError(t, s.SetUserReadSeqs(ctx, "user", map[string]int64{"group": 200}))
require.Equal(t, "346", rdb.HGet(ctx, key, "value").Val())
}
func TestReadSeqInt64Precision(t *testing.T) {
s, rdb, server := readSeqTestCache(t, &readSeqStore{})
ctx := context.Background()
const high int64 = 9223372036854775807
key := s.getSeqUserReadSeqKey("group", "user")
require.NoError(t, rdb.HSet(ctx, key, "value", strconv.FormatInt(high, 10)).Err())
require.NoError(t, s.SetUserReadSeqs(ctx, "user", map[string]int64{"group": high - 1}))
require.Equal(t, strconv.FormatInt(high, 10), rdb.HGet(ctx, key, "value").Val())
require.Equal(t, s.readExpireTime, server.TTL(key))
}
func TestReadSeqFencesLegacyRocksCacheRefill(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
s, rdb, _ := readSeqTestCache(t, &readSeqStore{})
key := s.getSeqUserReadSeqKey("group", "user")
started, resume := make(chan struct{}), make(chan struct{})
done := make(chan error, 1)
go func() {
_, err := s.rocks.GetClient().Fetch2(ctx, key, time.Hour, func() (string, error) {
close(started)
select {
case <-resume:
case <-ctx.Done():
}
return "100", nil
})
done <- err
}()
select {
case <-started:
case <-ctx.Done():
t.Fatal("legacy refill did not start")
}
effective, writeErr := s.SetUserReadSeq(ctx, "group", "user", 346)
close(resume)
require.NoError(t, writeErr)
require.EqualValues(t, 346, effective)
require.NoError(t, <-done)
require.Equal(t, "346", rdb.HGet(ctx, key, "value").Val())
}
func TestReadSeqWithoutRedis(t *testing.T) {
var stored int64 = 346
store := &readSeqStore{
read: func(context.Context, string, string) (int64, error) { return stored, nil },
write: func(_ context.Context, _, _ string, seq int64) error { stored = max(stored, seq); return nil },
}
s := NewSeqUserCacheRedis(nil, store)
ctx := context.Background()
effective, err := s.SetUserReadSeq(ctx, "group", "user", 200)
require.NoError(t, err)
require.EqualValues(t, 346, effective)
require.NoError(t, s.SetUserReadSeqs(ctx, "user", map[string]int64{"group": 400}))
require.EqualValues(t, 400, stored)
}
func TestReadSeqMissingAndErrorCases(t *testing.T) {
t.Run("missing rows fill zero and duplicate IDs are deduplicated", func(t *testing.T) {
store := &readSeqStore{readBatch: func(_ context.Context, _ string, convs []string) (map[string]int64, error) {
require.Equal(t, []string{"group"}, convs)
return nil, nil
}}
s, _, _ := readSeqTestCache(t, store)
got, err := s.GetUserReadSeqs(context.Background(), "user", []string{"group", "group"})
require.NoError(t, err)
require.Equal(t, map[string]int64{"group": 0}, got)
})
t.Run("MongoDB errors do not populate the cache", func(t *testing.T) {
failure := fmt.Errorf("database unavailable")
store := &readSeqStore{read: func(context.Context, string, string) (int64, error) { return 0, failure }}
s, rdb, _ := readSeqTestCache(t, store)
_, err := s.SetUserReadSeq(context.Background(), "group", "user", 346)
require.ErrorIs(t, err, failure)
require.Zero(t, rdb.Exists(context.Background(), s.getSeqUserReadSeqKey("group", "user")).Val())
})
t.Run("negative writes are rejected", func(t *testing.T) {
s, _, _ := readSeqTestCache(t, &readSeqStore{})
_, err := s.SetUserReadSeq(context.Background(), "group", "user", -1)
require.Error(t, err)
require.Error(t, s.SetUserReadSeqs(context.Background(), "user", map[string]int64{"group": -1}))
})
}

View File

@ -8,7 +8,8 @@ type SeqUser interface {
GetUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error)
SetUserMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error
GetUserReadSeq(ctx context.Context, conversationID string, userID string) (int64, error)
SetUserReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error
// SetUserReadSeq advances read progress monotonically and returns its effective value.
SetUserReadSeq(ctx context.Context, conversationID string, userID string, seq int64) (int64, error)
SetUserReadSeqToDB(ctx context.Context, conversationID string, userID string, seq int64) error
SetUserMinSeqs(ctx context.Context, userID string, seqs map[string]int64) error
SetUserReadSeqs(ctx context.Context, userID string, seqs map[string]int64) error

View File

@ -71,7 +71,7 @@ type CommonMsgDatabase interface {
SetMinSeq(ctx context.Context, conversationID string, seq int64) error
SetUserConversationsMinSeqs(ctx context.Context, userID string, seqs map[string]int64) (err error)
SetHasReadSeq(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error
SetHasReadSeq(ctx context.Context, userID string, conversationID string, hasReadSeq int64) (int64, error)
GetHasReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error)
GetHasReadSeq(ctx context.Context, userID string, conversationID string) (int64, error)
UserSetHasReadSeqs(ctx context.Context, userID string, hasReadSeqs map[string]int64) error
@ -585,7 +585,7 @@ func (db *commonMsgDatabase) UserSetHasReadSeqs(ctx context.Context, userID stri
return db.seqUser.SetUserReadSeqs(ctx, userID, hasReadSeqs)
}
func (db *commonMsgDatabase) SetHasReadSeq(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error {
func (db *commonMsgDatabase) SetHasReadSeq(ctx context.Context, userID string, conversationID string, hasReadSeq int64) (int64, error) {
return db.seqUser.SetUserReadSeq(ctx, conversationID, userID, hasReadSeq)
}

View File

@ -238,9 +238,12 @@ func (db *msgTransferDatabase) BatchInsertChat2Cache(ctx context.Context, conver
func (db *msgTransferDatabase) SetHasReadSeqs(ctx context.Context, conversationID string, userSeqMap map[string]int64) error {
for userID, seq := range userSeqMap {
if err := db.seqUser.SetUserReadSeq(ctx, conversationID, userID, seq); err != nil {
effectiveSeq, err := db.seqUser.SetUserReadSeq(ctx, conversationID, userID, seq)
if err != nil {
return err
}
// The same map is queued for asynchronous persistence by msgtransfer.
userSeqMap[userID] = effectiveSeq
}
return nil
}

View File

@ -0,0 +1,22 @@
package controller
import (
"context"
"testing"
"github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache"
"github.com/stretchr/testify/require"
)
type effectiveReadSeqCache struct{ cache.SeqUser }
func (*effectiveReadSeqCache) SetUserReadSeq(context.Context, string, string, int64) (int64, error) {
return 346, nil
}
func TestReadSeqSenderQueuesEffectiveProgress(t *testing.T) {
db := &msgTransferDatabase{seqUser: &effectiveReadSeqCache{}}
pending := map[string]int64{"user": 200}
require.NoError(t, db.SetHasReadSeqs(context.Background(), "group", pending))
require.EqualValues(t, 346, pending["user"])
}

View File

@ -3,9 +3,11 @@ package mgo
import (
"context"
"errors"
"github.com/openimsdk/open-im-server/v3/pkg/common/storage/database"
"github.com/openimsdk/open-im-server/v3/pkg/common/storage/model"
"github.com/openimsdk/tools/db/mongoutil"
"github.com/openimsdk/tools/errs"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
@ -18,9 +20,13 @@ func NewSeqUserMongo(db *mongo.Database) (database.SeqUser, error) {
{Key: "user_id", Value: 1},
{Key: "conversation_id", Value: 1},
},
// A distinct name allows the old non-unique index to coexist on
// supported MongoDB versions. Existing duplicate rows must be resolved
// before rollout; never silently pick one read sequence from duplicates.
Options: options.Index().SetName("user_conversation_unique").SetUnique(true),
})
if err != nil {
return nil, err
return nil, errs.WrapMsg(err, "ensure unique seq_user index; check duplicate user_id/conversation_id rows before rollout")
}
return &seqUserMongo{coll: coll}, nil
}
@ -30,6 +36,10 @@ type seqUserMongo struct {
}
func (s *seqUserMongo) setSeq(ctx context.Context, conversationID string, userID string, seq int64, field string) error {
return s.updateSeq(ctx, conversationID, userID, seq, field, "$set")
}
func (s *seqUserMongo) updateSeq(ctx context.Context, conversationID, userID string, seq int64, field, operator string) error {
filter := map[string]any{
"user_id": userID,
"conversation_id": conversationID,
@ -43,13 +53,19 @@ func (s *seqUserMongo) setSeq(ctx context.Context, conversationID string, userID
}
delete(insert, field)
update := map[string]any{
"$set": bson.M{
operator: bson.M{
field: seq,
},
"$setOnInsert": insert,
}
opt := options.Update().SetUpsert(true)
return mongoutil.UpdateOne(ctx, s.coll, filter, update, false, opt)
err := mongoutil.UpdateOne(ctx, s.coll, filter, update, false, opt)
if mongo.IsDuplicateKeyError(err) {
// Another writer may have inserted the same user/conversation during
// this upsert. Retry against that row without creating another one.
return mongoutil.UpdateOne(ctx, s.coll, filter, update, true)
}
return err
}
func (s *seqUserMongo) getSeq(ctx context.Context, conversationID string, userID string, failed string) (int64, error) {
@ -115,12 +131,8 @@ func (s *seqUserMongo) GetUserReadSeqs(ctx context.Context, userID string, conve
}
func (s *seqUserMongo) SetUserReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error {
dbSeq, err := s.GetUserReadSeq(ctx, conversationID, userID)
if err != nil {
return err
if seq < 0 {
return errs.ErrArgs.WrapMsg("read seq must not be negative")
}
if dbSeq > seq {
return nil
}
return s.setSeq(ctx, conversationID, userID, seq, "read_seq")
return s.updateSeq(ctx, conversationID, userID, seq, "read_seq", "$max")
}

View File

@ -0,0 +1,85 @@
package mgo
import (
"context"
"testing"
"github.com/stretchr/testify/require"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo/integration/mtest"
)
func TestReadSeqMongoAtomicMax(t *testing.T) {
mt := mtest.New(t, mtest.NewOptions().ClientType(mtest.Mock))
mt.Run("out of order receipts use max without a preceding read", func(mt *mtest.T) {
s := &seqUserMongo{coll: mt.Coll}
for _, seq := range []int64{346, 200, 346} {
mt.AddMockResponses(mtest.CreateSuccessResponse(bson.E{Key: "n", Value: 1}, bson.E{Key: "nModified", Value: 1}))
require.NoError(mt, s.SetUserReadSeq(context.Background(), "group", "user", seq))
}
events := mt.GetAllStartedEvents()
require.Len(mt, events, 3)
for i, event := range events {
require.Equal(mt, "update", event.CommandName)
updates, err := event.Command.Lookup("updates").Array().Values()
require.NoError(mt, err)
update := updates[0].Document()
require.Equal(mt, "group", update.Lookup("q", "conversation_id").StringValue())
require.Equal(mt, "user", update.Lookup("q", "user_id").StringValue())
require.Equal(mt, []int64{346, 200, 346}[i], update.Lookup("u", "$max", "read_seq").Int64())
_, err = update.LookupErr("u", "$setOnInsert", "read_seq")
require.Error(mt, err, "insertion defaults must not conflict with $max")
require.True(mt, update.Lookup("upsert").Boolean())
}
})
mt.Run("concurrent upsert retries the existing unique row", func(mt *mtest.T) {
mt.AddMockResponses(
mtest.CreateWriteErrorsResponse(mtest.WriteError{Index: 0, Code: 11000, Message: "duplicate user/conversation"}),
mtest.CreateSuccessResponse(bson.E{Key: "n", Value: 1}, bson.E{Key: "nModified", Value: 1}),
)
s := &seqUserMongo{coll: mt.Coll}
require.NoError(mt, s.SetUserReadSeq(context.Background(), "group", "user", 346))
events := mt.GetAllStartedEvents()
require.Len(mt, events, 2)
updates, err := events[1].Command.Lookup("updates").Array().Values()
require.NoError(mt, err)
update := updates[0].Document()
upsert, err := update.LookupErr("upsert")
require.True(mt, err != nil || !upsert.Boolean())
require.EqualValues(mt, 346, update.Lookup("u", "$max", "read_seq").Int64())
})
mt.Run("database failures propagate", func(mt *mtest.T) {
mt.AddMockResponses(mtest.CreateCommandErrorResponse(mtest.CommandError{Code: 91, Message: "unavailable"}))
s := &seqUserMongo{coll: mt.Coll}
require.Error(mt, s.SetUserReadSeq(context.Background(), "group", "user", 346))
})
mt.Run("negative sequences do not issue a command", func(mt *mtest.T) {
s := &seqUserMongo{coll: mt.Coll}
require.Error(mt, s.SetUserReadSeq(context.Background(), "group", "user", -1))
require.Empty(mt, mt.GetAllStartedEvents())
})
}
func TestReadSeqMongoUniqueIndex(t *testing.T) {
mt := mtest.New(t, mtest.NewOptions().ClientType(mtest.Mock))
mt.Run("one record per user and conversation", func(mt *mtest.T) {
mt.AddMockResponses(mtest.CreateSuccessResponse())
_, err := NewSeqUserMongo(mt.DB)
require.NoError(mt, err)
events := mt.GetAllStartedEvents()
require.Len(mt, events, 1)
require.Equal(mt, "createIndexes", events[0].CommandName)
indexes, err := events[0].Command.Lookup("indexes").Array().Values()
require.NoError(mt, err)
index := indexes[0].Document()
require.True(mt, index.Lookup("unique").Boolean())
require.Equal(mt, "user_conversation_unique", index.Lookup("name").StringValue())
require.EqualValues(mt, 1, index.Lookup("key", "user_id").Int32())
require.EqualValues(mt, 1, index.Lookup("key", "conversation_id").Int32())
})
mt.Run("existing duplicates fail explicitly", func(mt *mtest.T) {
mt.AddMockResponses(mtest.CreateCommandErrorResponse(mtest.CommandError{Code: 11000, Message: "duplicate rows"}))
_, err := NewSeqUserMongo(mt.DB)
require.ErrorContains(mt, err, "check duplicate user_id/conversation_id rows before rollout")
})
}