From e3bd4ea48d2c7a403c335c55332a1cfe9e67648d Mon Sep 17 00:00:00 2001 From: john Date: Mon, 19 Oct 2026 15:35:57 +0800 Subject: [PATCH] fix(msg): keep read sequences monotonic across devices --- docs/contrib/read-seq-rollout.md | 48 +++ go.mod | 3 + go.sum | 6 + internal/rpc/msg/as_read.go | 33 +-- internal/rpc/msg/as_read_monotonic_test.go | 107 +++++++ pkg/common/storage/cache/redis/seq_user.go | 233 ++++++++++----- .../cache/redis/seq_user_monotonic_test.go | 274 ++++++++++++++++++ pkg/common/storage/cache/seq_user.go | 3 +- pkg/common/storage/controller/msg.go | 4 +- pkg/common/storage/controller/msg_transfer.go | 5 +- .../controller/msg_transfer_read_seq_test.go | 22 ++ pkg/common/storage/database/mgo/seq_user.go | 32 +- .../database/mgo/seq_user_monotonic_test.go | 85 ++++++ 13 files changed, 745 insertions(+), 110 deletions(-) create mode 100644 docs/contrib/read-seq-rollout.md create mode 100644 internal/rpc/msg/as_read_monotonic_test.go create mode 100644 pkg/common/storage/cache/redis/seq_user_monotonic_test.go create mode 100644 pkg/common/storage/controller/msg_transfer_read_seq_test.go create mode 100644 pkg/common/storage/database/mgo/seq_user_monotonic_test.go diff --git a/docs/contrib/read-seq-rollout.md b/docs/contrib/read-seq-rollout.md new file mode 100644 index 000000000..30bff9c1f --- /dev/null +++ b/docs/contrib/read-seq-rollout.md @@ -0,0 +1,48 @@ +# Rolling out monotonic read sequences + +Read progress is shared by a user across devices. Redis updates, cache fills, and +MongoDB writes must retain the greatest confirmed `read_seq` for each +`(user_id, conversation_id)`. Read-receipt notifications continue to use the +existing asynchronous queue and MQ pipeline. + +## MongoDB preflight + +The server creates a unique compound index named `user_conversation_unique` on +`seq_user`. The explicit name allows it to coexist with the previous non-unique +index on MongoDB 5.0 and later (the documented deployment uses MongoDB 6.0). +The old index is not automatically dropped. + +Before deploying, check for duplicate rows in the intended OpenIM database: + +```javascript +db.seq_user.aggregate([ + { $group: { + _id: { user_id: "$user_id", conversation_id: "$conversation_id" }, + count: { $sum: 1 } + } }, + { $match: { count: { $gt: 1 } } }, + { $limit: 20 } +]) +``` + +Resolve duplicates through a reviewed data migration before rollout, preserving +the greatest confirmed `read_seq` and reviewing the independent `min_seq` and +`max_seq` bounds. Coordinate the migration with writers. The application does not +delete or merge historical records automatically; index creation fails explicitly +if duplicates remain. Check index build time and permissions for the deployment. +If an equivalent unique index has already been installed manually, reconcile its +name with `user_conversation_unique` before rollout. + +## Redis compatibility and limits + +Read-sequence keys keep the existing Redis hash `value` field and 30-day write +TTL. Their dedicated monotonic fill path replaces generic RocksCache fills; +updating a value also invalidates an older RocksCache fill owner. Complete the +rollout across all writers, since old processes can still perform unconditional +writes. Other cached fields retain their existing cache behavior. + +Atomic maximum updates tolerate concurrent devices, duplicate receipts, and +out-of-order persistence. They do not repair a previously lost persistence event +or guarantee delivery through an in-memory queue. Reliable retry/compensation and +reconciliation of existing Redis/MongoDB differences remain separate work. Do not +mark all messages as read to repair a discrepancy; use confirmed read progress. diff --git a/go.mod b/go.mod index 8a8dae98f..34650f500 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.25.0 require ( firebase.google.com/go/v4 v4.14.1 + github.com/alicebob/miniredis/v2 v2.34.0 github.com/dtm-labs/rockscache v0.1.1 github.com/gin-gonic/gin v1.9.1 github.com/go-playground/validator/v10 v10.20.0 @@ -56,6 +57,7 @@ require ( cloud.google.com/go/longrunning v0.5.5 // indirect cloud.google.com/go/storage v1.40.0 // indirect github.com/MicahParks/keyfunc v1.9.0 // indirect + github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302 // indirect github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible // indirect github.com/aws/aws-sdk-go-v2 v1.32.5 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1 // indirect @@ -182,6 +184,7 @@ require ( github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect + github.com/yuin/gopher-lua v1.1.1 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect go.etcd.io/etcd/api/v3 v3.5.13 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.13 // indirect diff --git a/go.sum b/go.sum index 5d0c9e012..9d9673c4c 100644 --- a/go.sum +++ b/go.sum @@ -19,6 +19,10 @@ github.com/IBM/sarama v1.43.0/go.mod h1:zlE6HEbC/SMQ9mhEYaF7nNLYOUyrs0obySKCckWP github.com/MicahParks/keyfunc v1.9.0 h1:lhKd5xrFHLNOWrDc4Tyb/Q1AJ4LCzQ48GVJyVIID3+o= github.com/MicahParks/keyfunc v1.9.0/go.mod h1:IdnCilugA0O/99dW+/MkvlyrsX8+L8+x95xuVNtM5jw= github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM= +github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302 h1:uvdUDbHQHO85qeSydJtItA4T55Pw6BtAejd0APRJOCE= +github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= +github.com/alicebob/miniredis/v2 v2.34.0 h1:mBFWMaJSNL9RwdGRyEDoAAv8OQc5UlEhLDQggTglU/0= +github.com/alicebob/miniredis/v2 v2.34.0/go.mod h1:kWShP4b58T1CW0Y5dViCd5ztzrDqRWqM3nksiyXk5s8= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/aws/aws-sdk-go-v2 v1.32.5 h1:U8vdWJuY7ruAkzaOdD7guwJjD06YSKmnKCJs7s3IkIo= @@ -478,6 +482,8 @@ github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7Jul github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= +github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.etcd.io/etcd/api/v3 v3.5.13 h1:8WXU2/NBge6AUF1K1gOexB6e07NgsN1hXK0rSTtgSp4= diff --git a/internal/rpc/msg/as_read.go b/internal/rpc/msg/as_read.go index c52ce9c07..68048c12b 100644 --- a/internal/rpc/msg/as_read.go +++ b/internal/rpc/msg/as_read.go @@ -96,10 +96,11 @@ func (m *msgServer) SetConversationHasReadSeq(ctx context.Context, req *msg.SetC if req.HasReadSeq > maxSeq { return nil, errs.ErrArgs.WrapMsg("hasReadSeq must not be bigger than maxSeq") } - if err := m.MsgDatabase.SetHasReadSeq(ctx, req.UserID, req.ConversationID, req.HasReadSeq); err != nil { + hasReadSeq, err := m.MsgDatabase.SetHasReadSeq(ctx, req.UserID, req.ConversationID, req.HasReadSeq) + if err != nil { return nil, err } - m.sendMarkAsReadNotification(ctx, req.ConversationID, constant.SingleChatType, req.UserID, req.UserID, nil, req.HasReadSeq) + m.sendMarkAsReadNotification(ctx, req.ConversationID, constant.SingleChatType, req.UserID, req.UserID, nil, hasReadSeq) return &msg.SetConversationHasReadSeqResp{}, nil } @@ -122,16 +123,10 @@ func (m *msgServer) MarkMsgsAsRead(ctx context.Context, req *msg.MarkMsgsAsReadR if err := m.MsgDatabase.MarkSingleChatMsgsAsRead(ctx, req.UserID, req.ConversationID, req.Seqs); err != nil { return nil, err } - currentHasReadSeq, err := m.MsgDatabase.GetHasReadSeq(ctx, req.UserID, req.ConversationID) - if err != nil && !errors.Is(err, redis.Nil) { + hasReadSeq, err = m.MsgDatabase.SetHasReadSeq(ctx, req.UserID, req.ConversationID, hasReadSeq) + if err != nil { return nil, err } - if hasReadSeq > currentHasReadSeq { - err = m.MsgDatabase.SetHasReadSeq(ctx, req.UserID, req.ConversationID, hasReadSeq) - if err != nil { - return nil, err - } - } reqCallback := &cbapi.CallbackSingleMsgReadReq{ ConversationID: conversation.ConversationID, @@ -176,23 +171,17 @@ func (m *msgServer) MarkConversationAsRead(ctx context.Context, req *msg.MarkCon return nil, err } } - if req.HasReadSeq > hasReadSeq { - err = m.MsgDatabase.SetHasReadSeq(ctx, req.UserID, req.ConversationID, req.HasReadSeq) - if err != nil { - return nil, err - } - hasReadSeq = req.HasReadSeq + hasReadSeq, err = m.MsgDatabase.SetHasReadSeq(ctx, req.UserID, req.ConversationID, req.HasReadSeq) + if err != nil { + return nil, err } m.sendMarkAsReadNotification(ctx, req.ConversationID, conversation.ConversationType, req.UserID, m.conversationAndGetRecvID(conversation, req.UserID), seqs, hasReadSeq) } else if conversation.ConversationType == constant.ReadGroupChatType || conversation.ConversationType == constant.NotificationChatType { - if req.HasReadSeq > hasReadSeq { - err = m.MsgDatabase.SetHasReadSeq(ctx, req.UserID, req.ConversationID, req.HasReadSeq) - if err != nil { - return nil, err - } - hasReadSeq = req.HasReadSeq + hasReadSeq, err = m.MsgDatabase.SetHasReadSeq(ctx, req.UserID, req.ConversationID, req.HasReadSeq) + if err != nil { + return nil, err } m.sendMarkAsReadNotification(ctx, req.ConversationID, constant.SingleChatType, req.UserID, req.UserID, seqs, hasReadSeq) diff --git a/internal/rpc/msg/as_read_monotonic_test.go b/internal/rpc/msg/as_read_monotonic_test.go new file mode 100644 index 000000000..aae2c45af --- /dev/null +++ b/internal/rpc/msg/as_read_monotonic_test.go @@ -0,0 +1,107 @@ +package msg + +import ( + "context" + "encoding/json" + "fmt" + "testing" + "time" + + "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" + "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" + "github.com/openimsdk/open-im-server/v3/pkg/notification" + "github.com/openimsdk/open-im-server/v3/pkg/rpccache" + "github.com/openimsdk/open-im-server/v3/pkg/rpcli" + "github.com/openimsdk/protocol/constant" + "github.com/openimsdk/protocol/conversation" + pbmsg "github.com/openimsdk/protocol/msg" + "github.com/openimsdk/protocol/sdkws" + "github.com/openimsdk/tools/mcontext" + "github.com/openimsdk/tools/mq/memamq" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" +) + +type receiptDatabase struct { + controller.CommonMsgDatabase + calls int + initial int64 +} + +func (*receiptDatabase) GetMaxSeq(context.Context, string) (int64, error) { return 346, nil } +func (db *receiptDatabase) GetHasReadSeq(context.Context, string, string) (int64, error) { + return db.initial, nil +} +func (db *receiptDatabase) SetHasReadSeq(context.Context, string, string, int64) (int64, error) { + db.calls++ + return 346, nil // Another device advanced after the RPC's initial read. +} +func (*receiptDatabase) MarkSingleChatMsgsAsRead(context.Context, string, string, []int64) error { + return nil +} + +type receiptConversationClient struct { + conversation.ConversationClient + sessionType int32 +} + +func (c *receiptConversationClient) GetConversation(_ context.Context, req *conversation.GetConversationReq, _ ...grpc.CallOption) (*conversation.GetConversationResp, error) { + return &conversation.GetConversationResp{Conversation: &conversation.Conversation{ + ConversationID: req.ConversationID, OwnerUserID: req.OwnerUserID, + ConversationType: c.sessionType, UserID: "peer", GroupID: "group", + }}, nil +} + +func TestReadSeqReceiptsUseEffectiveProgress(t *testing.T) { + for _, method := range []string{"set", "messages", "single conversation", "group conversation"} { + for _, requested := range []int64{200, 346} { + for _, initial := range []int64{100, 346} { + t.Run(fmt.Sprintf("%s/initial=%d/requested=%d", method, initial, requested), func(t *testing.T) { + ctx := mcontext.WithOpUserIDContext(context.Background(), "user") + db := &receiptDatabase{initial: initial} + queue := memamq.NewMemoryQueue(1, 8) + defer queue.Stop() + sent := make(chan *sdkws.MsgData, 1) + notifier := notification.NewNotificationSender(&config.Notification{}, notification.WithQueue(queue), + notification.WithLocalSendMsg(func(_ context.Context, req *pbmsg.SendMsgReq) (*pbmsg.SendMsgResp, error) { + sent <- req.MsgData + return &pbmsg.SendMsgResp{}, nil + })) + sessionType := int32(constant.SingleChatType) + if method == "group conversation" { + sessionType = constant.ReadGroupChatType + } + client := &rpcli.ConversationClient{ConversationClient: &receiptConversationClient{sessionType: sessionType}} + server := &msgServer{MsgDatabase: db, notificationSender: notifier, config: &Config{}, + ConversationLocalCache: rpccache.NewConversationLocalCache(client, &config.LocalCache{}, nil), + webhookClient: webhook.NewWebhookClient("", queue), + } + var err error + switch method { + case "set": + _, err = server.SetConversationHasReadSeq(ctx, &pbmsg.SetConversationHasReadSeqReq{UserID: "user", ConversationID: "group", HasReadSeq: requested}) + case "messages": + _, err = server.MarkMsgsAsRead(ctx, &pbmsg.MarkMsgsAsReadReq{UserID: "user", ConversationID: "group", Seqs: []int64{requested}}) + default: + _, err = server.MarkConversationAsRead(ctx, &pbmsg.MarkConversationAsReadReq{UserID: "user", ConversationID: "group", HasReadSeq: requested}) + } + require.NoError(t, err) + require.Equal(t, 1, db.calls, "equal reports must still allow persistence/notification repair") + select { + case message := <-sent: + require.EqualValues(t, constant.HasReadReceipt, message.ContentType) + var elem sdkws.NotificationElem + require.NoError(t, json.Unmarshal(message.Content, &elem)) + var tips sdkws.MarkAsReadTips + require.NoError(t, json.Unmarshal([]byte(elem.Detail), &tips)) + require.EqualValues(t, 346, tips.HasReadSeq) + require.Equal(t, "group", tips.ConversationID) + case <-time.After(3 * time.Second): + t.Fatal("read receipt was not emitted") + } + }) + } + } + } +} diff --git a/pkg/common/storage/cache/redis/seq_user.go b/pkg/common/storage/cache/redis/seq_user.go index af9cbef5a..da3bb195e 100644 --- a/pkg/common/storage/cache/redis/seq_user.go +++ b/pkg/common/storage/cache/redis/seq_user.go @@ -65,26 +65,110 @@ func (s *seqUserCacheRedis) SetUserMinSeq(ctx context.Context, conversationID st return s.SetUserMinSeqs(ctx, userID, map[string]int64{conversationID: seq}) } +// Read sequences use their own monotonic fill path. A generic RocksCache +// refill can otherwise overwrite a concurrent read receipt with an older DB value. func (s *seqUserCacheRedis) GetUserReadSeq(ctx context.Context, conversationID string, userID string) (int64, error) { - return getCache(ctx, s.rocks, s.getSeqUserReadSeqKey(conversationID, userID), s.readExpireTime, func(ctx context.Context) (int64, error) { + if s.rocks.Disable() { return s.mgo.GetUserReadSeq(ctx, conversationID, userID) - }) + } + key := s.getSeqUserReadSeqKey(conversationID, userID) + value, err := s.rocks.GetRedis().HGet(ctx, key, "value").Result() + if err == nil && value != "" { + return strconv.ParseInt(value, 10, 64) + } + if err != nil && err != redis.Nil { + return 0, errs.Wrap(err) + } + seq, err := s.mgo.GetUserReadSeq(ctx, conversationID, userID) + if err != nil { + return 0, err + } + return s.mergeReadSeq(ctx, key, seq) } -func (s *seqUserCacheRedis) SetUserReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error { - if s.rocks.GetRedis() == nil { - return s.SetUserReadSeqToDB(ctx, conversationID, userID, seq) +// SetUserReadSeq returns the effective sequence, including any concurrent advance. +func (s *seqUserCacheRedis) SetUserReadSeq(ctx context.Context, conversationID string, userID string, seq int64) (int64, error) { + if seq < 0 { + return 0, errs.ErrArgs.WrapMsg("read seq must not be negative") } - dbSeq, err := s.GetUserReadSeq(ctx, conversationID, userID) + if s.rocks.Disable() { + if err := s.SetUserReadSeqToDB(ctx, conversationID, userID, seq); err != nil { + return 0, err + } + return s.mgo.GetUserReadSeq(ctx, conversationID, userID) + } + // Preserve the MongoDB baseline on a cold cache, as well as the value read + // before a key expires between this read and the atomic update. + current, err := s.GetUserReadSeq(ctx, conversationID, userID) if err != nil { - return err + return 0, err } - if dbSeq < seq { - if err := s.rocks.GetClient().RawSet(ctx, s.getSeqUserReadSeqKey(conversationID, userID), strconv.Itoa(int(seq)), s.readExpireTime); err != nil { - return errs.Wrap(err) + return s.mergeReadSeq(ctx, s.getSeqUserReadSeqKey(conversationID, userID), max(current, seq)) +} + +// Compare decimal strings rather than Lua doubles to preserve the full int64 range. +// Removing the refill owner also fences any RocksCache fill started by an older +// server instance before this write during a rolling upgrade. +var mergeReadSeqScript = redis.NewScript(` +local incoming = ARGV[1] +local current = redis.call('HGET', KEYS[1], 'value') +if current and current ~= '' then + if not string.match(current, '^%d+$') then + return redis.error_reply('invalid cached read sequence') + end + if #current > #incoming or (#current == #incoming and current > incoming) then + incoming = current + end +end +redis.call('HSET', KEYS[1], 'value', incoming) +redis.call('HDEL', KEYS[1], 'lockUntil', 'lockOwner') +redis.call('PEXPIRE', KEYS[1], ARGV[2]) +return incoming +`) + +func (s *seqUserCacheRedis) mergeReadSeq(ctx context.Context, key string, seq int64) (int64, error) { + if seq < 0 { + return 0, errs.ErrArgs.WrapMsg("read seq must not be negative") + } + value, err := mergeReadSeqScript.Run(ctx, s.rocks.GetRedis(), []string{key}, + strconv.FormatInt(seq, 10), s.readExpireTime.Milliseconds()).Text() + if err != nil { + return 0, errs.Wrap(err) + } + return strconv.ParseInt(value, 10, 64) +} + +func (s *seqUserCacheRedis) mergeReadSeqs(ctx context.Context, userID string, seqs map[string]int64) (map[string]int64, error) { + pipe := s.rocks.GetRedis().Pipeline() + commands := make(map[string]*redis.Cmd, len(seqs)) + for conversationID, seq := range seqs { + if seq < 0 { + return nil, errs.ErrArgs.WrapMsg("read seq must not be negative") + } + // Each script touches a single key, so ClusterClient can route the + // pipeline across slots without a cross-slot transaction. + commands[conversationID] = mergeReadSeqScript.Eval(ctx, pipe, + []string{s.getSeqUserReadSeqKey(conversationID, userID)}, + strconv.FormatInt(seq, 10), s.readExpireTime.Milliseconds()) + } + if len(commands) > 0 { + if _, err := pipe.Exec(ctx); err != nil { + return nil, errs.Wrap(err) } } - return nil + result := make(map[string]int64, len(commands)) + for conversationID, cmd := range commands { + value, err := cmd.Text() + if err != nil { + return nil, errs.Wrap(err) + } + seq, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return nil, errs.Wrap(err) + } + result[conversationID] = seq + } + return result, nil } func (s *seqUserCacheRedis) SetUserReadSeqToDB(ctx context.Context, conversationID string, userID string, seq int64) error { @@ -102,83 +186,84 @@ func (s *seqUserCacheRedis) SetUserMinSeqs(ctx context.Context, userID string, s return DeleteCacheBySlot(ctx, s.rocks, keys) } -func (s *seqUserCacheRedis) setUserRedisReadSeqs(ctx context.Context, userID string, seqs map[string]int64) error { - keys := make([]string, 0, len(seqs)) - keySeq := make(map[string]int64) +func (s *seqUserCacheRedis) SetUserReadSeqs(ctx context.Context, userID string, seqs map[string]int64) error { + conversationIDs := make([]string, 0, len(seqs)) for conversationID, seq := range seqs { - key := s.getSeqUserReadSeqKey(conversationID, userID) - keys = append(keys, key) - keySeq[key] = seq + if seq < 0 { + return errs.ErrArgs.WrapMsg("read seq must not be negative") + } + conversationIDs = append(conversationIDs, conversationID) } - slotKeys, err := groupKeysBySlot(ctx, s.rocks.GetRedis(), keys) + if s.rocks.Disable() { + for conversationID, seq := range seqs { + if err := s.SetUserReadSeqToDB(ctx, conversationID, userID, seq); err != nil { + return err + } + } + return nil + } + current, err := s.GetUserReadSeqs(ctx, userID, conversationIDs) if err != nil { return err } - for _, keys := range slotKeys { - pipe := s.rocks.GetRedis().Pipeline() - for _, key := range keys { - pipe.HSet(ctx, key, "value", strconv.FormatInt(keySeq[key], 10)) - pipe.Expire(ctx, key, s.readExpireTime) - } - if _, err := pipe.Exec(ctx); err != nil { - return err - } + updates := make(map[string]int64, len(seqs)) + for conversationID, seq := range seqs { + updates[conversationID] = max(current[conversationID], seq) } - return nil -} - -func (s *seqUserCacheRedis) SetUserReadSeqs(ctx context.Context, userID string, seqs map[string]int64) error { - if len(seqs) == 0 { - return nil - } - if err := s.setUserRedisReadSeqs(ctx, userID, seqs); err != nil { - return err - } - return nil + _, err = s.mergeReadSeqs(ctx, userID, updates) + return err } func (s *seqUserCacheRedis) GetUserReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) { - res, err := batchGetCache2(ctx, s.rocks, s.readExpireTime, conversationIDs, func(conversationID string) string { - return s.getSeqUserReadSeqKey(conversationID, userID) - }, func(v *readSeqModel) string { - return v.ConversationID - }, func(ctx context.Context, conversationIDs []string) ([]*readSeqModel, error) { - seqs, err := s.mgo.GetUserReadSeqs(ctx, userID, conversationIDs) + if s.rocks.Disable() { + return s.mgo.GetUserReadSeqs(ctx, userID, conversationIDs) + } + pipe := s.rocks.GetRedis().Pipeline() + commands := make(map[string]*redis.StringCmd, len(conversationIDs)) + for _, conversationID := range conversationIDs { + if _, ok := commands[conversationID]; !ok { + commands[conversationID] = pipe.HGet(ctx, s.getSeqUserReadSeqKey(conversationID, userID), "value") + } + } + if len(commands) > 0 { + if _, err := pipe.Exec(ctx); err != nil && err != redis.Nil { + return nil, errs.Wrap(err) + } + } + result := make(map[string]int64, len(commands)) + missing := make([]string, 0) + for conversationID, cmd := range commands { + value, err := cmd.Result() + if err == redis.Nil || (err == nil && value == "") { + missing = append(missing, conversationID) + continue + } if err != nil { - return nil, err + return nil, errs.Wrap(err) } - res := make([]*readSeqModel, 0, len(seqs)) - for conversationID, seq := range seqs { - res = append(res, &readSeqModel{ConversationID: conversationID, Seq: seq}) + seq, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return nil, errs.Wrap(err) } - return res, nil - }) + result[conversationID] = seq + } + if len(missing) == 0 { + return result, nil + } + stored, err := s.mgo.GetUserReadSeqs(ctx, userID, missing) if err != nil { return nil, err } - data := make(map[string]int64) - for _, v := range res { - data[v.ConversationID] = v.Seq + fills := make(map[string]int64, len(missing)) + for _, conversationID := range missing { + fills[conversationID] = stored[conversationID] } - return data, nil -} - -var _ BatchCacheCallback[string] = (*readSeqModel)(nil) - -type readSeqModel struct { - ConversationID string - Seq int64 -} - -func (r *readSeqModel) BatchCache(conversationID string) { - r.ConversationID = conversationID -} - -func (r *readSeqModel) UnmarshalJSON(bytes []byte) (err error) { - r.Seq, err = strconv.ParseInt(string(bytes), 10, 64) - return -} - -func (r *readSeqModel) MarshalJSON() ([]byte, error) { - return []byte(strconv.FormatInt(r.Seq, 10)), nil + filled, err := s.mergeReadSeqs(ctx, userID, fills) + if err != nil { + return nil, err + } + for conversationID, seq := range filled { + result[conversationID] = seq + } + return result, nil } diff --git a/pkg/common/storage/cache/redis/seq_user_monotonic_test.go b/pkg/common/storage/cache/redis/seq_user_monotonic_test.go new file mode 100644 index 000000000..a795f0518 --- /dev/null +++ b/pkg/common/storage/cache/redis/seq_user_monotonic_test.go @@ -0,0 +1,274 @@ +package redis + +import ( + "context" + "fmt" + "strconv" + "sync" + "testing" + "time" + + "github.com/alicebob/miniredis/v2" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/require" +) + +type readSeqStore struct { + database.SeqUser + read func(context.Context, string, string) (int64, error) + readBatch func(context.Context, string, []string) (map[string]int64, error) + write func(context.Context, string, string, int64) error +} + +func (s *readSeqStore) SetUserReadSeq(ctx context.Context, conv, user string, seq int64) error { + return s.write(ctx, conv, user, seq) +} + +func (s *readSeqStore) GetUserReadSeq(ctx context.Context, conv, user string) (int64, error) { + if s.read != nil { + return s.read(ctx, conv, user) + } + return 100, nil +} + +func (s *readSeqStore) GetUserReadSeqs(ctx context.Context, user string, convs []string) (map[string]int64, error) { + if s.readBatch != nil { + return s.readBatch(ctx, user, convs) + } + result := make(map[string]int64, len(convs)) + for _, conv := range convs { + result[conv] = 100 + } + return result, nil +} + +func readSeqTestCache(t *testing.T, store database.SeqUser) (*seqUserCacheRedis, *redis.Client, *miniredis.Miniredis) { + t.Helper() + server := miniredis.RunT(t) + rdb := redis.NewClient(&redis.Options{Addr: server.Addr()}) + t.Cleanup(func() { _ = rdb.Close() }) + return NewSeqUserCacheRedis(rdb, store).(*seqUserCacheRedis), rdb, server +} + +// Delay the lower writer at the Redis command boundary, after its cache read. +// This covers both the old HSET and the atomic-script implementation. +type pauseReadSeqWrite struct { + started chan struct{} + resume chan struct{} + once sync.Once +} + +func (h *pauseReadSeqWrite) DialHook(next redis.DialHook) redis.DialHook { return next } +func (h *pauseReadSeqWrite) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook { + return next +} +func (h *pauseReadSeqWrite) ProcessHook(next redis.ProcessHook) redis.ProcessHook { + return func(ctx context.Context, cmd redis.Cmder) error { + if cmd.Name() == "hset" || cmd.Name() == "evalsha" || cmd.Name() == "eval" { + for _, arg := range cmd.Args() { + if fmt.Sprint(arg) == "200" { + h.once.Do(func() { + close(h.started) + select { + case <-h.resume: + case <-ctx.Done(): + } + }) + break + } + } + } + return next(ctx, cmd) + } +} + +// Keep a compile-time check that this hook remains usable by go-redis. +var _ redis.Hook = (*pauseReadSeqWrite)(nil) + +func TestReadSeqConcurrentWritersNeverRegress(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + s, rdb, _ := readSeqTestCache(t, &readSeqStore{}) + key := s.getSeqUserReadSeqKey("group", "user") + require.NoError(t, rdb.HSet(ctx, key, "value", "100").Err()) + hook := &pauseReadSeqWrite{started: make(chan struct{}), resume: make(chan struct{})} + rdb.AddHook(hook) + done := make(chan error, 1) + go func() { _, err := s.SetUserReadSeq(ctx, "group", "user", 200); done <- err }() + select { + case <-hook.started: + case <-ctx.Done(): + t.Fatal("lower writer never reached Redis") + } + highSeq, highErr := s.SetUserReadSeq(ctx, "group", "user", 346) + close(hook.resume) + require.NoError(t, highErr) + require.EqualValues(t, 346, highSeq) + require.NoError(t, <-done) + require.Equal(t, "346", rdb.HGet(ctx, key, "value").Val()) +} + +func TestReadSeqRefillDoesNotOverwriteNewerProgress(t *testing.T) { + for _, batch := range []bool{false, true} { + t.Run(fmt.Sprintf("batch=%v", batch), func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + started, resume := make(chan struct{}), make(chan struct{}) + var once sync.Once + wait := func() { + once.Do(func() { + close(started) + select { + case <-resume: + case <-ctx.Done(): + } + }) + } + store := &readSeqStore{} + if batch { + store.readBatch = func(context.Context, string, []string) (map[string]int64, error) { + wait() + return map[string]int64{"group": 100}, nil + } + } else { + store.read = func(context.Context, string, string) (int64, error) { wait(); return 100, nil } + } + s, rdb, _ := readSeqTestCache(t, store) + type result struct { + seq int64 + err error + } + done := make(chan result, 1) + go func() { + if batch { + values, err := s.GetUserReadSeqs(ctx, "user", []string{"group"}) + done <- result{values["group"], err} + } else { + seq, err := s.GetUserReadSeq(ctx, "group", "user") + done <- result{seq, err} + } + }() + select { + case <-started: + case <-ctx.Done(): + t.Fatal("refill never reached MongoDB") + } + // Seed the new baseline as another request completes its refill, then + // advance it through the real writer while the older query is pending. + key := s.getSeqUserReadSeqKey("group", "user") + require.NoError(t, rdb.HSet(ctx, key, "value", "100").Err()) + require.NoError(t, rdb.HDel(ctx, key, "lockUntil", "lockOwner").Err()) + writeErr := s.SetUserReadSeqs(ctx, "user", map[string]int64{"group": 346}) + close(resume) + require.NoError(t, writeErr) + got := <-done + require.NoError(t, got.err) + require.EqualValues(t, 346, got.seq, "the response must include the concurrent advancement") + require.Equal(t, "346", rdb.HGet(ctx, key, "value").Val()) + }) + } +} + +func TestReadSeqBatchPreservesMongoBaseline(t *testing.T) { + store := &readSeqStore{readBatch: func(context.Context, string, []string) (map[string]int64, error) { + return map[string]int64{"group": 346}, nil + }} + s, rdb, _ := readSeqTestCache(t, store) + ctx := context.Background() + require.NoError(t, s.SetUserReadSeqs(ctx, "user", map[string]int64{"group": 200})) + require.Equal(t, "346", rdb.HGet(ctx, s.getSeqUserReadSeqKey("group", "user"), "value").Val()) +} + +func TestReadSeqBatchNeverRegresses(t *testing.T) { + s, rdb, _ := readSeqTestCache(t, &readSeqStore{}) + ctx := context.Background() + key := s.getSeqUserReadSeqKey("group", "user") + require.NoError(t, rdb.HSet(ctx, key, "value", "346").Err()) + require.NoError(t, s.SetUserReadSeqs(ctx, "user", map[string]int64{"group": 200})) + require.Equal(t, "346", rdb.HGet(ctx, key, "value").Val()) +} + +func TestReadSeqInt64Precision(t *testing.T) { + s, rdb, server := readSeqTestCache(t, &readSeqStore{}) + ctx := context.Background() + const high int64 = 9223372036854775807 + key := s.getSeqUserReadSeqKey("group", "user") + require.NoError(t, rdb.HSet(ctx, key, "value", strconv.FormatInt(high, 10)).Err()) + require.NoError(t, s.SetUserReadSeqs(ctx, "user", map[string]int64{"group": high - 1})) + require.Equal(t, strconv.FormatInt(high, 10), rdb.HGet(ctx, key, "value").Val()) + require.Equal(t, s.readExpireTime, server.TTL(key)) +} + +func TestReadSeqFencesLegacyRocksCacheRefill(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + s, rdb, _ := readSeqTestCache(t, &readSeqStore{}) + key := s.getSeqUserReadSeqKey("group", "user") + started, resume := make(chan struct{}), make(chan struct{}) + done := make(chan error, 1) + go func() { + _, err := s.rocks.GetClient().Fetch2(ctx, key, time.Hour, func() (string, error) { + close(started) + select { + case <-resume: + case <-ctx.Done(): + } + return "100", nil + }) + done <- err + }() + select { + case <-started: + case <-ctx.Done(): + t.Fatal("legacy refill did not start") + } + effective, writeErr := s.SetUserReadSeq(ctx, "group", "user", 346) + close(resume) + require.NoError(t, writeErr) + require.EqualValues(t, 346, effective) + require.NoError(t, <-done) + require.Equal(t, "346", rdb.HGet(ctx, key, "value").Val()) +} + +func TestReadSeqWithoutRedis(t *testing.T) { + var stored int64 = 346 + store := &readSeqStore{ + read: func(context.Context, string, string) (int64, error) { return stored, nil }, + write: func(_ context.Context, _, _ string, seq int64) error { stored = max(stored, seq); return nil }, + } + s := NewSeqUserCacheRedis(nil, store) + ctx := context.Background() + effective, err := s.SetUserReadSeq(ctx, "group", "user", 200) + require.NoError(t, err) + require.EqualValues(t, 346, effective) + require.NoError(t, s.SetUserReadSeqs(ctx, "user", map[string]int64{"group": 400})) + require.EqualValues(t, 400, stored) +} + +func TestReadSeqMissingAndErrorCases(t *testing.T) { + t.Run("missing rows fill zero and duplicate IDs are deduplicated", func(t *testing.T) { + store := &readSeqStore{readBatch: func(_ context.Context, _ string, convs []string) (map[string]int64, error) { + require.Equal(t, []string{"group"}, convs) + return nil, nil + }} + s, _, _ := readSeqTestCache(t, store) + got, err := s.GetUserReadSeqs(context.Background(), "user", []string{"group", "group"}) + require.NoError(t, err) + require.Equal(t, map[string]int64{"group": 0}, got) + }) + t.Run("MongoDB errors do not populate the cache", func(t *testing.T) { + failure := fmt.Errorf("database unavailable") + store := &readSeqStore{read: func(context.Context, string, string) (int64, error) { return 0, failure }} + s, rdb, _ := readSeqTestCache(t, store) + _, err := s.SetUserReadSeq(context.Background(), "group", "user", 346) + require.ErrorIs(t, err, failure) + require.Zero(t, rdb.Exists(context.Background(), s.getSeqUserReadSeqKey("group", "user")).Val()) + }) + t.Run("negative writes are rejected", func(t *testing.T) { + s, _, _ := readSeqTestCache(t, &readSeqStore{}) + _, err := s.SetUserReadSeq(context.Background(), "group", "user", -1) + require.Error(t, err) + require.Error(t, s.SetUserReadSeqs(context.Background(), "user", map[string]int64{"group": -1})) + }) +} diff --git a/pkg/common/storage/cache/seq_user.go b/pkg/common/storage/cache/seq_user.go index cef414e16..5a1178e9b 100644 --- a/pkg/common/storage/cache/seq_user.go +++ b/pkg/common/storage/cache/seq_user.go @@ -8,7 +8,8 @@ type SeqUser interface { GetUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) SetUserMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error GetUserReadSeq(ctx context.Context, conversationID string, userID string) (int64, error) - SetUserReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error + // SetUserReadSeq advances read progress monotonically and returns its effective value. + SetUserReadSeq(ctx context.Context, conversationID string, userID string, seq int64) (int64, error) SetUserReadSeqToDB(ctx context.Context, conversationID string, userID string, seq int64) error SetUserMinSeqs(ctx context.Context, userID string, seqs map[string]int64) error SetUserReadSeqs(ctx context.Context, userID string, seqs map[string]int64) error diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index 6500dbd1e..84dbb34bf 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -71,7 +71,7 @@ type CommonMsgDatabase interface { SetMinSeq(ctx context.Context, conversationID string, seq int64) error SetUserConversationsMinSeqs(ctx context.Context, userID string, seqs map[string]int64) (err error) - SetHasReadSeq(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error + SetHasReadSeq(ctx context.Context, userID string, conversationID string, hasReadSeq int64) (int64, error) GetHasReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) GetHasReadSeq(ctx context.Context, userID string, conversationID string) (int64, error) UserSetHasReadSeqs(ctx context.Context, userID string, hasReadSeqs map[string]int64) error @@ -585,7 +585,7 @@ func (db *commonMsgDatabase) UserSetHasReadSeqs(ctx context.Context, userID stri return db.seqUser.SetUserReadSeqs(ctx, userID, hasReadSeqs) } -func (db *commonMsgDatabase) SetHasReadSeq(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error { +func (db *commonMsgDatabase) SetHasReadSeq(ctx context.Context, userID string, conversationID string, hasReadSeq int64) (int64, error) { return db.seqUser.SetUserReadSeq(ctx, conversationID, userID, hasReadSeq) } diff --git a/pkg/common/storage/controller/msg_transfer.go b/pkg/common/storage/controller/msg_transfer.go index 8a8d9e7d2..2ba12a8d1 100644 --- a/pkg/common/storage/controller/msg_transfer.go +++ b/pkg/common/storage/controller/msg_transfer.go @@ -238,9 +238,12 @@ func (db *msgTransferDatabase) BatchInsertChat2Cache(ctx context.Context, conver func (db *msgTransferDatabase) SetHasReadSeqs(ctx context.Context, conversationID string, userSeqMap map[string]int64) error { for userID, seq := range userSeqMap { - if err := db.seqUser.SetUserReadSeq(ctx, conversationID, userID, seq); err != nil { + effectiveSeq, err := db.seqUser.SetUserReadSeq(ctx, conversationID, userID, seq) + if err != nil { return err } + // The same map is queued for asynchronous persistence by msgtransfer. + userSeqMap[userID] = effectiveSeq } return nil } diff --git a/pkg/common/storage/controller/msg_transfer_read_seq_test.go b/pkg/common/storage/controller/msg_transfer_read_seq_test.go new file mode 100644 index 000000000..8cd35f33e --- /dev/null +++ b/pkg/common/storage/controller/msg_transfer_read_seq_test.go @@ -0,0 +1,22 @@ +package controller + +import ( + "context" + "testing" + + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" + "github.com/stretchr/testify/require" +) + +type effectiveReadSeqCache struct{ cache.SeqUser } + +func (*effectiveReadSeqCache) SetUserReadSeq(context.Context, string, string, int64) (int64, error) { + return 346, nil +} + +func TestReadSeqSenderQueuesEffectiveProgress(t *testing.T) { + db := &msgTransferDatabase{seqUser: &effectiveReadSeqCache{}} + pending := map[string]int64{"user": 200} + require.NoError(t, db.SetHasReadSeqs(context.Background(), "group", pending)) + require.EqualValues(t, 346, pending["user"]) +} diff --git a/pkg/common/storage/database/mgo/seq_user.go b/pkg/common/storage/database/mgo/seq_user.go index 244de3000..15864cc31 100644 --- a/pkg/common/storage/database/mgo/seq_user.go +++ b/pkg/common/storage/database/mgo/seq_user.go @@ -3,9 +3,11 @@ package mgo import ( "context" "errors" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/tools/db/mongoutil" + "github.com/openimsdk/tools/errs" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" @@ -18,9 +20,13 @@ func NewSeqUserMongo(db *mongo.Database) (database.SeqUser, error) { {Key: "user_id", Value: 1}, {Key: "conversation_id", Value: 1}, }, + // A distinct name allows the old non-unique index to coexist on + // supported MongoDB versions. Existing duplicate rows must be resolved + // before rollout; never silently pick one read sequence from duplicates. + Options: options.Index().SetName("user_conversation_unique").SetUnique(true), }) if err != nil { - return nil, err + return nil, errs.WrapMsg(err, "ensure unique seq_user index; check duplicate user_id/conversation_id rows before rollout") } return &seqUserMongo{coll: coll}, nil } @@ -30,6 +36,10 @@ type seqUserMongo struct { } func (s *seqUserMongo) setSeq(ctx context.Context, conversationID string, userID string, seq int64, field string) error { + return s.updateSeq(ctx, conversationID, userID, seq, field, "$set") +} + +func (s *seqUserMongo) updateSeq(ctx context.Context, conversationID, userID string, seq int64, field, operator string) error { filter := map[string]any{ "user_id": userID, "conversation_id": conversationID, @@ -43,13 +53,19 @@ func (s *seqUserMongo) setSeq(ctx context.Context, conversationID string, userID } delete(insert, field) update := map[string]any{ - "$set": bson.M{ + operator: bson.M{ field: seq, }, "$setOnInsert": insert, } opt := options.Update().SetUpsert(true) - return mongoutil.UpdateOne(ctx, s.coll, filter, update, false, opt) + err := mongoutil.UpdateOne(ctx, s.coll, filter, update, false, opt) + if mongo.IsDuplicateKeyError(err) { + // Another writer may have inserted the same user/conversation during + // this upsert. Retry against that row without creating another one. + return mongoutil.UpdateOne(ctx, s.coll, filter, update, true) + } + return err } func (s *seqUserMongo) getSeq(ctx context.Context, conversationID string, userID string, failed string) (int64, error) { @@ -115,12 +131,8 @@ func (s *seqUserMongo) GetUserReadSeqs(ctx context.Context, userID string, conve } func (s *seqUserMongo) SetUserReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error { - dbSeq, err := s.GetUserReadSeq(ctx, conversationID, userID) - if err != nil { - return err + if seq < 0 { + return errs.ErrArgs.WrapMsg("read seq must not be negative") } - if dbSeq > seq { - return nil - } - return s.setSeq(ctx, conversationID, userID, seq, "read_seq") + return s.updateSeq(ctx, conversationID, userID, seq, "read_seq", "$max") } diff --git a/pkg/common/storage/database/mgo/seq_user_monotonic_test.go b/pkg/common/storage/database/mgo/seq_user_monotonic_test.go new file mode 100644 index 000000000..fcd80886f --- /dev/null +++ b/pkg/common/storage/database/mgo/seq_user_monotonic_test.go @@ -0,0 +1,85 @@ +package mgo + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo/integration/mtest" +) + +func TestReadSeqMongoAtomicMax(t *testing.T) { + mt := mtest.New(t, mtest.NewOptions().ClientType(mtest.Mock)) + mt.Run("out of order receipts use max without a preceding read", func(mt *mtest.T) { + s := &seqUserMongo{coll: mt.Coll} + for _, seq := range []int64{346, 200, 346} { + mt.AddMockResponses(mtest.CreateSuccessResponse(bson.E{Key: "n", Value: 1}, bson.E{Key: "nModified", Value: 1})) + require.NoError(mt, s.SetUserReadSeq(context.Background(), "group", "user", seq)) + } + events := mt.GetAllStartedEvents() + require.Len(mt, events, 3) + for i, event := range events { + require.Equal(mt, "update", event.CommandName) + updates, err := event.Command.Lookup("updates").Array().Values() + require.NoError(mt, err) + update := updates[0].Document() + require.Equal(mt, "group", update.Lookup("q", "conversation_id").StringValue()) + require.Equal(mt, "user", update.Lookup("q", "user_id").StringValue()) + require.Equal(mt, []int64{346, 200, 346}[i], update.Lookup("u", "$max", "read_seq").Int64()) + _, err = update.LookupErr("u", "$setOnInsert", "read_seq") + require.Error(mt, err, "insertion defaults must not conflict with $max") + require.True(mt, update.Lookup("upsert").Boolean()) + } + }) + mt.Run("concurrent upsert retries the existing unique row", func(mt *mtest.T) { + mt.AddMockResponses( + mtest.CreateWriteErrorsResponse(mtest.WriteError{Index: 0, Code: 11000, Message: "duplicate user/conversation"}), + mtest.CreateSuccessResponse(bson.E{Key: "n", Value: 1}, bson.E{Key: "nModified", Value: 1}), + ) + s := &seqUserMongo{coll: mt.Coll} + require.NoError(mt, s.SetUserReadSeq(context.Background(), "group", "user", 346)) + events := mt.GetAllStartedEvents() + require.Len(mt, events, 2) + updates, err := events[1].Command.Lookup("updates").Array().Values() + require.NoError(mt, err) + update := updates[0].Document() + upsert, err := update.LookupErr("upsert") + require.True(mt, err != nil || !upsert.Boolean()) + require.EqualValues(mt, 346, update.Lookup("u", "$max", "read_seq").Int64()) + }) + mt.Run("database failures propagate", func(mt *mtest.T) { + mt.AddMockResponses(mtest.CreateCommandErrorResponse(mtest.CommandError{Code: 91, Message: "unavailable"})) + s := &seqUserMongo{coll: mt.Coll} + require.Error(mt, s.SetUserReadSeq(context.Background(), "group", "user", 346)) + }) + mt.Run("negative sequences do not issue a command", func(mt *mtest.T) { + s := &seqUserMongo{coll: mt.Coll} + require.Error(mt, s.SetUserReadSeq(context.Background(), "group", "user", -1)) + require.Empty(mt, mt.GetAllStartedEvents()) + }) +} + +func TestReadSeqMongoUniqueIndex(t *testing.T) { + mt := mtest.New(t, mtest.NewOptions().ClientType(mtest.Mock)) + mt.Run("one record per user and conversation", func(mt *mtest.T) { + mt.AddMockResponses(mtest.CreateSuccessResponse()) + _, err := NewSeqUserMongo(mt.DB) + require.NoError(mt, err) + events := mt.GetAllStartedEvents() + require.Len(mt, events, 1) + require.Equal(mt, "createIndexes", events[0].CommandName) + indexes, err := events[0].Command.Lookup("indexes").Array().Values() + require.NoError(mt, err) + index := indexes[0].Document() + require.True(mt, index.Lookup("unique").Boolean()) + require.Equal(mt, "user_conversation_unique", index.Lookup("name").StringValue()) + require.EqualValues(mt, 1, index.Lookup("key", "user_id").Int32()) + require.EqualValues(mt, 1, index.Lookup("key", "conversation_id").Int32()) + }) + mt.Run("existing duplicates fail explicitly", func(mt *mtest.T) { + mt.AddMockResponses(mtest.CreateCommandErrorResponse(mtest.CommandError{Code: 11000, Message: "duplicate rows"})) + _, err := NewSeqUserMongo(mt.DB) + require.ErrorContains(mt, err, "check duplicate user_id/conversation_id rows before rollout") + }) +}