mirror of
https://github.com/openimsdk/open-im-server.git
synced 2026-11-22 09:05:06 +08:00
fix(msg): keep read sequences monotonic across devices
This commit is contained in:
parent
a5b427a3d6
commit
e3bd4ea48d
48
docs/contrib/read-seq-rollout.md
Normal file
48
docs/contrib/read-seq-rollout.md
Normal file
@ -0,0 +1,48 @@
|
||||
# Rolling out monotonic read sequences
|
||||
|
||||
Read progress is shared by a user across devices. Redis updates, cache fills, and
|
||||
MongoDB writes must retain the greatest confirmed `read_seq` for each
|
||||
`(user_id, conversation_id)`. Read-receipt notifications continue to use the
|
||||
existing asynchronous queue and MQ pipeline.
|
||||
|
||||
## MongoDB preflight
|
||||
|
||||
The server creates a unique compound index named `user_conversation_unique` on
|
||||
`seq_user`. The explicit name allows it to coexist with the previous non-unique
|
||||
index on MongoDB 5.0 and later (the documented deployment uses MongoDB 6.0).
|
||||
The old index is not automatically dropped.
|
||||
|
||||
Before deploying, check for duplicate rows in the intended OpenIM database:
|
||||
|
||||
```javascript
|
||||
db.seq_user.aggregate([
|
||||
{ $group: {
|
||||
_id: { user_id: "$user_id", conversation_id: "$conversation_id" },
|
||||
count: { $sum: 1 }
|
||||
} },
|
||||
{ $match: { count: { $gt: 1 } } },
|
||||
{ $limit: 20 }
|
||||
])
|
||||
```
|
||||
|
||||
Resolve duplicates through a reviewed data migration before rollout, preserving
|
||||
the greatest confirmed `read_seq` and reviewing the independent `min_seq` and
|
||||
`max_seq` bounds. Coordinate the migration with writers. The application does not
|
||||
delete or merge historical records automatically; index creation fails explicitly
|
||||
if duplicates remain. Check index build time and permissions for the deployment.
|
||||
If an equivalent unique index has already been installed manually, reconcile its
|
||||
name with `user_conversation_unique` before rollout.
|
||||
|
||||
## Redis compatibility and limits
|
||||
|
||||
Read-sequence keys keep the existing Redis hash `value` field and 30-day write
|
||||
TTL. Their dedicated monotonic fill path replaces generic RocksCache fills;
|
||||
updating a value also invalidates an older RocksCache fill owner. Complete the
|
||||
rollout across all writers, since old processes can still perform unconditional
|
||||
writes. Other cached fields retain their existing cache behavior.
|
||||
|
||||
Atomic maximum updates tolerate concurrent devices, duplicate receipts, and
|
||||
out-of-order persistence. They do not repair a previously lost persistence event
|
||||
or guarantee delivery through an in-memory queue. Reliable retry/compensation and
|
||||
reconciliation of existing Redis/MongoDB differences remain separate work. Do not
|
||||
mark all messages as read to repair a discrepancy; use confirmed read progress.
|
||||
3
go.mod
3
go.mod
@ -4,6 +4,7 @@ go 1.25.0
|
||||
|
||||
require (
|
||||
firebase.google.com/go/v4 v4.14.1
|
||||
github.com/alicebob/miniredis/v2 v2.34.0
|
||||
github.com/dtm-labs/rockscache v0.1.1
|
||||
github.com/gin-gonic/gin v1.9.1
|
||||
github.com/go-playground/validator/v10 v10.20.0
|
||||
@ -56,6 +57,7 @@ require (
|
||||
cloud.google.com/go/longrunning v0.5.5 // indirect
|
||||
cloud.google.com/go/storage v1.40.0 // indirect
|
||||
github.com/MicahParks/keyfunc v1.9.0 // indirect
|
||||
github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302 // indirect
|
||||
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible // indirect
|
||||
github.com/aws/aws-sdk-go-v2 v1.32.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1 // indirect
|
||||
@ -182,6 +184,7 @@ require (
|
||||
github.com/xdg-go/scram v1.1.2 // indirect
|
||||
github.com/xdg-go/stringprep v1.0.4 // indirect
|
||||
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
|
||||
github.com/yuin/gopher-lua v1.1.1 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
||||
go.etcd.io/etcd/api/v3 v3.5.13 // indirect
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.5.13 // indirect
|
||||
|
||||
6
go.sum
6
go.sum
@ -19,6 +19,10 @@ github.com/IBM/sarama v1.43.0/go.mod h1:zlE6HEbC/SMQ9mhEYaF7nNLYOUyrs0obySKCckWP
|
||||
github.com/MicahParks/keyfunc v1.9.0 h1:lhKd5xrFHLNOWrDc4Tyb/Q1AJ4LCzQ48GVJyVIID3+o=
|
||||
github.com/MicahParks/keyfunc v1.9.0/go.mod h1:IdnCilugA0O/99dW+/MkvlyrsX8+L8+x95xuVNtM5jw=
|
||||
github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM=
|
||||
github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302 h1:uvdUDbHQHO85qeSydJtItA4T55Pw6BtAejd0APRJOCE=
|
||||
github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
|
||||
github.com/alicebob/miniredis/v2 v2.34.0 h1:mBFWMaJSNL9RwdGRyEDoAAv8OQc5UlEhLDQggTglU/0=
|
||||
github.com/alicebob/miniredis/v2 v2.34.0/go.mod h1:kWShP4b58T1CW0Y5dViCd5ztzrDqRWqM3nksiyXk5s8=
|
||||
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g=
|
||||
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
|
||||
github.com/aws/aws-sdk-go-v2 v1.32.5 h1:U8vdWJuY7ruAkzaOdD7guwJjD06YSKmnKCJs7s3IkIo=
|
||||
@ -478,6 +482,8 @@ github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7Jul
|
||||
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
|
||||
github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
|
||||
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
|
||||
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||
go.etcd.io/etcd/api/v3 v3.5.13 h1:8WXU2/NBge6AUF1K1gOexB6e07NgsN1hXK0rSTtgSp4=
|
||||
|
||||
@ -96,10 +96,11 @@ func (m *msgServer) SetConversationHasReadSeq(ctx context.Context, req *msg.SetC
|
||||
if req.HasReadSeq > maxSeq {
|
||||
return nil, errs.ErrArgs.WrapMsg("hasReadSeq must not be bigger than maxSeq")
|
||||
}
|
||||
if err := m.MsgDatabase.SetHasReadSeq(ctx, req.UserID, req.ConversationID, req.HasReadSeq); err != nil {
|
||||
hasReadSeq, err := m.MsgDatabase.SetHasReadSeq(ctx, req.UserID, req.ConversationID, req.HasReadSeq)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m.sendMarkAsReadNotification(ctx, req.ConversationID, constant.SingleChatType, req.UserID, req.UserID, nil, req.HasReadSeq)
|
||||
m.sendMarkAsReadNotification(ctx, req.ConversationID, constant.SingleChatType, req.UserID, req.UserID, nil, hasReadSeq)
|
||||
return &msg.SetConversationHasReadSeqResp{}, nil
|
||||
}
|
||||
|
||||
@ -122,16 +123,10 @@ func (m *msgServer) MarkMsgsAsRead(ctx context.Context, req *msg.MarkMsgsAsReadR
|
||||
if err := m.MsgDatabase.MarkSingleChatMsgsAsRead(ctx, req.UserID, req.ConversationID, req.Seqs); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
currentHasReadSeq, err := m.MsgDatabase.GetHasReadSeq(ctx, req.UserID, req.ConversationID)
|
||||
if err != nil && !errors.Is(err, redis.Nil) {
|
||||
hasReadSeq, err = m.MsgDatabase.SetHasReadSeq(ctx, req.UserID, req.ConversationID, hasReadSeq)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if hasReadSeq > currentHasReadSeq {
|
||||
err = m.MsgDatabase.SetHasReadSeq(ctx, req.UserID, req.ConversationID, hasReadSeq)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
reqCallback := &cbapi.CallbackSingleMsgReadReq{
|
||||
ConversationID: conversation.ConversationID,
|
||||
@ -176,23 +171,17 @@ func (m *msgServer) MarkConversationAsRead(ctx context.Context, req *msg.MarkCon
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if req.HasReadSeq > hasReadSeq {
|
||||
err = m.MsgDatabase.SetHasReadSeq(ctx, req.UserID, req.ConversationID, req.HasReadSeq)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hasReadSeq = req.HasReadSeq
|
||||
hasReadSeq, err = m.MsgDatabase.SetHasReadSeq(ctx, req.UserID, req.ConversationID, req.HasReadSeq)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m.sendMarkAsReadNotification(ctx, req.ConversationID, conversation.ConversationType, req.UserID,
|
||||
m.conversationAndGetRecvID(conversation, req.UserID), seqs, hasReadSeq)
|
||||
} else if conversation.ConversationType == constant.ReadGroupChatType ||
|
||||
conversation.ConversationType == constant.NotificationChatType {
|
||||
if req.HasReadSeq > hasReadSeq {
|
||||
err = m.MsgDatabase.SetHasReadSeq(ctx, req.UserID, req.ConversationID, req.HasReadSeq)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hasReadSeq = req.HasReadSeq
|
||||
hasReadSeq, err = m.MsgDatabase.SetHasReadSeq(ctx, req.UserID, req.ConversationID, req.HasReadSeq)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m.sendMarkAsReadNotification(ctx, req.ConversationID, constant.SingleChatType, req.UserID,
|
||||
req.UserID, seqs, hasReadSeq)
|
||||
|
||||
107
internal/rpc/msg/as_read_monotonic_test.go
Normal file
107
internal/rpc/msg/as_read_monotonic_test.go
Normal file
@ -0,0 +1,107 @@
|
||||
package msg
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/openimsdk/open-im-server/v3/pkg/common/config"
|
||||
"github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller"
|
||||
"github.com/openimsdk/open-im-server/v3/pkg/common/webhook"
|
||||
"github.com/openimsdk/open-im-server/v3/pkg/notification"
|
||||
"github.com/openimsdk/open-im-server/v3/pkg/rpccache"
|
||||
"github.com/openimsdk/open-im-server/v3/pkg/rpcli"
|
||||
"github.com/openimsdk/protocol/constant"
|
||||
"github.com/openimsdk/protocol/conversation"
|
||||
pbmsg "github.com/openimsdk/protocol/msg"
|
||||
"github.com/openimsdk/protocol/sdkws"
|
||||
"github.com/openimsdk/tools/mcontext"
|
||||
"github.com/openimsdk/tools/mq/memamq"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
type receiptDatabase struct {
|
||||
controller.CommonMsgDatabase
|
||||
calls int
|
||||
initial int64
|
||||
}
|
||||
|
||||
func (*receiptDatabase) GetMaxSeq(context.Context, string) (int64, error) { return 346, nil }
|
||||
func (db *receiptDatabase) GetHasReadSeq(context.Context, string, string) (int64, error) {
|
||||
return db.initial, nil
|
||||
}
|
||||
func (db *receiptDatabase) SetHasReadSeq(context.Context, string, string, int64) (int64, error) {
|
||||
db.calls++
|
||||
return 346, nil // Another device advanced after the RPC's initial read.
|
||||
}
|
||||
func (*receiptDatabase) MarkSingleChatMsgsAsRead(context.Context, string, string, []int64) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
type receiptConversationClient struct {
|
||||
conversation.ConversationClient
|
||||
sessionType int32
|
||||
}
|
||||
|
||||
func (c *receiptConversationClient) GetConversation(_ context.Context, req *conversation.GetConversationReq, _ ...grpc.CallOption) (*conversation.GetConversationResp, error) {
|
||||
return &conversation.GetConversationResp{Conversation: &conversation.Conversation{
|
||||
ConversationID: req.ConversationID, OwnerUserID: req.OwnerUserID,
|
||||
ConversationType: c.sessionType, UserID: "peer", GroupID: "group",
|
||||
}}, nil
|
||||
}
|
||||
|
||||
func TestReadSeqReceiptsUseEffectiveProgress(t *testing.T) {
|
||||
for _, method := range []string{"set", "messages", "single conversation", "group conversation"} {
|
||||
for _, requested := range []int64{200, 346} {
|
||||
for _, initial := range []int64{100, 346} {
|
||||
t.Run(fmt.Sprintf("%s/initial=%d/requested=%d", method, initial, requested), func(t *testing.T) {
|
||||
ctx := mcontext.WithOpUserIDContext(context.Background(), "user")
|
||||
db := &receiptDatabase{initial: initial}
|
||||
queue := memamq.NewMemoryQueue(1, 8)
|
||||
defer queue.Stop()
|
||||
sent := make(chan *sdkws.MsgData, 1)
|
||||
notifier := notification.NewNotificationSender(&config.Notification{}, notification.WithQueue(queue),
|
||||
notification.WithLocalSendMsg(func(_ context.Context, req *pbmsg.SendMsgReq) (*pbmsg.SendMsgResp, error) {
|
||||
sent <- req.MsgData
|
||||
return &pbmsg.SendMsgResp{}, nil
|
||||
}))
|
||||
sessionType := int32(constant.SingleChatType)
|
||||
if method == "group conversation" {
|
||||
sessionType = constant.ReadGroupChatType
|
||||
}
|
||||
client := &rpcli.ConversationClient{ConversationClient: &receiptConversationClient{sessionType: sessionType}}
|
||||
server := &msgServer{MsgDatabase: db, notificationSender: notifier, config: &Config{},
|
||||
ConversationLocalCache: rpccache.NewConversationLocalCache(client, &config.LocalCache{}, nil),
|
||||
webhookClient: webhook.NewWebhookClient("", queue),
|
||||
}
|
||||
var err error
|
||||
switch method {
|
||||
case "set":
|
||||
_, err = server.SetConversationHasReadSeq(ctx, &pbmsg.SetConversationHasReadSeqReq{UserID: "user", ConversationID: "group", HasReadSeq: requested})
|
||||
case "messages":
|
||||
_, err = server.MarkMsgsAsRead(ctx, &pbmsg.MarkMsgsAsReadReq{UserID: "user", ConversationID: "group", Seqs: []int64{requested}})
|
||||
default:
|
||||
_, err = server.MarkConversationAsRead(ctx, &pbmsg.MarkConversationAsReadReq{UserID: "user", ConversationID: "group", HasReadSeq: requested})
|
||||
}
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, db.calls, "equal reports must still allow persistence/notification repair")
|
||||
select {
|
||||
case message := <-sent:
|
||||
require.EqualValues(t, constant.HasReadReceipt, message.ContentType)
|
||||
var elem sdkws.NotificationElem
|
||||
require.NoError(t, json.Unmarshal(message.Content, &elem))
|
||||
var tips sdkws.MarkAsReadTips
|
||||
require.NoError(t, json.Unmarshal([]byte(elem.Detail), &tips))
|
||||
require.EqualValues(t, 346, tips.HasReadSeq)
|
||||
require.Equal(t, "group", tips.ConversationID)
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("read receipt was not emitted")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
233
pkg/common/storage/cache/redis/seq_user.go
vendored
233
pkg/common/storage/cache/redis/seq_user.go
vendored
@ -65,26 +65,110 @@ func (s *seqUserCacheRedis) SetUserMinSeq(ctx context.Context, conversationID st
|
||||
return s.SetUserMinSeqs(ctx, userID, map[string]int64{conversationID: seq})
|
||||
}
|
||||
|
||||
// Read sequences use their own monotonic fill path. A generic RocksCache
|
||||
// refill can otherwise overwrite a concurrent read receipt with an older DB value.
|
||||
func (s *seqUserCacheRedis) GetUserReadSeq(ctx context.Context, conversationID string, userID string) (int64, error) {
|
||||
return getCache(ctx, s.rocks, s.getSeqUserReadSeqKey(conversationID, userID), s.readExpireTime, func(ctx context.Context) (int64, error) {
|
||||
if s.rocks.Disable() {
|
||||
return s.mgo.GetUserReadSeq(ctx, conversationID, userID)
|
||||
})
|
||||
}
|
||||
key := s.getSeqUserReadSeqKey(conversationID, userID)
|
||||
value, err := s.rocks.GetRedis().HGet(ctx, key, "value").Result()
|
||||
if err == nil && value != "" {
|
||||
return strconv.ParseInt(value, 10, 64)
|
||||
}
|
||||
if err != nil && err != redis.Nil {
|
||||
return 0, errs.Wrap(err)
|
||||
}
|
||||
seq, err := s.mgo.GetUserReadSeq(ctx, conversationID, userID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return s.mergeReadSeq(ctx, key, seq)
|
||||
}
|
||||
|
||||
func (s *seqUserCacheRedis) SetUserReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error {
|
||||
if s.rocks.GetRedis() == nil {
|
||||
return s.SetUserReadSeqToDB(ctx, conversationID, userID, seq)
|
||||
// SetUserReadSeq returns the effective sequence, including any concurrent advance.
|
||||
func (s *seqUserCacheRedis) SetUserReadSeq(ctx context.Context, conversationID string, userID string, seq int64) (int64, error) {
|
||||
if seq < 0 {
|
||||
return 0, errs.ErrArgs.WrapMsg("read seq must not be negative")
|
||||
}
|
||||
dbSeq, err := s.GetUserReadSeq(ctx, conversationID, userID)
|
||||
if s.rocks.Disable() {
|
||||
if err := s.SetUserReadSeqToDB(ctx, conversationID, userID, seq); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return s.mgo.GetUserReadSeq(ctx, conversationID, userID)
|
||||
}
|
||||
// Preserve the MongoDB baseline on a cold cache, as well as the value read
|
||||
// before a key expires between this read and the atomic update.
|
||||
current, err := s.GetUserReadSeq(ctx, conversationID, userID)
|
||||
if err != nil {
|
||||
return err
|
||||
return 0, err
|
||||
}
|
||||
if dbSeq < seq {
|
||||
if err := s.rocks.GetClient().RawSet(ctx, s.getSeqUserReadSeqKey(conversationID, userID), strconv.Itoa(int(seq)), s.readExpireTime); err != nil {
|
||||
return errs.Wrap(err)
|
||||
return s.mergeReadSeq(ctx, s.getSeqUserReadSeqKey(conversationID, userID), max(current, seq))
|
||||
}
|
||||
|
||||
// Compare decimal strings rather than Lua doubles to preserve the full int64 range.
|
||||
// Removing the refill owner also fences any RocksCache fill started by an older
|
||||
// server instance before this write during a rolling upgrade.
|
||||
var mergeReadSeqScript = redis.NewScript(`
|
||||
local incoming = ARGV[1]
|
||||
local current = redis.call('HGET', KEYS[1], 'value')
|
||||
if current and current ~= '' then
|
||||
if not string.match(current, '^%d+$') then
|
||||
return redis.error_reply('invalid cached read sequence')
|
||||
end
|
||||
if #current > #incoming or (#current == #incoming and current > incoming) then
|
||||
incoming = current
|
||||
end
|
||||
end
|
||||
redis.call('HSET', KEYS[1], 'value', incoming)
|
||||
redis.call('HDEL', KEYS[1], 'lockUntil', 'lockOwner')
|
||||
redis.call('PEXPIRE', KEYS[1], ARGV[2])
|
||||
return incoming
|
||||
`)
|
||||
|
||||
func (s *seqUserCacheRedis) mergeReadSeq(ctx context.Context, key string, seq int64) (int64, error) {
|
||||
if seq < 0 {
|
||||
return 0, errs.ErrArgs.WrapMsg("read seq must not be negative")
|
||||
}
|
||||
value, err := mergeReadSeqScript.Run(ctx, s.rocks.GetRedis(), []string{key},
|
||||
strconv.FormatInt(seq, 10), s.readExpireTime.Milliseconds()).Text()
|
||||
if err != nil {
|
||||
return 0, errs.Wrap(err)
|
||||
}
|
||||
return strconv.ParseInt(value, 10, 64)
|
||||
}
|
||||
|
||||
func (s *seqUserCacheRedis) mergeReadSeqs(ctx context.Context, userID string, seqs map[string]int64) (map[string]int64, error) {
|
||||
pipe := s.rocks.GetRedis().Pipeline()
|
||||
commands := make(map[string]*redis.Cmd, len(seqs))
|
||||
for conversationID, seq := range seqs {
|
||||
if seq < 0 {
|
||||
return nil, errs.ErrArgs.WrapMsg("read seq must not be negative")
|
||||
}
|
||||
// Each script touches a single key, so ClusterClient can route the
|
||||
// pipeline across slots without a cross-slot transaction.
|
||||
commands[conversationID] = mergeReadSeqScript.Eval(ctx, pipe,
|
||||
[]string{s.getSeqUserReadSeqKey(conversationID, userID)},
|
||||
strconv.FormatInt(seq, 10), s.readExpireTime.Milliseconds())
|
||||
}
|
||||
if len(commands) > 0 {
|
||||
if _, err := pipe.Exec(ctx); err != nil {
|
||||
return nil, errs.Wrap(err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
result := make(map[string]int64, len(commands))
|
||||
for conversationID, cmd := range commands {
|
||||
value, err := cmd.Text()
|
||||
if err != nil {
|
||||
return nil, errs.Wrap(err)
|
||||
}
|
||||
seq, err := strconv.ParseInt(value, 10, 64)
|
||||
if err != nil {
|
||||
return nil, errs.Wrap(err)
|
||||
}
|
||||
result[conversationID] = seq
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *seqUserCacheRedis) SetUserReadSeqToDB(ctx context.Context, conversationID string, userID string, seq int64) error {
|
||||
@ -102,83 +186,84 @@ func (s *seqUserCacheRedis) SetUserMinSeqs(ctx context.Context, userID string, s
|
||||
return DeleteCacheBySlot(ctx, s.rocks, keys)
|
||||
}
|
||||
|
||||
func (s *seqUserCacheRedis) setUserRedisReadSeqs(ctx context.Context, userID string, seqs map[string]int64) error {
|
||||
keys := make([]string, 0, len(seqs))
|
||||
keySeq := make(map[string]int64)
|
||||
func (s *seqUserCacheRedis) SetUserReadSeqs(ctx context.Context, userID string, seqs map[string]int64) error {
|
||||
conversationIDs := make([]string, 0, len(seqs))
|
||||
for conversationID, seq := range seqs {
|
||||
key := s.getSeqUserReadSeqKey(conversationID, userID)
|
||||
keys = append(keys, key)
|
||||
keySeq[key] = seq
|
||||
if seq < 0 {
|
||||
return errs.ErrArgs.WrapMsg("read seq must not be negative")
|
||||
}
|
||||
conversationIDs = append(conversationIDs, conversationID)
|
||||
}
|
||||
slotKeys, err := groupKeysBySlot(ctx, s.rocks.GetRedis(), keys)
|
||||
if s.rocks.Disable() {
|
||||
for conversationID, seq := range seqs {
|
||||
if err := s.SetUserReadSeqToDB(ctx, conversationID, userID, seq); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
current, err := s.GetUserReadSeqs(ctx, userID, conversationIDs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, keys := range slotKeys {
|
||||
pipe := s.rocks.GetRedis().Pipeline()
|
||||
for _, key := range keys {
|
||||
pipe.HSet(ctx, key, "value", strconv.FormatInt(keySeq[key], 10))
|
||||
pipe.Expire(ctx, key, s.readExpireTime)
|
||||
}
|
||||
if _, err := pipe.Exec(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
updates := make(map[string]int64, len(seqs))
|
||||
for conversationID, seq := range seqs {
|
||||
updates[conversationID] = max(current[conversationID], seq)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *seqUserCacheRedis) SetUserReadSeqs(ctx context.Context, userID string, seqs map[string]int64) error {
|
||||
if len(seqs) == 0 {
|
||||
return nil
|
||||
}
|
||||
if err := s.setUserRedisReadSeqs(ctx, userID, seqs); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
_, err = s.mergeReadSeqs(ctx, userID, updates)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *seqUserCacheRedis) GetUserReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) {
|
||||
res, err := batchGetCache2(ctx, s.rocks, s.readExpireTime, conversationIDs, func(conversationID string) string {
|
||||
return s.getSeqUserReadSeqKey(conversationID, userID)
|
||||
}, func(v *readSeqModel) string {
|
||||
return v.ConversationID
|
||||
}, func(ctx context.Context, conversationIDs []string) ([]*readSeqModel, error) {
|
||||
seqs, err := s.mgo.GetUserReadSeqs(ctx, userID, conversationIDs)
|
||||
if s.rocks.Disable() {
|
||||
return s.mgo.GetUserReadSeqs(ctx, userID, conversationIDs)
|
||||
}
|
||||
pipe := s.rocks.GetRedis().Pipeline()
|
||||
commands := make(map[string]*redis.StringCmd, len(conversationIDs))
|
||||
for _, conversationID := range conversationIDs {
|
||||
if _, ok := commands[conversationID]; !ok {
|
||||
commands[conversationID] = pipe.HGet(ctx, s.getSeqUserReadSeqKey(conversationID, userID), "value")
|
||||
}
|
||||
}
|
||||
if len(commands) > 0 {
|
||||
if _, err := pipe.Exec(ctx); err != nil && err != redis.Nil {
|
||||
return nil, errs.Wrap(err)
|
||||
}
|
||||
}
|
||||
result := make(map[string]int64, len(commands))
|
||||
missing := make([]string, 0)
|
||||
for conversationID, cmd := range commands {
|
||||
value, err := cmd.Result()
|
||||
if err == redis.Nil || (err == nil && value == "") {
|
||||
missing = append(missing, conversationID)
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, errs.Wrap(err)
|
||||
}
|
||||
res := make([]*readSeqModel, 0, len(seqs))
|
||||
for conversationID, seq := range seqs {
|
||||
res = append(res, &readSeqModel{ConversationID: conversationID, Seq: seq})
|
||||
seq, err := strconv.ParseInt(value, 10, 64)
|
||||
if err != nil {
|
||||
return nil, errs.Wrap(err)
|
||||
}
|
||||
return res, nil
|
||||
})
|
||||
result[conversationID] = seq
|
||||
}
|
||||
if len(missing) == 0 {
|
||||
return result, nil
|
||||
}
|
||||
stored, err := s.mgo.GetUserReadSeqs(ctx, userID, missing)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
data := make(map[string]int64)
|
||||
for _, v := range res {
|
||||
data[v.ConversationID] = v.Seq
|
||||
fills := make(map[string]int64, len(missing))
|
||||
for _, conversationID := range missing {
|
||||
fills[conversationID] = stored[conversationID]
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
var _ BatchCacheCallback[string] = (*readSeqModel)(nil)
|
||||
|
||||
type readSeqModel struct {
|
||||
ConversationID string
|
||||
Seq int64
|
||||
}
|
||||
|
||||
func (r *readSeqModel) BatchCache(conversationID string) {
|
||||
r.ConversationID = conversationID
|
||||
}
|
||||
|
||||
func (r *readSeqModel) UnmarshalJSON(bytes []byte) (err error) {
|
||||
r.Seq, err = strconv.ParseInt(string(bytes), 10, 64)
|
||||
return
|
||||
}
|
||||
|
||||
func (r *readSeqModel) MarshalJSON() ([]byte, error) {
|
||||
return []byte(strconv.FormatInt(r.Seq, 10)), nil
|
||||
filled, err := s.mergeReadSeqs(ctx, userID, fills)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for conversationID, seq := range filled {
|
||||
result[conversationID] = seq
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
274
pkg/common/storage/cache/redis/seq_user_monotonic_test.go
vendored
Normal file
274
pkg/common/storage/cache/redis/seq_user_monotonic_test.go
vendored
Normal file
@ -0,0 +1,274 @@
|
||||
package redis
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/alicebob/miniredis/v2"
|
||||
"github.com/openimsdk/open-im-server/v3/pkg/common/storage/database"
|
||||
"github.com/redis/go-redis/v9"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type readSeqStore struct {
|
||||
database.SeqUser
|
||||
read func(context.Context, string, string) (int64, error)
|
||||
readBatch func(context.Context, string, []string) (map[string]int64, error)
|
||||
write func(context.Context, string, string, int64) error
|
||||
}
|
||||
|
||||
func (s *readSeqStore) SetUserReadSeq(ctx context.Context, conv, user string, seq int64) error {
|
||||
return s.write(ctx, conv, user, seq)
|
||||
}
|
||||
|
||||
func (s *readSeqStore) GetUserReadSeq(ctx context.Context, conv, user string) (int64, error) {
|
||||
if s.read != nil {
|
||||
return s.read(ctx, conv, user)
|
||||
}
|
||||
return 100, nil
|
||||
}
|
||||
|
||||
func (s *readSeqStore) GetUserReadSeqs(ctx context.Context, user string, convs []string) (map[string]int64, error) {
|
||||
if s.readBatch != nil {
|
||||
return s.readBatch(ctx, user, convs)
|
||||
}
|
||||
result := make(map[string]int64, len(convs))
|
||||
for _, conv := range convs {
|
||||
result[conv] = 100
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func readSeqTestCache(t *testing.T, store database.SeqUser) (*seqUserCacheRedis, *redis.Client, *miniredis.Miniredis) {
|
||||
t.Helper()
|
||||
server := miniredis.RunT(t)
|
||||
rdb := redis.NewClient(&redis.Options{Addr: server.Addr()})
|
||||
t.Cleanup(func() { _ = rdb.Close() })
|
||||
return NewSeqUserCacheRedis(rdb, store).(*seqUserCacheRedis), rdb, server
|
||||
}
|
||||
|
||||
// Delay the lower writer at the Redis command boundary, after its cache read.
|
||||
// This covers both the old HSET and the atomic-script implementation.
|
||||
type pauseReadSeqWrite struct {
|
||||
started chan struct{}
|
||||
resume chan struct{}
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
func (h *pauseReadSeqWrite) DialHook(next redis.DialHook) redis.DialHook { return next }
|
||||
func (h *pauseReadSeqWrite) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook {
|
||||
return next
|
||||
}
|
||||
func (h *pauseReadSeqWrite) ProcessHook(next redis.ProcessHook) redis.ProcessHook {
|
||||
return func(ctx context.Context, cmd redis.Cmder) error {
|
||||
if cmd.Name() == "hset" || cmd.Name() == "evalsha" || cmd.Name() == "eval" {
|
||||
for _, arg := range cmd.Args() {
|
||||
if fmt.Sprint(arg) == "200" {
|
||||
h.once.Do(func() {
|
||||
close(h.started)
|
||||
select {
|
||||
case <-h.resume:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
})
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return next(ctx, cmd)
|
||||
}
|
||||
}
|
||||
|
||||
// Keep a compile-time check that this hook remains usable by go-redis.
|
||||
var _ redis.Hook = (*pauseReadSeqWrite)(nil)
|
||||
|
||||
func TestReadSeqConcurrentWritersNeverRegress(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
s, rdb, _ := readSeqTestCache(t, &readSeqStore{})
|
||||
key := s.getSeqUserReadSeqKey("group", "user")
|
||||
require.NoError(t, rdb.HSet(ctx, key, "value", "100").Err())
|
||||
hook := &pauseReadSeqWrite{started: make(chan struct{}), resume: make(chan struct{})}
|
||||
rdb.AddHook(hook)
|
||||
done := make(chan error, 1)
|
||||
go func() { _, err := s.SetUserReadSeq(ctx, "group", "user", 200); done <- err }()
|
||||
select {
|
||||
case <-hook.started:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("lower writer never reached Redis")
|
||||
}
|
||||
highSeq, highErr := s.SetUserReadSeq(ctx, "group", "user", 346)
|
||||
close(hook.resume)
|
||||
require.NoError(t, highErr)
|
||||
require.EqualValues(t, 346, highSeq)
|
||||
require.NoError(t, <-done)
|
||||
require.Equal(t, "346", rdb.HGet(ctx, key, "value").Val())
|
||||
}
|
||||
|
||||
func TestReadSeqRefillDoesNotOverwriteNewerProgress(t *testing.T) {
|
||||
for _, batch := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("batch=%v", batch), func(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
started, resume := make(chan struct{}), make(chan struct{})
|
||||
var once sync.Once
|
||||
wait := func() {
|
||||
once.Do(func() {
|
||||
close(started)
|
||||
select {
|
||||
case <-resume:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
})
|
||||
}
|
||||
store := &readSeqStore{}
|
||||
if batch {
|
||||
store.readBatch = func(context.Context, string, []string) (map[string]int64, error) {
|
||||
wait()
|
||||
return map[string]int64{"group": 100}, nil
|
||||
}
|
||||
} else {
|
||||
store.read = func(context.Context, string, string) (int64, error) { wait(); return 100, nil }
|
||||
}
|
||||
s, rdb, _ := readSeqTestCache(t, store)
|
||||
type result struct {
|
||||
seq int64
|
||||
err error
|
||||
}
|
||||
done := make(chan result, 1)
|
||||
go func() {
|
||||
if batch {
|
||||
values, err := s.GetUserReadSeqs(ctx, "user", []string{"group"})
|
||||
done <- result{values["group"], err}
|
||||
} else {
|
||||
seq, err := s.GetUserReadSeq(ctx, "group", "user")
|
||||
done <- result{seq, err}
|
||||
}
|
||||
}()
|
||||
select {
|
||||
case <-started:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("refill never reached MongoDB")
|
||||
}
|
||||
// Seed the new baseline as another request completes its refill, then
|
||||
// advance it through the real writer while the older query is pending.
|
||||
key := s.getSeqUserReadSeqKey("group", "user")
|
||||
require.NoError(t, rdb.HSet(ctx, key, "value", "100").Err())
|
||||
require.NoError(t, rdb.HDel(ctx, key, "lockUntil", "lockOwner").Err())
|
||||
writeErr := s.SetUserReadSeqs(ctx, "user", map[string]int64{"group": 346})
|
||||
close(resume)
|
||||
require.NoError(t, writeErr)
|
||||
got := <-done
|
||||
require.NoError(t, got.err)
|
||||
require.EqualValues(t, 346, got.seq, "the response must include the concurrent advancement")
|
||||
require.Equal(t, "346", rdb.HGet(ctx, key, "value").Val())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadSeqBatchPreservesMongoBaseline(t *testing.T) {
|
||||
store := &readSeqStore{readBatch: func(context.Context, string, []string) (map[string]int64, error) {
|
||||
return map[string]int64{"group": 346}, nil
|
||||
}}
|
||||
s, rdb, _ := readSeqTestCache(t, store)
|
||||
ctx := context.Background()
|
||||
require.NoError(t, s.SetUserReadSeqs(ctx, "user", map[string]int64{"group": 200}))
|
||||
require.Equal(t, "346", rdb.HGet(ctx, s.getSeqUserReadSeqKey("group", "user"), "value").Val())
|
||||
}
|
||||
|
||||
func TestReadSeqBatchNeverRegresses(t *testing.T) {
|
||||
s, rdb, _ := readSeqTestCache(t, &readSeqStore{})
|
||||
ctx := context.Background()
|
||||
key := s.getSeqUserReadSeqKey("group", "user")
|
||||
require.NoError(t, rdb.HSet(ctx, key, "value", "346").Err())
|
||||
require.NoError(t, s.SetUserReadSeqs(ctx, "user", map[string]int64{"group": 200}))
|
||||
require.Equal(t, "346", rdb.HGet(ctx, key, "value").Val())
|
||||
}
|
||||
|
||||
func TestReadSeqInt64Precision(t *testing.T) {
|
||||
s, rdb, server := readSeqTestCache(t, &readSeqStore{})
|
||||
ctx := context.Background()
|
||||
const high int64 = 9223372036854775807
|
||||
key := s.getSeqUserReadSeqKey("group", "user")
|
||||
require.NoError(t, rdb.HSet(ctx, key, "value", strconv.FormatInt(high, 10)).Err())
|
||||
require.NoError(t, s.SetUserReadSeqs(ctx, "user", map[string]int64{"group": high - 1}))
|
||||
require.Equal(t, strconv.FormatInt(high, 10), rdb.HGet(ctx, key, "value").Val())
|
||||
require.Equal(t, s.readExpireTime, server.TTL(key))
|
||||
}
|
||||
|
||||
func TestReadSeqFencesLegacyRocksCacheRefill(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
s, rdb, _ := readSeqTestCache(t, &readSeqStore{})
|
||||
key := s.getSeqUserReadSeqKey("group", "user")
|
||||
started, resume := make(chan struct{}), make(chan struct{})
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := s.rocks.GetClient().Fetch2(ctx, key, time.Hour, func() (string, error) {
|
||||
close(started)
|
||||
select {
|
||||
case <-resume:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
return "100", nil
|
||||
})
|
||||
done <- err
|
||||
}()
|
||||
select {
|
||||
case <-started:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("legacy refill did not start")
|
||||
}
|
||||
effective, writeErr := s.SetUserReadSeq(ctx, "group", "user", 346)
|
||||
close(resume)
|
||||
require.NoError(t, writeErr)
|
||||
require.EqualValues(t, 346, effective)
|
||||
require.NoError(t, <-done)
|
||||
require.Equal(t, "346", rdb.HGet(ctx, key, "value").Val())
|
||||
}
|
||||
|
||||
func TestReadSeqWithoutRedis(t *testing.T) {
|
||||
var stored int64 = 346
|
||||
store := &readSeqStore{
|
||||
read: func(context.Context, string, string) (int64, error) { return stored, nil },
|
||||
write: func(_ context.Context, _, _ string, seq int64) error { stored = max(stored, seq); return nil },
|
||||
}
|
||||
s := NewSeqUserCacheRedis(nil, store)
|
||||
ctx := context.Background()
|
||||
effective, err := s.SetUserReadSeq(ctx, "group", "user", 200)
|
||||
require.NoError(t, err)
|
||||
require.EqualValues(t, 346, effective)
|
||||
require.NoError(t, s.SetUserReadSeqs(ctx, "user", map[string]int64{"group": 400}))
|
||||
require.EqualValues(t, 400, stored)
|
||||
}
|
||||
|
||||
func TestReadSeqMissingAndErrorCases(t *testing.T) {
|
||||
t.Run("missing rows fill zero and duplicate IDs are deduplicated", func(t *testing.T) {
|
||||
store := &readSeqStore{readBatch: func(_ context.Context, _ string, convs []string) (map[string]int64, error) {
|
||||
require.Equal(t, []string{"group"}, convs)
|
||||
return nil, nil
|
||||
}}
|
||||
s, _, _ := readSeqTestCache(t, store)
|
||||
got, err := s.GetUserReadSeqs(context.Background(), "user", []string{"group", "group"})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, map[string]int64{"group": 0}, got)
|
||||
})
|
||||
t.Run("MongoDB errors do not populate the cache", func(t *testing.T) {
|
||||
failure := fmt.Errorf("database unavailable")
|
||||
store := &readSeqStore{read: func(context.Context, string, string) (int64, error) { return 0, failure }}
|
||||
s, rdb, _ := readSeqTestCache(t, store)
|
||||
_, err := s.SetUserReadSeq(context.Background(), "group", "user", 346)
|
||||
require.ErrorIs(t, err, failure)
|
||||
require.Zero(t, rdb.Exists(context.Background(), s.getSeqUserReadSeqKey("group", "user")).Val())
|
||||
})
|
||||
t.Run("negative writes are rejected", func(t *testing.T) {
|
||||
s, _, _ := readSeqTestCache(t, &readSeqStore{})
|
||||
_, err := s.SetUserReadSeq(context.Background(), "group", "user", -1)
|
||||
require.Error(t, err)
|
||||
require.Error(t, s.SetUserReadSeqs(context.Background(), "user", map[string]int64{"group": -1}))
|
||||
})
|
||||
}
|
||||
3
pkg/common/storage/cache/seq_user.go
vendored
3
pkg/common/storage/cache/seq_user.go
vendored
@ -8,7 +8,8 @@ type SeqUser interface {
|
||||
GetUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error)
|
||||
SetUserMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error
|
||||
GetUserReadSeq(ctx context.Context, conversationID string, userID string) (int64, error)
|
||||
SetUserReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error
|
||||
// SetUserReadSeq advances read progress monotonically and returns its effective value.
|
||||
SetUserReadSeq(ctx context.Context, conversationID string, userID string, seq int64) (int64, error)
|
||||
SetUserReadSeqToDB(ctx context.Context, conversationID string, userID string, seq int64) error
|
||||
SetUserMinSeqs(ctx context.Context, userID string, seqs map[string]int64) error
|
||||
SetUserReadSeqs(ctx context.Context, userID string, seqs map[string]int64) error
|
||||
|
||||
@ -71,7 +71,7 @@ type CommonMsgDatabase interface {
|
||||
SetMinSeq(ctx context.Context, conversationID string, seq int64) error
|
||||
|
||||
SetUserConversationsMinSeqs(ctx context.Context, userID string, seqs map[string]int64) (err error)
|
||||
SetHasReadSeq(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error
|
||||
SetHasReadSeq(ctx context.Context, userID string, conversationID string, hasReadSeq int64) (int64, error)
|
||||
GetHasReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error)
|
||||
GetHasReadSeq(ctx context.Context, userID string, conversationID string) (int64, error)
|
||||
UserSetHasReadSeqs(ctx context.Context, userID string, hasReadSeqs map[string]int64) error
|
||||
@ -585,7 +585,7 @@ func (db *commonMsgDatabase) UserSetHasReadSeqs(ctx context.Context, userID stri
|
||||
return db.seqUser.SetUserReadSeqs(ctx, userID, hasReadSeqs)
|
||||
}
|
||||
|
||||
func (db *commonMsgDatabase) SetHasReadSeq(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error {
|
||||
func (db *commonMsgDatabase) SetHasReadSeq(ctx context.Context, userID string, conversationID string, hasReadSeq int64) (int64, error) {
|
||||
return db.seqUser.SetUserReadSeq(ctx, conversationID, userID, hasReadSeq)
|
||||
}
|
||||
|
||||
|
||||
@ -238,9 +238,12 @@ func (db *msgTransferDatabase) BatchInsertChat2Cache(ctx context.Context, conver
|
||||
|
||||
func (db *msgTransferDatabase) SetHasReadSeqs(ctx context.Context, conversationID string, userSeqMap map[string]int64) error {
|
||||
for userID, seq := range userSeqMap {
|
||||
if err := db.seqUser.SetUserReadSeq(ctx, conversationID, userID, seq); err != nil {
|
||||
effectiveSeq, err := db.seqUser.SetUserReadSeq(ctx, conversationID, userID, seq)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// The same map is queued for asynchronous persistence by msgtransfer.
|
||||
userSeqMap[userID] = effectiveSeq
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
22
pkg/common/storage/controller/msg_transfer_read_seq_test.go
Normal file
22
pkg/common/storage/controller/msg_transfer_read_seq_test.go
Normal file
@ -0,0 +1,22 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type effectiveReadSeqCache struct{ cache.SeqUser }
|
||||
|
||||
func (*effectiveReadSeqCache) SetUserReadSeq(context.Context, string, string, int64) (int64, error) {
|
||||
return 346, nil
|
||||
}
|
||||
|
||||
func TestReadSeqSenderQueuesEffectiveProgress(t *testing.T) {
|
||||
db := &msgTransferDatabase{seqUser: &effectiveReadSeqCache{}}
|
||||
pending := map[string]int64{"user": 200}
|
||||
require.NoError(t, db.SetHasReadSeqs(context.Background(), "group", pending))
|
||||
require.EqualValues(t, 346, pending["user"])
|
||||
}
|
||||
@ -3,9 +3,11 @@ package mgo
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"github.com/openimsdk/open-im-server/v3/pkg/common/storage/database"
|
||||
"github.com/openimsdk/open-im-server/v3/pkg/common/storage/model"
|
||||
"github.com/openimsdk/tools/db/mongoutil"
|
||||
"github.com/openimsdk/tools/errs"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
@ -18,9 +20,13 @@ func NewSeqUserMongo(db *mongo.Database) (database.SeqUser, error) {
|
||||
{Key: "user_id", Value: 1},
|
||||
{Key: "conversation_id", Value: 1},
|
||||
},
|
||||
// A distinct name allows the old non-unique index to coexist on
|
||||
// supported MongoDB versions. Existing duplicate rows must be resolved
|
||||
// before rollout; never silently pick one read sequence from duplicates.
|
||||
Options: options.Index().SetName("user_conversation_unique").SetUnique(true),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, errs.WrapMsg(err, "ensure unique seq_user index; check duplicate user_id/conversation_id rows before rollout")
|
||||
}
|
||||
return &seqUserMongo{coll: coll}, nil
|
||||
}
|
||||
@ -30,6 +36,10 @@ type seqUserMongo struct {
|
||||
}
|
||||
|
||||
func (s *seqUserMongo) setSeq(ctx context.Context, conversationID string, userID string, seq int64, field string) error {
|
||||
return s.updateSeq(ctx, conversationID, userID, seq, field, "$set")
|
||||
}
|
||||
|
||||
func (s *seqUserMongo) updateSeq(ctx context.Context, conversationID, userID string, seq int64, field, operator string) error {
|
||||
filter := map[string]any{
|
||||
"user_id": userID,
|
||||
"conversation_id": conversationID,
|
||||
@ -43,13 +53,19 @@ func (s *seqUserMongo) setSeq(ctx context.Context, conversationID string, userID
|
||||
}
|
||||
delete(insert, field)
|
||||
update := map[string]any{
|
||||
"$set": bson.M{
|
||||
operator: bson.M{
|
||||
field: seq,
|
||||
},
|
||||
"$setOnInsert": insert,
|
||||
}
|
||||
opt := options.Update().SetUpsert(true)
|
||||
return mongoutil.UpdateOne(ctx, s.coll, filter, update, false, opt)
|
||||
err := mongoutil.UpdateOne(ctx, s.coll, filter, update, false, opt)
|
||||
if mongo.IsDuplicateKeyError(err) {
|
||||
// Another writer may have inserted the same user/conversation during
|
||||
// this upsert. Retry against that row without creating another one.
|
||||
return mongoutil.UpdateOne(ctx, s.coll, filter, update, true)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *seqUserMongo) getSeq(ctx context.Context, conversationID string, userID string, failed string) (int64, error) {
|
||||
@ -115,12 +131,8 @@ func (s *seqUserMongo) GetUserReadSeqs(ctx context.Context, userID string, conve
|
||||
}
|
||||
|
||||
func (s *seqUserMongo) SetUserReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error {
|
||||
dbSeq, err := s.GetUserReadSeq(ctx, conversationID, userID)
|
||||
if err != nil {
|
||||
return err
|
||||
if seq < 0 {
|
||||
return errs.ErrArgs.WrapMsg("read seq must not be negative")
|
||||
}
|
||||
if dbSeq > seq {
|
||||
return nil
|
||||
}
|
||||
return s.setSeq(ctx, conversationID, userID, seq, "read_seq")
|
||||
return s.updateSeq(ctx, conversationID, userID, seq, "read_seq", "$max")
|
||||
}
|
||||
|
||||
85
pkg/common/storage/database/mgo/seq_user_monotonic_test.go
Normal file
85
pkg/common/storage/database/mgo/seq_user_monotonic_test.go
Normal file
@ -0,0 +1,85 @@
|
||||
package mgo
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo/integration/mtest"
|
||||
)
|
||||
|
||||
func TestReadSeqMongoAtomicMax(t *testing.T) {
|
||||
mt := mtest.New(t, mtest.NewOptions().ClientType(mtest.Mock))
|
||||
mt.Run("out of order receipts use max without a preceding read", func(mt *mtest.T) {
|
||||
s := &seqUserMongo{coll: mt.Coll}
|
||||
for _, seq := range []int64{346, 200, 346} {
|
||||
mt.AddMockResponses(mtest.CreateSuccessResponse(bson.E{Key: "n", Value: 1}, bson.E{Key: "nModified", Value: 1}))
|
||||
require.NoError(mt, s.SetUserReadSeq(context.Background(), "group", "user", seq))
|
||||
}
|
||||
events := mt.GetAllStartedEvents()
|
||||
require.Len(mt, events, 3)
|
||||
for i, event := range events {
|
||||
require.Equal(mt, "update", event.CommandName)
|
||||
updates, err := event.Command.Lookup("updates").Array().Values()
|
||||
require.NoError(mt, err)
|
||||
update := updates[0].Document()
|
||||
require.Equal(mt, "group", update.Lookup("q", "conversation_id").StringValue())
|
||||
require.Equal(mt, "user", update.Lookup("q", "user_id").StringValue())
|
||||
require.Equal(mt, []int64{346, 200, 346}[i], update.Lookup("u", "$max", "read_seq").Int64())
|
||||
_, err = update.LookupErr("u", "$setOnInsert", "read_seq")
|
||||
require.Error(mt, err, "insertion defaults must not conflict with $max")
|
||||
require.True(mt, update.Lookup("upsert").Boolean())
|
||||
}
|
||||
})
|
||||
mt.Run("concurrent upsert retries the existing unique row", func(mt *mtest.T) {
|
||||
mt.AddMockResponses(
|
||||
mtest.CreateWriteErrorsResponse(mtest.WriteError{Index: 0, Code: 11000, Message: "duplicate user/conversation"}),
|
||||
mtest.CreateSuccessResponse(bson.E{Key: "n", Value: 1}, bson.E{Key: "nModified", Value: 1}),
|
||||
)
|
||||
s := &seqUserMongo{coll: mt.Coll}
|
||||
require.NoError(mt, s.SetUserReadSeq(context.Background(), "group", "user", 346))
|
||||
events := mt.GetAllStartedEvents()
|
||||
require.Len(mt, events, 2)
|
||||
updates, err := events[1].Command.Lookup("updates").Array().Values()
|
||||
require.NoError(mt, err)
|
||||
update := updates[0].Document()
|
||||
upsert, err := update.LookupErr("upsert")
|
||||
require.True(mt, err != nil || !upsert.Boolean())
|
||||
require.EqualValues(mt, 346, update.Lookup("u", "$max", "read_seq").Int64())
|
||||
})
|
||||
mt.Run("database failures propagate", func(mt *mtest.T) {
|
||||
mt.AddMockResponses(mtest.CreateCommandErrorResponse(mtest.CommandError{Code: 91, Message: "unavailable"}))
|
||||
s := &seqUserMongo{coll: mt.Coll}
|
||||
require.Error(mt, s.SetUserReadSeq(context.Background(), "group", "user", 346))
|
||||
})
|
||||
mt.Run("negative sequences do not issue a command", func(mt *mtest.T) {
|
||||
s := &seqUserMongo{coll: mt.Coll}
|
||||
require.Error(mt, s.SetUserReadSeq(context.Background(), "group", "user", -1))
|
||||
require.Empty(mt, mt.GetAllStartedEvents())
|
||||
})
|
||||
}
|
||||
|
||||
func TestReadSeqMongoUniqueIndex(t *testing.T) {
|
||||
mt := mtest.New(t, mtest.NewOptions().ClientType(mtest.Mock))
|
||||
mt.Run("one record per user and conversation", func(mt *mtest.T) {
|
||||
mt.AddMockResponses(mtest.CreateSuccessResponse())
|
||||
_, err := NewSeqUserMongo(mt.DB)
|
||||
require.NoError(mt, err)
|
||||
events := mt.GetAllStartedEvents()
|
||||
require.Len(mt, events, 1)
|
||||
require.Equal(mt, "createIndexes", events[0].CommandName)
|
||||
indexes, err := events[0].Command.Lookup("indexes").Array().Values()
|
||||
require.NoError(mt, err)
|
||||
index := indexes[0].Document()
|
||||
require.True(mt, index.Lookup("unique").Boolean())
|
||||
require.Equal(mt, "user_conversation_unique", index.Lookup("name").StringValue())
|
||||
require.EqualValues(mt, 1, index.Lookup("key", "user_id").Int32())
|
||||
require.EqualValues(mt, 1, index.Lookup("key", "conversation_id").Int32())
|
||||
})
|
||||
mt.Run("existing duplicates fail explicitly", func(mt *mtest.T) {
|
||||
mt.AddMockResponses(mtest.CreateCommandErrorResponse(mtest.CommandError{Code: 11000, Message: "duplicate rows"}))
|
||||
_, err := NewSeqUserMongo(mt.DB)
|
||||
require.ErrorContains(mt, err, "check duplicate user_id/conversation_id rows before rollout")
|
||||
})
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user