diff --git a/pkg/common/storage/database/mgo/conversation.go b/pkg/common/storage/database/mgo/conversation.go index 30f08f5e5..c8ca07774 100644 --- a/pkg/common/storage/database/mgo/conversation.go +++ b/pkg/common/storage/database/mgo/conversation.go @@ -29,6 +29,7 @@ import ( "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/db/pagination" "github.com/openimsdk/tools/errs" + "github.com/openimsdk/tools/utils/datautil" ) func NewConversationMongo(db *mongo.Database) (*ConversationMgo, error) { @@ -70,15 +71,35 @@ type ConversationMgo struct { } func (c *ConversationMgo) Create(ctx context.Context, conversations []*model.Conversation) (err error) { + if len(conversations) == 0 { + return nil + } return mongoutil.IncrVersion(func() error { - return mongoutil.InsertMany(ctx, c.coll, conversations) + models := make([]mongo.WriteModel, len(conversations)) + for i, conversation := range conversations { + models[i] = mongo.NewUpdateOneModel(). + SetFilter(bson.M{ + "owner_user_id": conversation.OwnerUserID, + "conversation_id": conversation.ConversationID, + }). + SetUpdate(bson.M{"$setOnInsert": conversation}). + SetUpsert(true) + } + err := mongoutil.BulkWrite(ctx, c.coll, models, options.BulkWrite().SetOrdered(false)) + if err != nil { + if IsAllDuplicateKeyError(err) { + return nil + } + return err + } + return nil }, func() error { userConversation := make(map[string][]string) for _, conversation := range conversations { userConversation[conversation.OwnerUserID] = append(userConversation[conversation.OwnerUserID], conversation.ConversationID) } for userID, conversationIDs := range userConversation { - if err := c.version.IncrVersion(ctx, userID, conversationIDs, model.VersionStateInsert); err != nil { + if err := c.version.IncrVersion(ctx, userID, datautil.Distinct(conversationIDs), model.VersionStateInsert); err != nil { return err } } diff --git a/pkg/common/storage/database/mgo/helpers.go b/pkg/common/storage/database/mgo/helpers.go index 23e66236a..f77d433f1 100644 --- a/pkg/common/storage/database/mgo/helpers.go +++ b/pkg/common/storage/database/mgo/helpers.go @@ -15,6 +15,9 @@ package mgo import ( + "errors" + "strings" + "github.com/openimsdk/tools/errs" "go.mongodb.org/mongo-driver/mongo" ) @@ -22,3 +25,49 @@ import ( func IsNotFound(err error) bool { return errs.Unwrap(err) == mongo.ErrNoDocuments } + +// IsAllDuplicateKeyError returns true if err is a duplicate key error or +// if err is a bulk write error where all write errors are duplicate key errors. +func IsAllDuplicateKeyError(err error) bool { + if err == nil { + return false + } + var bwe mongo.BulkWriteException + if errors.As(err, &bwe) { + if bwe.WriteConcernError != nil { + return false + } + if len(bwe.WriteErrors) == 0 { + return false + } + for _, we := range bwe.WriteErrors { + if !isDuplicateKeyWriteError(we.WriteError) { + return false + } + } + return true + } + var we mongo.WriteException + if errors.As(err, &we) { + if we.WriteConcernError != nil { + return false + } + if len(we.WriteErrors) == 0 { + return false + } + for _, w := range we.WriteErrors { + if !isDuplicateKeyWriteError(w) { + return false + } + } + return true + } + return mongo.IsDuplicateKeyError(err) +} + +func isDuplicateKeyWriteError(we mongo.WriteError) bool { + return we.Code == 11000 || we.Code == 11001 || we.Code == 12582 || + strings.Contains(we.Message, " E11000 ") || + strings.Contains(we.Message, "duplicate key") || + mongo.IsDuplicateKeyError(we) +} diff --git a/pkg/common/storage/database/mgo/helpers_test.go b/pkg/common/storage/database/mgo/helpers_test.go new file mode 100644 index 000000000..d35a83477 --- /dev/null +++ b/pkg/common/storage/database/mgo/helpers_test.go @@ -0,0 +1,189 @@ +// Copyright © 2023 OpenIM. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mgo + +import ( + "errors" + "testing" + + "github.com/openimsdk/tools/errs" + "go.mongodb.org/mongo-driver/mongo" +) + +func TestIsAllDuplicateKeyError(t *testing.T) { + tests := []struct { + name string + err error + expected bool + }{ + { + name: "nil error", + err: nil, + expected: false, + }, + { + name: "generic non-duplicate error", + err: errors.New("connection refused"), + expected: false, + }, + { + name: "single CommandError duplicate key 11000", + err: mongo.CommandError{ + Code: 11000, + Message: "E11000 duplicate key error collection: openim_v3.conversation", + }, + expected: true, + }, + { + name: "single CommandError non-duplicate key 121", + err: mongo.CommandError{ + Code: 121, + Message: "Document failed validation", + }, + expected: false, + }, + { + name: "BulkWriteException with single duplicate key write error", + err: mongo.BulkWriteException{ + WriteErrors: []mongo.BulkWriteError{ + { + WriteError: mongo.WriteError{ + Code: 11000, + Message: "E11000 duplicate key error collection: openim_v3.conversation index: owner_user_id_1_conversation_id_1", + }, + }, + }, + }, + expected: true, + }, + { + name: "BulkWriteException with multiple duplicate key write errors", + err: mongo.BulkWriteException{ + WriteErrors: []mongo.BulkWriteError{ + { + WriteError: mongo.WriteError{ + Code: 11000, + Message: "E11000 duplicate key error index 0", + }, + }, + { + WriteError: mongo.WriteError{ + Code: 11001, + Message: "duplicate key error on update index 1", + }, + }, + }, + }, + expected: true, + }, + { + name: "BulkWriteException with mixed duplicate key and other write errors", + err: mongo.BulkWriteException{ + WriteErrors: []mongo.BulkWriteError{ + { + WriteError: mongo.WriteError{ + Code: 11000, + Message: "E11000 duplicate key error", + }, + }, + { + WriteError: mongo.WriteError{ + Code: 121, + Message: "Document failed validation", + }, + }, + }, + }, + expected: false, + }, + { + name: "BulkWriteException with duplicate key but has WriteConcernError", + err: mongo.BulkWriteException{ + WriteConcernError: &mongo.WriteConcernError{ + Code: 64, + Message: "waiting for replication timed out", + }, + WriteErrors: []mongo.BulkWriteError{ + { + WriteError: mongo.WriteError{ + Code: 11000, + Message: "E11000 duplicate key error", + }, + }, + }, + }, + expected: false, + }, + { + name: "BulkWriteException with empty WriteErrors", + err: mongo.BulkWriteException{ + WriteErrors: []mongo.BulkWriteError{}, + }, + expected: false, + }, + { + name: "WriteException with single duplicate key write error", + err: mongo.WriteException{ + WriteErrors: []mongo.WriteError{ + { + Code: 11000, + Message: "E11000 duplicate key error", + }, + }, + }, + expected: true, + }, + { + name: "WriteException with mixed errors", + err: mongo.WriteException{ + WriteErrors: []mongo.WriteError{ + { + Code: 11000, + Message: "E11000 duplicate key error", + }, + { + Code: 14, + Message: "TypeMismatch", + }, + }, + }, + expected: false, + }, + { + name: "Wrapped BulkWriteException using errs.WrapMsg", + err: errs.WrapMsg(mongo.BulkWriteException{ + WriteErrors: []mongo.BulkWriteError{ + { + WriteError: mongo.WriteError{ + Code: 11000, + Message: "E11000 duplicate key error", + }, + }, + }, + }, "mongo bulk write"), + expected: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := IsAllDuplicateKeyError(tc.err) + if got != tc.expected { + t.Errorf("IsAllDuplicateKeyError() = %v, want %v", got, tc.expected) + } + }) + } +} +