blackteay/open-im-server
1
0
Fork 0
mirror of https://github.com/openimsdk/open-im-server.git synced 2025-04-06 04:15:46 +08:00
Code Issues Packages Projects Releases Wiki Activity

Refactoring code

Browse Source
This commit is contained in:
wenxu12345 2022-02-21 15:01:06 +08:00
parent 4858390346
commit 571c83314a
8 changed files with 354 additions and 119 deletions

2
cmd/Open-IM-SDK-Core

@ -1 +1 @@
Subproject commit e43ec7d427a84702eea7a6aaa358a7a0a809019d
Subproject commit bdb27fcff0e9785cb0f92d9d04bdafe544a04de0

69
cmd/test/main.go Normal file

@ -0,0 +1,69 @@
package main
import (
"Open_IM/pkg/utils"
"context"
"fmt"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"time"
)
type MongoMsg struct {
UID string
Msg []string
}
func main() {
//"mongodb://%s:%s@%s/%s/?maxPoolSize=%d"
uri := "mongodb://user:pass@sample.host:27017/?maxPoolSize=20&w=majority"
DBAddress := "127.0.0.1:37017"
DBDatabase := "new-test-db"
Collection := "new-test-collection"
DBMaxPoolSize := 100
uri = fmt.Sprintf("mongodb://%s/%s/?maxPoolSize=%d",
DBAddress,DBDatabase,
DBMaxPoolSize)
mongoClient, err := mongo.Connect(context.TODO(), options.Client().ApplyURI(uri))
if err != nil {
panic(err)
}
filter := bson.M{"uid":"my_uid"}
ctx, _ := context.WithTimeout(context.Background(), 30*time.Second)
for i:=0; i < 2; i++{
if err = mongoClient.Database(DBDatabase).Collection(Collection).FindOneAndUpdate(ctx, filter,
bson.M{"$push": bson.M{"msg": utils.Int32ToString(int32(i))}}).Err(); err != nil{
fmt.Println("FindOneAndUpdate failed ", i, )
var mmsg MongoMsg
mmsg.UID = "my_uid"
mmsg.Msg = append(mmsg.Msg, utils.Int32ToString(int32(i)))
_, err := mongoClient.Database(DBDatabase).Collection(Collection).InsertOne(ctx, &mmsg)
if err != nil {
fmt.Println("insertone failed ", err.Error(), i)
} else{
fmt.Println("insertone ok ", i)
}
}else {
fmt.Println("FindOneAndUpdate ok ", i)
}
}
var mmsg MongoMsg
if err = mongoClient.Database(DBDatabase).Collection(Collection).FindOne(ctx, filter).Decode(&mmsg); err != nil {
fmt.Println("findone failed ", err.Error())
}else{
fmt.Println("findone ok ", mmsg.UID)
for i, v:=range mmsg.Msg{
fmt.Println("find value: ", i, v)
}
}
}

1
go.mod

@ -45,6 +45,7 @@ require (
github.com/tencentyun/qcloud-cos-sts-sdk v0.0.0-20210325043845-84a0811633ca
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect
go.etcd.io/etcd v0.0.0-20200402134248-51bdeb39e698
go.mongodb.org/mongo-driver v1.8.3
golang.org/x/image v0.0.0-20210220032944-ac19c3e999fb
golang.org/x/net v0.0.0-20210917221730-978cfadd31cf
google.golang.org/grpc v1.40.0

15
pkg/base_info/conversation_api_struct.go

@ -31,3 +31,18 @@ type SetReceiveMessageOptResp struct {
CommResp
ConversationOptResultList []*OptResult `json:"data"`
}
//type Conversation struct {
// OwnerUserID string `gorm:"column:owner_user_id;primary_key;type:char(128)" json:"OwnerUserID"`
// ConversationID string `gorm:"column:conversation_id;primary_key;type:char(128)" json:"conversationID"`
// ConversationType int32 `gorm:"column:conversation_type" json:"conversationType"`
// UserID string `gorm:"column:user_id;type:char(64)" json:"userID"`
// GroupID string `gorm:"column:group_id;type:char(128)" json:"groupID"`
// RecvMsgOpt int32 `gorm:"column:recv_msg_opt" json:"recvMsgOpt"`
// UnreadCount int32 `gorm:"column:unread_count" json:"unreadCount"`
// DraftTextTime int64 `gorm:"column:draft_text_time" json:"draftTextTime"`
// IsPinned bool `gorm:"column:is_pinned" json:"isPinned"`
// AttachedInfo string `gorm:"column:attached_info;type:varchar(1024)" json:"attachedInfo"`
// Ex string `gorm:"column:ex;type:varchar(1024)" json:"ex"`
//}

14
pkg/base_info/public_struct.go

@ -16,6 +16,20 @@ type ApiUserInfo struct {
Ex string `json:"ex" binding:"omitempty,max=1024"`
}
//type Conversation struct {
// OwnerUserID string `gorm:"column:owner_user_id;primary_key;type:char(128)" json:"OwnerUserID"`
// ConversationID string `gorm:"column:conversation_id;primary_key;type:char(128)" json:"conversationID"`
// ConversationType int32 `gorm:"column:conversation_type" json:"conversationType"`
// UserID string `gorm:"column:user_id;type:char(64)" json:"userID"`
// GroupID string `gorm:"column:group_id;type:char(128)" json:"groupID"`
// RecvMsgOpt int32 `gorm:"column:recv_msg_opt" json:"recvMsgOpt"`
// UnreadCount int32 `gorm:"column:unread_count" json:"unreadCount"`
// DraftTextTime int64 `gorm:"column:draft_text_time" json:"draftTextTime"`
// IsPinned bool `gorm:"column:is_pinned" json:"isPinned"`
// AttachedInfo string `gorm:"column:attached_info;type:varchar(1024)" json:"attachedInfo"`
// Ex string `gorm:"column:ex;type:varchar(1024)" json:"ex"`
//}
type GroupAddMemberInfo struct {
UserID string `json:"userID" binding:"required"`
RoleLevel int32 `json:"roleLevel" binding:"required"`

2
pkg/common/constant/constant.go

@ -176,3 +176,5 @@ func GroupIsBanPrivateChat(status int32) bool {
}
return true
}
const BigVersion = "v3"

30
pkg/common/db/model.go

@ -3,9 +3,16 @@ package db
import (
"Open_IM/pkg/common/config"
"Open_IM/pkg/common/log"
"context"
"fmt"
"github.com/garyburd/redigo/redis"
"gopkg.in/mgo.v2"
"time"
//"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
var DB DataBases
@ -14,6 +21,7 @@ type DataBases struct {
MysqlDB mysqlDB
mgoSession *mgo.Session
redisPool *redis.Pool
mongoClient *mongo.Client
}
func key(dbAddress, dbName string) string {
@ -22,10 +30,21 @@ func key(dbAddress, dbName string) string {
func init() {
var mgoSession *mgo.Session
var mongoClient *mongo.Client
var err1 error
//mysql init
initMysqlDB()
// mongo init
// "mongodb://sysop:moon@localhost/records"
// uri := "mongodb://user:pass@sample.host:27017/?maxPoolSize=20&w=majority"
uri := fmt.Sprintf("mongodb://%s:%s@%s/%s/?maxPoolSize=%d",
config.Config.Mongo.DBUserName, config.Config.Mongo.DBPassword,
config.Config.Mongo.DBAddress[0],config.Config.Mongo.DBDatabase,
config.Config.Mongo.DBMaxPoolSize)
mongoClient, err := mongo.Connect(context.TODO(), options.Client().ApplyURI(uri))
mgoDailInfo := &mgo.DialInfo{
Addrs: config.Config.Mongo.DBAddress,
Direct: config.Config.Mongo.DBDirect,
@ -36,17 +55,19 @@ func init() {
Password: config.Config.Mongo.DBPassword,
PoolLimit: config.Config.Mongo.DBMaxPoolSize,
}
mgoSession, err := mgo.DialWithInfo(mgoDailInfo)
if err != nil {
log.NewError("mgo init err", err.Error(), mgoDailInfo)
}
mgoSession, err = mgo.DialWithInfo(mgoDailInfo)
if err != nil {
log.NewError(" mongo.Connect failed, try ", err.Error(), uri)
time.Sleep(time.Duration(30) * time.Second)
mongoClient, err1 = mongo.Connect(context.TODO(), options.Client().ApplyURI(uri))
mgoSession, err1 = mgo.DialWithInfo(mgoDailInfo)
if err1 != nil {
log.NewError(" mongo.Connect failed, panic", err.Error(), uri)
panic(err1.Error())
}
}
DB.mongoClient = mongoClient
DB.mgoSession = mgoSession
DB.mgoSession.SetMode(mgo.Monotonic, true)
c := DB.mgoSession.DB(config.Config.Mongo.DBDatabase).C(cChat)
@ -55,6 +76,7 @@ func init() {
panic(err.Error())
}
// redis pool init
DB.redisPool = &redis.Pool{
MaxIdle: config.Config.Redis.DBMaxIdle,

340
pkg/common/db/mongoModel.go

@ -6,10 +6,12 @@ import (
pbMsg "Open_IM/pkg/proto/chat"
open_im_sdk "Open_IM/pkg/proto/sdk_ws"
"Open_IM/pkg/utils"
"context"
"errors"
"github.com/garyburd/redigo/redis"
//"github.com/garyburd/redigo/redis"
"github.com/golang/protobuf/proto"
"gopkg.in/mgo.v2/bson"
"strconv"
"time"
)
@ -34,33 +36,35 @@ type GroupMember_x struct {
}
func (d *DataBases) GetMinSeqFromMongo(uid string) (MinSeq uint32, err error) {
var i, NB uint32
var seqUid string
session := d.mgoSession.Clone()
if session == nil {
return MinSeq, errors.New("session == nil")
}
defer session.Close()
c := session.DB(config.Config.Mongo.DBDatabase).C(cChat)
MaxSeq, err := d.GetUserMaxSeq(uid)
if err != nil && err != redis.ErrNil {
return MinSeq, err
}
NB = uint32(MaxSeq / singleGocMsgNum)
for i = 0; i <= NB; i++ {
seqUid = indexGen(uid, i)
n, err := c.Find(bson.M{"uid": seqUid}).Count()
if err == nil && n != 0 {
if i == 0 {
MinSeq = 1
} else {
MinSeq = uint32(i * singleGocMsgNum)
}
break
}
}
return MinSeq, nil
return 1, nil
//var i, NB uint32
//var seqUid string
//session := d.mgoSession.Clone()
//if session == nil {
// return MinSeq, errors.New("session == nil")
//}
//defer session.Close()
//c := session.DB(config.Config.Mongo.DBDatabase).C(cChat)
//MaxSeq, err := d.GetUserMaxSeq(uid)
//if err != nil && err != redis.ErrNil {
// return MinSeq, err
//}
//NB = uint32(MaxSeq / singleGocMsgNum)
//for i = 0; i <= NB; i++ {
// seqUid = indexGen(uid, i)
// n, err := c.Find(bson.M{"uid": seqUid}).Count()
// if err == nil && n != 0 {
// if i == 0 {
// MinSeq = 1
// } else {
// MinSeq = uint32(i * singleGocMsgNum)
// }
// break
// }
//}
//return MinSeq, nil
}
func (d *DataBases) GetMsgBySeqList(uid string, seqList []uint32, operationID string) (seqMsg []*open_im_sdk.MsgData, err error) {
var hasSeqList []uint32
singleCount := 0
@ -115,6 +119,61 @@ func (d *DataBases) GetMsgBySeqList(uid string, seqList []uint32, operationID st
}
return seqMsg, nil
}
func (d *DataBases) GetMsgBySeqListMongo2(uid string, seqList []uint32, operationID string) (seqMsg []*open_im_sdk.MsgData, err error) {
var hasSeqList []uint32
singleCount := 0
ctx, _ := context.WithTimeout(context.Background(), time.Duration(config.Config.Mongo.DBTimeout)*time.Second)
c := d.mongoClient.Database(config.Config.Mongo.DBDatabase).Collection(cChat)
m := func(uid string, seqList []uint32) map[string][]uint32 {
t := make(map[string][]uint32)
for i := 0; i < len(seqList); i++ {
seqUid := getSeqUid(uid, seqList[i])
if value, ok := t[seqUid]; !ok {
var temp []uint32
t[seqUid] = append(temp, seqList[i])
} else {
t[seqUid] = append(value, seqList[i])
}
}
return t
}(uid, seqList)
sChat := UserChat{}
for seqUid, value := range m {
if err = c.FindOne(ctx, bson.M{"uid": seqUid}).Decode(&sChat); err != nil {
log.NewError(operationID, "not find seqUid", seqUid, value, uid, seqList, err.Error())
continue
}
singleCount = 0
for i := 0; i < len(sChat.Msg); i++ {
msg := new(open_im_sdk.MsgData)
if err = proto.Unmarshal(sChat.Msg[i].Msg, msg); err != nil {
log.NewError(operationID, "Unmarshal err", seqUid, value, uid, seqList, err.Error())
return nil, err
}
if isContainInt32(msg.Seq, value) {
seqMsg = append(seqMsg, msg)
hasSeqList = append(hasSeqList, msg.Seq)
singleCount++
if singleCount == len(value) {
break
}
}
}
}
if len(hasSeqList) != len(seqList) {
var diff []uint32
diff = utils.Difference(hasSeqList, seqList)
exceptionMSg := genExceptionMessageBySeqList(diff)
seqMsg = append(seqMsg, exceptionMSg...)
}
return seqMsg, nil
}
func genExceptionMessageBySeqList(seqList []uint32) (exceptionMsg []*open_im_sdk.MsgData) {
for _, v := range seqList {
msg := new(open_im_sdk.MsgData)
@ -124,6 +183,37 @@ func genExceptionMessageBySeqList(seqList []uint32) (exceptionMsg []*open_im_sdk
return exceptionMsg
}
func (d *DataBases) SaveUserChatMongo2(uid string, sendTime int64, m *pbMsg.MsgDataToDB) error {
ctx, _ := context.WithTimeout(context.Background(), time.Duration(config.Config.Mongo.DBTimeout)*time.Second)
c := d.mongoClient.Database(config.Config.Mongo.DBDatabase).Collection(cChat)
newTime := getCurrentTimestampByMill()
operationID := ""
seqUid := getSeqUid(uid, m.MsgData.Seq)
filter := bson.M{"uid": seqUid}
var err error
sMsg := MsgInfo{}
sMsg.SendTime = sendTime
if sMsg.Msg, err = proto.Marshal(m.MsgData); err != nil {
return utils.Wrap(err,"")
}
err = c.FindOneAndUpdate(ctx, filter, bson.M{"$push": bson.M{"msg": sMsg}}).Err()
log.NewDebug(operationID, "get mgoSession cost time", getCurrentTimestampByMill()-newTime)
if err != nil {
sChat := UserChat{}
sChat.UID = seqUid
sChat.Msg = append(sChat.Msg, sMsg)
if _, err = c.InsertOne(ctx, &sChat) ; err != nil{
log.NewDebug(operationID, "InsertOne failed", filter)
return utils.Wrap(err, "")
}
}else{
log.NewDebug(operationID, "FindOneAndUpdate ok", filter)
}
log.NewDebug(operationID, "find mgo uid cost time", getCurrentTimestampByMill()-newTime)
return nil
}
func (d *DataBases) SaveUserChat(uid string, sendTime int64, m *pbMsg.MsgDataToDB) error {
var seqUid string
newTime := getCurrentTimestampByMill()
@ -163,115 +253,137 @@ func (d *DataBases) SaveUserChat(uid string, sendTime int64, m *pbMsg.MsgDataToD
return nil
}
func (d *DataBases) DelUserChat(uid string) error {
session := d.mgoSession.Clone()
if session == nil {
return errors.New("session == nil")
}
defer session.Close()
c := session.DB(config.Config.Mongo.DBDatabase).C(cChat)
func (d *DataBases) DelUserChat(uid string) error {
return nil
//session := d.mgoSession.Clone()
//if session == nil {
// return errors.New("session == nil")
//}
//defer session.Close()
//
//c := session.DB(config.Config.Mongo.DBDatabase).C(cChat)
//
//delTime := time.Now().Unix() - int64(config.Config.Mongo.DBRetainChatRecords)*24*3600
//if err := c.Update(bson.M{"uid": uid}, bson.M{"$pull": bson.M{"msg": bson.M{"sendtime": bson.M{"$lte": delTime}}}}); err != nil {
// return err
//}
//
//return nil
}
func (d *DataBases) DelUserChatMongo2(uid string) error {
ctx, _ := context.WithTimeout(context.Background(), time.Duration(config.Config.Mongo.DBTimeout)*time.Second)
c := d.mongoClient.Database(config.Config.Mongo.DBDatabase).Collection(cChat)
filter := bson.M{"uid": uid}
delTime := time.Now().Unix() - int64(config.Config.Mongo.DBRetainChatRecords)*24*3600
if err := c.Update(bson.M{"uid": uid}, bson.M{"$pull": bson.M{"msg": bson.M{"sendtime": bson.M{"$lte": delTime}}}}); err != nil {
return err
if _, err := c.UpdateOne(ctx, filter, bson.M{"$pull": bson.M{"msg": bson.M{"sendtime": bson.M{"$lte": delTime}}}}); err != nil {
return utils.Wrap(err, "")
}
return nil
}
func (d *DataBases) MgoUserCount() (int, error) {
session := d.mgoSession.Clone()
if session == nil {
return 0, errors.New("session == nil")
}
defer session.Close()
c := session.DB(config.Config.Mongo.DBDatabase).C(cChat)
return c.Find(nil).Count()
return 0, nil
//session := d.mgoSession.Clone()
//if session == nil {
// return 0, errors.New("session == nil")
//}
//defer session.Close()
//
//c := session.DB(config.Config.Mongo.DBDatabase).C(cChat)
//
//return c.Find(nil).Count()
}
func (d *DataBases) MgoSkipUID(count int) (string, error) {
session := d.mgoSession.Clone()
if session == nil {
return "", errors.New("session == nil")
}
defer session.Close()
c := session.DB(config.Config.Mongo.DBDatabase).C(cChat)
sChat := UserChat{}
c.Find(nil).Skip(count).Limit(1).One(&sChat)
return sChat.UID, nil
return "", nil
//session := d.mgoSession.Clone()
//if session == nil {
// return "", errors.New("session == nil")
//}
//defer session.Close()
//
//c := session.DB(config.Config.Mongo.DBDatabase).C(cChat)
//
//sChat := UserChat{}
//c.Find(nil).Skip(count).Limit(1).One(&sChat)
//return sChat.UID, nil
}
func (d *DataBases) GetGroupMember(groupID string) []string {
groupInfo := GroupMember_x{}
groupInfo.GroupID = groupID
groupInfo.UIDList = make([]string, 0)
session := d.mgoSession.Clone()
if session == nil {
return groupInfo.UIDList
}
defer session.Close()
c := session.DB(config.Config.Mongo.DBDatabase).C(cGroup)
if err := c.Find(bson.M{"groupid": groupInfo.GroupID}).One(&groupInfo); err != nil {
return groupInfo.UIDList
}
return groupInfo.UIDList
return nil
//groupInfo := GroupMember_x{}
//groupInfo.GroupID = groupID
//groupInfo.UIDList = make([]string, 0)
//
//session := d.mgoSession.Clone()
//if session == nil {
// return groupInfo.UIDList
//}
//defer session.Close()
//
//c := session.DB(config.Config.Mongo.DBDatabase).C(cGroup)
//
//if err := c.Find(bson.M{"groupid": groupInfo.GroupID}).One(&groupInfo); err != nil {
// return groupInfo.UIDList
//}
//
//return groupInfo.UIDList
}
func (d *DataBases) AddGroupMember(groupID, uid string) error {
session := d.mgoSession.Clone()
if session == nil {
return errors.New("session == nil")
}
defer session.Close()
c := session.DB(config.Config.Mongo.DBDatabase).C(cGroup)
n, err := c.Find(bson.M{"groupid": groupID}).Count()
if err != nil {
return err
}
if n == 0 {
groupInfo := GroupMember_x{}
groupInfo.GroupID = groupID
groupInfo.UIDList = append(groupInfo.UIDList, uid)
err = c.Insert(&groupInfo)
if err != nil {
return err
}
} else {
err = c.Update(bson.M{"groupid": groupID}, bson.M{"$addToSet": bson.M{"uidlist": uid}})
if err != nil {
return err
}
}
return nil
//session := d.mgoSession.Clone()
//if session == nil {
// return errors.New("session == nil")
//}
//defer session.Close()
//
//c := session.DB(config.Config.Mongo.DBDatabase).C(cGroup)
//
//n, err := c.Find(bson.M{"groupid": groupID}).Count()
//if err != nil {
// return err
//}
//
//if n == 0 {
// groupInfo := GroupMember_x{}
// groupInfo.GroupID = groupID
// groupInfo.UIDList = append(groupInfo.UIDList, uid)
// err = c.Insert(&groupInfo)
// if err != nil {
// return err
// }
//} else {
// err = c.Update(bson.M{"groupid": groupID}, bson.M{"$addToSet": bson.M{"uidlist": uid}})
// if err != nil {
// return err
// }
//}
//
//return nil
}
func (d *DataBases) DelGroupMember(groupID, uid string) error {
session := d.mgoSession.Clone()
if session == nil {
return errors.New("session == nil")
}
defer session.Close()
c := session.DB(config.Config.Mongo.DBDatabase).C(cGroup)
if err := c.Update(bson.M{"groupid": groupID}, bson.M{"$pull": bson.M{"uidlist": uid}}); err != nil {
return err
}
return nil
//session := d.mgoSession.Clone()
//if session == nil {
// return errors.New("session == nil")
//}
//defer session.Close()
//
//c := session.DB(config.Config.Mongo.DBDatabase).C(cGroup)
//
//if err := c.Update(bson.M{"groupid": groupID}, bson.M{"$pull": bson.M{"uidlist": uid}}); err != nil {
// return err
//}
//
//return nil
}
func getCurrentTimestampByMill() int64 {