blackteay/open-im-server
1
0
Fork 0
mirror of https://github.com/openimsdk/open-im-server.git synced 2025-04-05 20:11:14 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: v2 to v3 data conversion (#1176)

Browse Source 
* feat: v2 to v3 data conversion

* feat: v2 to v3 data conversion
This commit is contained in:
withchao 2023-10-11 00:13:44 +08:00 committed by GitHub
parent b72b72f74f
commit 8e6ee2b80f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
26 changed files with 11923 additions and 529 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
tools/data-conversion/chat/chat.go Normal file
View File

@ -0,0 +1,61 @@
package main
import (
"fmt"
"github.com/openimsdk/open-im-server/v3/tools/data-conversion/chat/conversion"
"github.com/openimsdk/open-im-server/v3/tools/data-conversion/utils"
"gorm.io/driver/mysql"
"gorm.io/gorm"
"gorm.io/gorm/logger"
"log"
)
func main() {
var (
usernameV2 = "root" // v2版本mysql用户名
passwordV2 = "openIM" // v2版本mysql密码
addrV2 = "127.0.0.1:13306" // v2版本mysql地址
databaseV2 = "admin_chat" // v2版本mysql数据库名字
)
var (
usernameV3 = "root" // v3版本mysql用户名
passwordV3 = "openIM123" // v3版本mysql密码
addrV3 = "127.0.0.1:13306" // v3版本mysql地址
databaseV3 = "openim_enterprise" // v3版本mysql数据库名字
)
var concurrency = 1 // 并发数量
log.SetFlags(log.LstdFlags | log.Llongfile)
dsnV2 := fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8mb4&parseTime=True&loc=Local", usernameV2, passwordV2, addrV2, databaseV2)
dsnV3 := fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8mb4&parseTime=True&loc=Local", usernameV3, passwordV3, addrV3, databaseV3)
dbV2, err := gorm.Open(mysql.Open(dsnV2), &gorm.Config{Logger: logger.Discard})
if err != nil {
log.Println("open v2 db failed", err)
return
}
dbV3, err := gorm.Open(mysql.Open(dsnV3), &gorm.Config{Logger: logger.Discard})
if err != nil {
log.Println("open v3 db failed", err)
return
}
var tasks utils.TakeList
tasks.Append(func() (string, error) { return utils.FindAndInsert(dbV2, dbV3, conversion.Account) })
tasks.Append(func() (string, error) { return utils.FindAndInsert(dbV2, dbV3, conversion.Attribute) })
tasks.Append(func() (string, error) { return utils.FindAndInsert(dbV2, dbV3, conversion.Register) })
tasks.Append(func() (string, error) { return utils.FindAndInsert(dbV2, dbV3, conversion.UserLoginRecord) })
tasks.Append(func() (string, error) { return utils.FindAndInsert(dbV2, dbV3, conversion.Admin) })
tasks.Append(func() (string, error) { return utils.FindAndInsert(dbV2, dbV3, conversion.Applet) })
tasks.Append(func() (string, error) { return utils.FindAndInsert(dbV2, dbV3, conversion.ForbiddenAccount) })
tasks.Append(func() (string, error) { return utils.FindAndInsert(dbV2, dbV3, conversion.InvitationRegister) })
tasks.Append(func() (string, error) { return utils.FindAndInsert(dbV2, dbV3, conversion.IPForbidden) })
tasks.Append(func() (string, error) { return utils.FindAndInsert(dbV2, dbV3, conversion.LimitUserLoginIP) })
tasks.Append(func() (string, error) { return utils.FindAndInsert(dbV2, dbV3, conversion.RegisterAddFriend) })
tasks.Append(func() (string, error) { return utils.FindAndInsert(dbV2, dbV3, conversion.RegisterAddGroup) })
utils.RunTask(concurrency, tasks)
}

32
tools/data-conversion/chat/conversion/cmd.go
View File

@ -1,32 +0,0 @@
package conversion
import (
"fmt"
"gorm.io/gorm"
"gorm.io/gorm/schema"
)
func FindAndInsert[V2 any, V3 schema.Tabler](v2db *gorm.DB, v3db *gorm.DB, fn func(V2) V3) (string, error) {
var t V3
name := t.TableName()
if err := v3db.AutoMigrate(&t); err != nil {
return name, fmt.Errorf("auto migrate v3 %s failed %w", name, err)
}
const size = 100
for i := 0; ; i++ {
var v2s []V2
if err := v2db.Offset(i * size).Limit(size).Find(&v2s).Error; err != nil {
return name, fmt.Errorf("find v2 %s failed %w", name, err)
}
if len(v2s) == 0 {
return name, nil
}
v3s := make([]V3, 0, len(v2s))
for _, v := range v2s {
v3s = append(v3s, fn(v))
}
if err := v3db.Create(&v3s).Error; err != nil {
return name, fmt.Errorf("insert v3 %s failed %w", name, err)
}
}
}

61
tools/data-conversion/chat/conversion/conversion.go
View File

@ -4,21 +4,24 @@ import (
v2 "github.com/openimsdk/open-im-server/v3/tools/data-conversion/chat/v2"
"github.com/openimsdk/open-im-server/v3/tools/data-conversion/chat/v3/admin"
"github.com/openimsdk/open-im-server/v3/tools/data-conversion/chat/v3/chat"
"github.com/openimsdk/open-im-server/v3/tools/data-conversion/utils"
)
// ########## chat ##########
func Account(v v2.Account) chat.Account {
func Account(v v2.Account) (chat.Account, bool) {
utils.InitTime(&v.CreateTime, &v.ChangeTime)
return chat.Account{
UserID: v.UserID,
Password: v.Password,
CreateTime: v.CreateTime,
ChangeTime: v.ChangeTime,
OperatorUserID: v.OperatorUserID,
}
}, true
}
func Attribute(v v2.Attribute) chat.Attribute {
func Attribute(v v2.Attribute) (chat.Attribute, bool) {
utils.InitTime(&v.CreateTime, &v.ChangeTime, &v.BirthTime)
return chat.Attribute{
UserID: v.UserID,
Account: v.Account,
@ -36,10 +39,11 @@ func Attribute(v v2.Attribute) chat.Attribute {
AllowBeep: v.AllowBeep,
AllowAddFriend: v.AllowAddFriend,
GlobalRecvMsgOpt: 0,
}
}, true
}
func Register(v v2.Register) chat.Register {
func Register(v v2.Register) (chat.Register, bool) {
utils.InitTime(&v.CreateTime)
return chat.Register{
UserID: v.UserID,
DeviceID: v.DeviceID,
@ -48,22 +52,24 @@ func Register(v v2.Register) chat.Register {
AccountType: v.AccountType,
Mode: v.Mode,
CreateTime: v.CreateTime,
}
}, true
}
func UserLoginRecord(v v2.UserLoginRecord) chat.UserLoginRecord {
func UserLoginRecord(v v2.UserLoginRecord) (chat.UserLoginRecord, bool) {
utils.InitTime(&v.LoginTime)
return chat.UserLoginRecord{
UserID: v.UserID,
LoginTime: v.LoginTime,
IP: v.IP,
DeviceID: v.DeviceID,
Platform: v.Platform,
}
}, true
}
// ########## admin ##########
func Admin(v v2.Admin) admin.Admin {
func Admin(v v2.Admin) (admin.Admin, bool) {
utils.InitTime(&v.CreateTime)
return admin.Admin{
Account: v.Account,
Password: v.Password,
@ -72,10 +78,11 @@ func Admin(v v2.Admin) admin.Admin {
UserID: v.UserID,
Level: v.Level,
CreateTime: v.CreateTime,
}
}, true
}
func Applet(v v2.Applet) admin.Applet {
func Applet(v v2.Applet) (admin.Applet, bool) {
utils.InitTime(&v.CreateTime)
return admin.Applet{
ID: v.ID,
Name: v.Name,
@ -88,53 +95,59 @@ func Applet(v v2.Applet) admin.Applet {
Priority: v.Priority,
Status: v.Status,
CreateTime: v.CreateTime,
}
}, true
}
func ForbiddenAccount(v v2.ForbiddenAccount) admin.ForbiddenAccount {
func ForbiddenAccount(v v2.ForbiddenAccount) (admin.ForbiddenAccount, bool) {
utils.InitTime(&v.CreateTime)
return admin.ForbiddenAccount{
UserID: v.UserID,
Reason: v.Reason,
OperatorUserID: v.OperatorUserID,
CreateTime: v.CreateTime,
}
}, true
}
func InvitationRegister(v v2.InvitationRegister) admin.InvitationRegister {
func InvitationRegister(v v2.InvitationRegister) (admin.InvitationRegister, bool) {
utils.InitTime(&v.CreateTime)
return admin.InvitationRegister{
InvitationCode: v.InvitationCode,
UsedByUserID: v.UsedByUserID,
CreateTime: v.CreateTime,
}
}, true
}
func IPForbidden(v v2.IPForbidden) admin.IPForbidden {
func IPForbidden(v v2.IPForbidden) (admin.IPForbidden, bool) {
utils.InitTime(&v.CreateTime)
return admin.IPForbidden{
IP: v.IP,
LimitRegister: v.LimitRegister > 0,
LimitLogin: v.LimitLogin > 0,
CreateTime: v.CreateTime,
}
}, true
}
func LimitUserLoginIP(v v2.LimitUserLoginIP) admin.LimitUserLoginIP {
func LimitUserLoginIP(v v2.LimitUserLoginIP) (admin.LimitUserLoginIP, bool) {
utils.InitTime(&v.CreateTime)
return admin.LimitUserLoginIP{
UserID: v.UserID,
IP: v.IP,
CreateTime: v.CreateTime,
}
}, true
}
func RegisterAddFriend(v v2.RegisterAddFriend) admin.RegisterAddFriend {
func RegisterAddFriend(v v2.RegisterAddFriend) (admin.RegisterAddFriend, bool) {
utils.InitTime(&v.CreateTime)
return admin.RegisterAddFriend{
UserID: v.UserID,
CreateTime: v.CreateTime,
}
}, true
}
func RegisterAddGroup(v v2.RegisterAddGroup) admin.RegisterAddGroup {
func RegisterAddGroup(v v2.RegisterAddGroup) (admin.RegisterAddGroup, bool) {
utils.InitTime(&v.CreateTime)
return admin.RegisterAddGroup{
GroupID: v.GroupID,
CreateTime: v.CreateTime,
}
}, true
}

67
tools/data-conversion/chat/main.go
View File

@ -1,67 +0,0 @@
package main
import (
"fmt"
"github.com/openimsdk/open-im-server/v3/tools/data-conversion/chat/conversion"
"gorm.io/driver/mysql"
"gorm.io/gorm"
"log"
)
func main() {
var (
usernameV2 = "root"
passwordV2 = "openIM123"
addrV2 = "127.0.0.1:13306"
databaseV2 = "admin_chat"
)
var (
usernameV3 = "root"
passwordV3 = "openIM123"
addrV3 = "127.0.0.1:13306"
databaseV3 = "openim_enterprise"
)
dsnV2 := fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8mb4&parseTime=True&loc=Local", usernameV2, passwordV2, addrV2, databaseV2)
dsnV3 := fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8mb4&parseTime=True&loc=Local", usernameV3, passwordV3, addrV3, databaseV3)
dbV2, err := gorm.Open(mysql.Open(dsnV2), &gorm.Config{})
if err != nil {
log.Println("open v2 db failed", err)
return
}
dbV3, err := gorm.Open(mysql.Open(dsnV3), &gorm.Config{})
if err != nil {
log.Println("open v3 db failed", err)
return
}
var fns []func() (string, error)
Append := func(fn func() (string, error)) {
fns = append(fns, fn)
}
Append(func() (string, error) { return conversion.FindAndInsert(dbV2, dbV3, conversion.Account) })
Append(func() (string, error) { return conversion.FindAndInsert(dbV2, dbV3, conversion.Attribute) })
Append(func() (string, error) { return conversion.FindAndInsert(dbV2, dbV3, conversion.Register) })
Append(func() (string, error) { return conversion.FindAndInsert(dbV2, dbV3, conversion.UserLoginRecord) })
Append(func() (string, error) { return conversion.FindAndInsert(dbV2, dbV3, conversion.Admin) })
Append(func() (string, error) { return conversion.FindAndInsert(dbV2, dbV3, conversion.Applet) })
Append(func() (string, error) { return conversion.FindAndInsert(dbV2, dbV3, conversion.ForbiddenAccount) })
Append(func() (string, error) { return conversion.FindAndInsert(dbV2, dbV3, conversion.InvitationRegister) })
Append(func() (string, error) { return conversion.FindAndInsert(dbV2, dbV3, conversion.IPForbidden) })
Append(func() (string, error) { return conversion.FindAndInsert(dbV2, dbV3, conversion.LimitUserLoginIP) })
Append(func() (string, error) { return conversion.FindAndInsert(dbV2, dbV3, conversion.RegisterAddFriend) })
Append(func() (string, error) { return conversion.FindAndInsert(dbV2, dbV3, conversion.RegisterAddGroup) })
for i := range fns {
name, err := fns[i]()
if err == nil {
log.Printf("[%d/%d] %s success\n", i+1, len(fns), name)
} else {
log.Printf("[%d/%d] %s failed %s\n", i+1, len(fns), name, err)
return
}
}
}

30
tools/data-conversion/go.mod
View File

@ -4,21 +4,33 @@ go 1.18
require (
github.com/IBM/sarama v1.41.2
github.com/OpenIMSDK/protocol v0.0.23
github.com/OpenIMSDK/tools v0.0.14
github.com/golang/protobuf v1.5.3
github.com/openimsdk/open-im-server/v3 v3.3.2
golang.org/x/net v0.15.0
google.golang.org/grpc v1.57.0
google.golang.org/protobuf v1.31.0
gorm.io/driver/mysql v1.5.1
gorm.io/gorm v1.25.4
)
require (
github.com/OpenIMSDK/protocol v0.0.23 // indirect
github.com/bwmarrin/snowflake v0.3.0 // indirect
github.com/bytedance/sonic v1.9.1 // indirect
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/eapache/go-resiliency v1.4.0 // indirect
github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect
github.com/eapache/queue v1.1.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.2 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
github.com/gin-gonic/gin v1.9.1 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-playground/validator/v10 v10.15.3 // indirect
github.com/go-sql-driver/mysql v1.7.1 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/goccy/go-json v0.10.2 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/hashicorp/errwrap v1.0.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
@ -31,22 +43,30 @@ require (
github.com/jinzhu/copier v0.4.0 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.16.7 // indirect
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
github.com/leodido/go-urn v1.2.4 // indirect
github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect
github.com/lestrrat-go/strftime v1.0.6 // indirect
github.com/mattn/go-isatty v0.0.19 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 // indirect
github.com/pelletier/go-toml/v2 v2.0.8 // indirect
github.com/pierrec/lz4/v4 v4.1.18 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/ugorji/go/codec v1.2.11 // indirect
go.uber.org/atomic v1.7.0 // indirect
go.uber.org/multierr v1.6.0 // indirect
go.uber.org/zap v1.24.0 // indirect
golang.org/x/arch v0.3.0 // indirect
golang.org/x/crypto v0.13.0 // indirect
golang.org/x/image v0.12.0 // indirect
golang.org/x/net v0.15.0 // indirect
golang.org/x/sys v0.12.0 // indirect
golang.org/x/text v0.13.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230807174057-1744710a1577 // indirect
google.golang.org/grpc v1.57.0 // indirect
google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

54
tools/data-conversion/go.sum
View File

@ -7,6 +7,12 @@ github.com/OpenIMSDK/tools v0.0.14/go.mod h1:eg+q4A34Qmu73xkY0mt37FHGMCMfC6CtmOn
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
github.com/bwmarrin/snowflake v0.3.0 h1:xm67bEhkKh6ij1790JB83OujPR5CzNe8QuQqAgISZN0=
github.com/bwmarrin/snowflake v0.3.0/go.mod h1:NdZxfVWX+oR6y2K0o6qAYv6gIOP9rjG0/E9WsDpxqwE=
github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=
github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s=
github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U=
github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY=
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams=
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@ -17,9 +23,24 @@ github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3/go.mod h1
github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg=
github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU=
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
github.com/go-playground/validator/v10 v10.15.3 h1:S+sSpunYjNPDuXkWbK+x+bA7iXiW296KG4dL3X7xUZo=
github.com/go-playground/validator/v10 v10.15.3/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI=
github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
@ -27,6 +48,7 @@ github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
@ -55,18 +77,36 @@ github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkr
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg=
github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc h1:RKf14vYWi2ttpEmkA4aQ3j4u9dStX2t4M8UM6qqNsG8=
github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc/go.mod h1:kopuH9ugFRkIXf3YoqHKyrJ9YfUFsckUU9S7B+XP+is=
github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible h1:Y6sqxHMyB1D2YSzWkLibYKgg+SwmyFU9dF2hn6MdTj4=
github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible/go.mod h1:ZQnN8lSECaebrkQytbHj4xNgtg8CR7RYXnPok8e0EHA=
github.com/lestrrat-go/strftime v1.0.6 h1:CFGsDEt1pOpFNU+TJB0nhz9jl+K0hZSLE205AhTIGQQ=
github.com/lestrrat-go/strftime v1.0.6/go.mod h1:f7jQKgV5nnJpYgdEasS+/y7EsTb8ykN2z68n3TtcTaw=
github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
github.com/openimsdk/open-im-server/v3 v3.3.2 h1:uK6glaidrnWlYXFSwzOEq7fXS6jT1OyesUJENZJeptI=
github.com/openimsdk/open-im-server/v3 v3.3.2/go.mod h1:rqKiCkjav5P7tQmyqaixnMJcayWlM4XtXmwG+cZNw78=
github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ=
github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4=
github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ=
github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@ -75,15 +115,23 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM=
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU=
github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
@ -92,6 +140,9 @@ go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=
go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg=
golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k=
golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
@ -119,6 +170,7 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@ -145,6 +197,7 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ
google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
@ -154,3 +207,4 @@ gorm.io/driver/mysql v1.5.1/go.mod h1:Jo3Xu7mMhCyj8dlrb3WoCaRd1FhsVh+yMXb1jUInf5
gorm.io/gorm v1.25.1/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k=
gorm.io/gorm v1.25.4 h1:iyNd8fNAe8W9dvtlgeRI5zSVZPsq3OpcTu37cYcpCmw=
gorm.io/gorm v1.25.4/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k=
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=

68
tools/data-conversion/info.md Normal file
View File

@ -0,0 +1,68 @@
# v2数据迁移工具
### <font color=red>转换前请做好数据备份！！！</font>
### 转换OPENIM MYSQL数据
- open-im-server/v3/tools/data-conversion/openim/mysql.go
- 配置mysql.go数据库信息
- 需要手动创建v3版本数据库,字符集`utf8mb4`
```go
var (
usernameV2 = "root" // v2版本mysql用户名
passwordV2 = "openIM" // v2版本mysql密码
addrV2 = "127.0.0.1:13306" // v2版本mysql地址
databaseV2 = "openIM_v2" // v2版本mysql数据库名字
)
var (
usernameV3 = "root" // v3版本mysql用户名
passwordV3 = "openIM123" // v3版本mysql密码
addrV3 = "127.0.0.1:13306" // v3版本mysql地址
databaseV3 = "openIM_v3" // v3版本mysql数据库名字
)
```
```shell
go run mysql.go
```
### 转换聊天消息(可选)
- 目前只支持转换kafka中的消息
- open-im-server/v3/tools/data-conversion/openim/msg.go
- 配置msg.go数据库信息
```go
var (
topic = "ws2ms_chat" // v2版本配置文件kafka.topic.ws2ms_chat
kafkaAddr = "127.0.0.1:9092" // v2版本配置文件kafka.topic.addr
rpcAddr = "127.0.0.1:10130" // v3版本配置文件rpcPort.openImMessagePort
adminUserID = "openIM123456" // v3版本管理员userID
concurrency = 4 // 并发数量
)
```
```shell
go run msg.go
```
### 转换业务服务器数据(使用官方业务服务器需要转换)
- 目前只支持转换kafka中的消息
- open-im-server/v3/tools/data-conversion/chat/chat.go
- 需要手动创建v3版本数据库,字符集`utf8mb4`
- main.go数据库信息
```go
var (
usernameV2 = "root" // v2版本mysql用户名
passwordV2 = "openIM" // v2版本mysql密码
addrV2 = "127.0.0.1:13306" // v2版本mysql地址
databaseV2 = "admin_chat" // v2版本mysql数据库名字
)
var (
usernameV3 = "root" // v3版本mysql用户名
passwordV3 = "openIM123" // v3版本mysql密码
addrV3 = "127.0.0.1:13306" // v3版本mysql地址
databaseV3 = "openim_enterprise" // v3版本mysql数据库名字
)
```
```shell
go run chat.go
```

88
tools/data-conversion/kafka-conversation.go
View File

@ -1,88 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package data_conversion
import (
"fmt"
"github.com/IBM/sarama"
)
var (
topic = "latestMsgToRedis"
addr = "127.0.0.1:9092"
)
var (
consumer sarama.Consumer
producer sarama.SyncProducer
)
func init() {
// Producer
config := sarama.NewConfig() // Instantiate a sarama Config
config.Producer.Return.Successes = true // Whether to enable the successes channel to be notified after the message is sent successfully
config.Producer.Return.Errors = true
config.Producer.RequiredAcks = sarama.WaitForAll // Set producer Message Reply level 0 1 all
config.Producer.Partitioner = sarama.NewHashPartitioner // Set the hash-key automatic hash partition. When sending a message, you must specify the key value of the message. If there is no key, the partition will be selected randomly
client, err := sarama.NewSyncProducer([]string{addr}, config)
if err != nil {
fmt.Println("producer closed, err:", err)
}
producer = client
// Consumer
consumerT, err := sarama.NewConsumer([]string{addr}, sarama.NewConfig())
if err != nil {
fmt.Printf("fail to start consumer, err:%v\n", err)
}
consumer = consumerT
}
func SendMessage() {
// construct a message
msg := &sarama.ProducerMessage{}
msg.Topic = topic
msg.Value = sarama.StringEncoder("this is a test log")
// Send a message
pid, offset, err := producer.SendMessage(msg)
if err != nil {
fmt.Println("send msg failed, err:", err)
}
fmt.Printf("pid:%v offset:%v\n", pid, offset)
}
func GetMessage() {
partitionList, err := consumer.Partitions(topic) // Get all partitions according to topic
if err != nil {
fmt.Printf("fail to get list of partition:err%v\n", err)
}
fmt.Println(partitionList)
for partition := range partitionList { // iterate over all partitions
// Create a corresponding partition consumer for each partition
pc, err := consumer.ConsumePartition(topic, int32(partition), sarama.OffsetNewest)
if err != nil {
fmt.Printf("failed to start consumer for partition %d,err:%v\n", partition, err)
}
// Asynchronously consume information from each partition
go func(sarama.PartitionConsumer) {
for msg := range pc.Messages() {
fmt.Printf("Partition:%d Offset:%d Key:%v Value:%v", msg.Partition, msg.Offset, msg.Key, msg.Value)
}
}(pc)
}
}

15
tools/data-conversion/mongodb-conversion.go
View File

@ -1,15 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package data_conversion

228
tools/data-conversion/mysql-conversion.go
View File

@ -1,228 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package data_conversion
import (
"context"
"fmt"
"time"
"github.com/OpenIMSDK/tools/log"
"gorm.io/driver/mysql"
"gorm.io/gorm"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
)
var (
MysqlDb_v2 *gorm.DB
MysqlDb_v3 *gorm.DB
)
const (
username_v2 = "root"
password_v2 = "123456"
ip_v2 = "127.0.0.1:3306"
database_v2 = "openim_v2"
)
const (
username_v3 = "root"
password_v3 = "123456"
ip_v3 = "127.0.0.1:3306"
database_v3 = "openim_v3"
)
func init() {
dsn := fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8mb4&parseTime=True&loc=Local",
username_v2,
password_v2,
ip_v2,
database_v2,
)
db, err := gorm.Open(mysql.Open(dsn), &gorm.Config{})
MysqlDb_v2 = db
if err != nil {
log.ZDebug(context.Background(), "err", err)
}
dsn_v3 := fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8mb4&parseTime=True&loc=Local",
username_v3,
password_v3,
ip_v3,
database_v3,
)
db_v3, err := gorm.Open(mysql.Open(dsn_v3), &gorm.Config{})
MysqlDb_v3 = db_v3
if err != nil {
log.ZDebug(context.Background(), "err", err)
}
}
func UserConversion() {
var count int64
var user relation.UserModel
MysqlDb_v2.Model(&user).Count(&count)
batchSize := 100
offset := 0
for int64(offset) < count {
var results []relation.UserModel
MysqlDb_v2.Limit(batchSize).Offset(offset).Find(&results)
// Process query results
fmt.Println("============================batch data===================", offset, batchSize)
fmt.Println(results)
MysqlDb_v3.Create(results)
fmt.Println("======================================================")
offset += batchSize
}
}
func FriendConversion() {
var count int64
var friend relation.FriendModel
MysqlDb_v2.Model(&friend).Count(&count)
batchSize := 100
offset := 0
for int64(offset) < count {
var results []relation.FriendModel
MysqlDb_v2.Limit(batchSize).Offset(offset).Find(&results)
// Process query results
fmt.Println("============================batch data===================", offset, batchSize)
fmt.Println(results)
MysqlDb_v3.Create(results)
fmt.Println("======================================================")
offset += batchSize
}
}
func RequestConversion() {
var count int64
var friendRequest relation.FriendRequestModel
MysqlDb_v2.Model(&friendRequest).Count(&count)
batchSize := 100
offset := 0
for int64(offset) < count {
var results []relation.FriendRequestModel
MysqlDb_v2.Limit(batchSize).Offset(offset).Find(&results)
// Process query results
fmt.Println("============================batch data===================", offset, batchSize)
fmt.Println(results)
MysqlDb_v3.Create(results)
fmt.Println("======================================================")
offset += batchSize
}
var groupRequest relation.GroupRequestModel
MysqlDb_v2.Model(&groupRequest).Count(&count)
batchSize = 100
offset = 0
for int64(offset) < count {
var results []relation.GroupRequestModel
MysqlDb_v2.Limit(batchSize).Offset(offset).Find(&results)
// Process query results
fmt.Println("============================batch data===================", offset, batchSize)
fmt.Println(results)
MysqlDb_v3.Create(results)
fmt.Println("======================================================")
offset += batchSize
}
}
func GroupConversion() {
var count int64
var group relation.GroupModel
MysqlDb_v2.Model(&group).Count(&count)
batchSize := 100
offset := 0
for int64(offset) < count {
var results []relation.GroupModel
MysqlDb_v2.Limit(batchSize).Offset(offset).Find(&results)
for i, val := range results {
temp := time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)
if val.NotificationUpdateTime.Equal(temp) {
results[i].NotificationUpdateTime = time.Now()
// fmt.Println(val.NotificationUpdateTime)
}
}
// Process query results
fmt.Println("============================batch data===================", offset, batchSize)
fmt.Println(results)
MysqlDb_v3.Create(results)
fmt.Println("======================================================")
offset += batchSize
}
}
func GroupMemberConversion() {
var count int64
var groupMember relation.GroupMemberModel
MysqlDb_v2.Model(&groupMember).Count(&count)
batchSize := 100
offset := 0
for int64(offset) < count {
var results []relation.GroupMemberModel
MysqlDb_v2.Limit(batchSize).Offset(offset).Find(&results)
// Process query results
fmt.Println("============================batch data===================", offset, batchSize)
fmt.Println(results)
MysqlDb_v3.Create(results)
fmt.Println("======================================================")
offset += batchSize
}
}
func BlacksConversion() {
var count int64
var black relation.BlackModel
MysqlDb_v2.Model(&black).Count(&count)
batchSize := 100
offset := 0
for int64(offset) < count {
var results []relation.BlackModel
MysqlDb_v2.Limit(batchSize).Offset(offset).Find(&results)
// Process query results
fmt.Println("============================batch data===================", offset, batchSize)
fmt.Println(results)
MysqlDb_v3.Create(results)
fmt.Println("======================================================")
offset += batchSize
}
}
func ChatLogsConversion() {
var count int64
var chat relation.ChatLogModel
MysqlDb_v2.Model(&chat).Count(&count)
batchSize := 100
offset := 0
for int64(offset) < count {
var results []relation.ChatLogModel
MysqlDb_v2.Limit(batchSize).Offset(offset).Find(&results)
// Process query results
fmt.Println("============================batch data===================", offset, batchSize)
// fmt.Println(results)
MysqlDb_v3.Create(results)
fmt.Println("======================================================")
offset += batchSize
}
}

49
tools/data-conversion/mysql_test.go
View File

@ -1,49 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package data_conversion
import "testing"
// pass
func TestUserConversion(t *testing.T) {
UserConversion()
}
// pass
func TestFriendConversion(t *testing.T) {
FriendConversion()
}
// pass
func TestGroupConversion(t *testing.T) {
GroupConversion()
GroupMemberConversion()
}
// pass
func TestBlacksConversion(t *testing.T) {
BlacksConversion()
}
// pass
func TestRequestConversion(t *testing.T) {
RequestConversion()
}
// pass
func TestChatLogsConversion(t *testing.T) {
// If the printed result is too long, the console will not display it, but it can run normally
ChatLogsConversion()
}

65
tools/data-conversion/openim/common/config.go Normal file
View File

@ -0,0 +1,65 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package common
// =================================== V2 =====================================
// MySQL
// V2
const (
UsernameV2 = "root"
PasswordV2 = "openIM"
IpV2 = "121.5.182.23:13306"
DatabaseV2 = "openIM_v2"
)
// V2 chat
const (
ChatUsernameV2 = "root"
ChatPasswordV2 = "openIM"
ChatIpV2 = "121.5.182.23:13306"
ChatDatabaseV2 = "admin_chat"
)
// Kafka
const (
Topic = "ws2ms_chat"
KafkaAddr = "121.5.182.23:9092"
)
// =================================== V3 =====================================
// V3
const (
UsernameV3 = "root"
PasswordV3 = "openIM123"
IpV3 = "43.134.63.160:13306"
DatabaseV3 = "openIM_v3"
)
// V3 chat
const (
ChatUsernameV3 = "root"
ChatPasswordV3 = "openIM123"
ChatIpV3 = "43.134.63.160:13306"
ChatDatabaseV3 = "openim_enterprise"
)
// Zookeeper
const (
ZkAddr = "43.134.63.160:12181"
ZKSchema = "openim"
ZKUsername = ""
ZKPassword = ""
MsgRpcName = "Msg"
)

16
tools/data-conversion/kafka_test.go → tools/data-conversion/openim/common/console.go
View File

@ -12,14 +12,18 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package data_conversion
package common
import "testing"
import "fmt"
func TestGetMessage(t *testing.T) {
GetMessage()
func ErrorPrint(s string) {
fmt.Printf("\x1b[%dm%v\x1b[0m\n", 31, s)
}
func TestSendMessage(t *testing.T) {
SendMessage()
func SuccessPrint(s string) {
fmt.Printf("\x1b[%dm%v\x1b[0m\n", 32, s)
}
func WarningPrint(s string) {
fmt.Printf("\x1b[%dmWarning: But %v\x1b[0m\n", 33, s)
}

202
tools/data-conversion/openim/msg.go Normal file
View File

@ -0,0 +1,202 @@
package main
import (
"context"
"encoding/json"
"github.com/IBM/sarama"
"github.com/OpenIMSDK/protocol/constant"
"github.com/OpenIMSDK/protocol/msg"
"github.com/OpenIMSDK/protocol/sdkws"
"github.com/OpenIMSDK/tools/mw"
"github.com/golang/protobuf/proto"
"github.com/openimsdk/open-im-server/v3/pkg/apistruct"
pbmsg "github.com/openimsdk/open-im-server/v3/tools/data-conversion/openim/proto/msg"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
"log"
"sync"
"sync/atomic"
"time"
)
func main() {
var (
topic = "ws2ms_chat" // v2版本配置文件kafka.topic.ws2ms_chat
kafkaAddr = "127.0.0.1:9092" // v2版本配置文件kafka.topic.addr
rpcAddr = "127.0.0.1:10130" // v3版本配置文件rpcPort.openImMessagePort
adminUserID = "openIM123456" // v3版本管理员userID
concurrency = 1 // 并发数量
)
getRpcConn := func() (*grpc.ClientConn, error) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
defer cancel()
return grpc.DialContext(ctx, rpcAddr, grpc.WithTransportCredentials(insecure.NewCredentials()), mw.GrpcClient())
}
conn, err := getRpcConn()
if err != nil {
log.Println("get rpc conn", err)
return
}
defer conn.Close()
msgClient := msg.NewMsgClient(conn)
conf := sarama.NewConfig()
conf.Consumer.Offsets.Initial = sarama.OffsetOldest
consumer, err := sarama.NewConsumer([]string{kafkaAddr}, conf)
if err != nil {
log.Println("kafka consumer conn", err)
return
}
partitions, err := consumer.Partitions(topic) // Get all partitions according to topic
if err != nil {
log.Println("kafka partitions", err)
return
}
if len(partitions) == 0 {
log.Println("kafka partitions is empty")
return
}
log.Println("kafka partitions", partitions)
msgCh := make(chan *pbmsg.MsgDataToMQ, concurrency*2)
var kfkWg sync.WaitGroup
distinct := make(map[string]struct{})
var lock sync.Mutex
for _, partition := range partitions {
kfkWg.Add(1)
go func(partition int32) {
defer kfkWg.Done()
pc, err := consumer.ConsumePartition(topic, partition, sarama.OffsetOldest)
if err != nil {
log.Printf("kafka Consume Partition %d failed %s\n", partition, err)
return
}
defer pc.Close()
ch := pc.Messages()
for {
select {
case <-time.After(time.Second * 10): // 10s读取不到就关闭
return
case message, ok := <-ch:
if !ok {
return
}
msgFromMQV2 := pbmsg.MsgDataToMQ{}
err := proto.Unmarshal(message.Value, &msgFromMQV2)
if err != nil {
log.Printf("kafka msg partition %d offset %d unmarshal failed %s\n", message.Partition, message.Offset, message.Value)
continue
}
if msgFromMQV2.MsgData == nil || msgFromMQV2.OperationID == "" {
continue
}
if msgFromMQV2.MsgData.ContentType < constant.ContentTypeBegin || msgFromMQV2.MsgData.ContentType > constant.AdvancedText {
continue
}
lock.Lock()
_, exist := distinct[msgFromMQV2.MsgData.ClientMsgID]
if !exist {
distinct[msgFromMQV2.MsgData.ClientMsgID] = struct{}{}
}
lock.Unlock()
if exist {
continue
}
msgCh <- &msgFromMQV2
}
}
}(partition)
}
go func() {
kfkWg.Wait()
close(msgCh)
}()
var msgWg sync.WaitGroup
var (
success int64
failed int64
)
for i := 0; i < concurrency; i++ {
msgWg.Add(1)
go func() {
defer msgWg.Done()
for message := range msgCh {
HandlerV2Msg(msgClient, adminUserID, message, &success, &failed)
}
}()
}
msgWg.Wait()
log.Printf("total %d success %d failed %d\n", success+failed, success, failed)
}
func HandlerV2Msg(msgClient msg.MsgClient, adminUserID string, msgFromMQV2 *pbmsg.MsgDataToMQ, success *int64, failed *int64) {
msgData := &sdkws.MsgData{
SendID: msgFromMQV2.MsgData.SendID,
RecvID: msgFromMQV2.MsgData.RecvID,
GroupID: msgFromMQV2.MsgData.GroupID,
ClientMsgID: msgFromMQV2.MsgData.ClientMsgID,
ServerMsgID: msgFromMQV2.MsgData.ServerMsgID,
SenderPlatformID: msgFromMQV2.MsgData.SenderPlatformID,
SenderNickname: msgFromMQV2.MsgData.SenderNickname,
SenderFaceURL: msgFromMQV2.MsgData.SenderFaceURL,
SessionType: msgFromMQV2.MsgData.SessionType,
MsgFrom: msgFromMQV2.MsgData.MsgFrom,
ContentType: msgFromMQV2.MsgData.ContentType,
SendTime: msgFromMQV2.MsgData.SendTime,
CreateTime: msgFromMQV2.MsgData.CreateTime,
Status: msgFromMQV2.MsgData.Status,
IsRead: false,
Options: msgFromMQV2.MsgData.Options,
AtUserIDList: msgFromMQV2.MsgData.AtUserIDList,
AttachedInfo: msgFromMQV2.MsgData.AttachedInfo,
Ex: msgFromMQV2.MsgData.Ex,
}
if msgFromMQV2.MsgData.OfflinePushInfo != nil {
msgData.OfflinePushInfo = &sdkws.OfflinePushInfo{
Title: msgFromMQV2.MsgData.OfflinePushInfo.Title,
Desc: msgFromMQV2.MsgData.OfflinePushInfo.Desc,
Ex: msgFromMQV2.MsgData.OfflinePushInfo.Ex,
IOSPushSound: msgFromMQV2.MsgData.OfflinePushInfo.IOSPushSound,
IOSBadgeCount: msgFromMQV2.MsgData.OfflinePushInfo.IOSBadgeCount,
SignalInfo: "",
}
}
switch msgData.ContentType {
case constant.Text:
data, err := json.Marshal(apistruct.TextElem{
Content: string(msgFromMQV2.MsgData.Content),
})
if err != nil {
return
}
msgData.Content = data
default:
msgData.Content = msgFromMQV2.MsgData.Content
}
ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
defer cancel()
ctx = context.WithValue(context.Background(), constant.OperationID, msgFromMQV2.OperationID)
ctx = context.WithValue(ctx, constant.OpUserID, adminUserID)
resp, err := msgClient.SendMsg(ctx, &msg.SendMsgReq{MsgData: msgData})
if err != nil {
atomic.AddInt64(failed, 1)
log.Printf("send msg %+v failed %s\n", msgData, err)
return
}
atomic.AddInt64(success, 1)
log.Printf("send msg success %+v resp %+v\n", msgData, resp)
}

69
tools/data-conversion/openim/mysql.go Normal file
View File

@ -0,0 +1,69 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"fmt"
"github.com/openimsdk/open-im-server/v3/tools/data-conversion/openim/mysql/conversion"
"github.com/openimsdk/open-im-server/v3/tools/data-conversion/utils"
"gorm.io/driver/mysql"
"gorm.io/gorm"
"gorm.io/gorm/logger"
"log"
)
func main() {
var (
usernameV2 = "root" // v2版本mysql用户名
passwordV2 = "openIM" // v2版本mysql密码
addrV2 = "127.0.0.1:13306" // v2版本mysql地址
databaseV2 = "openIM_v2" // v2版本mysql数据库名字
)
var (
usernameV3 = "root" // v3版本mysql用户名
passwordV3 = "openIM123" // v3版本mysql密码
addrV3 = "127.0.0.1:13306" // v3版本mysql地址
databaseV3 = "openIM_v3" // v3版本mysql数据库名字
)
var concurrency = 1 // 并发数量
log.SetFlags(log.LstdFlags | log.Llongfile)
dsnV2 := fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8mb4&parseTime=True&loc=Local", usernameV2, passwordV2, addrV2, databaseV2)
dsnV3 := fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8mb4&parseTime=True&loc=Local", usernameV3, passwordV3, addrV3, databaseV3)
dbV2, err := gorm.Open(mysql.Open(dsnV2), &gorm.Config{Logger: logger.Discard})
if err != nil {
log.Println("open v2 db failed", err)
return
}
dbV3, err := gorm.Open(mysql.Open(dsnV3), &gorm.Config{Logger: logger.Discard})
if err != nil {
log.Println("open v3 db failed", err)
return
}
var tasks utils.TakeList
tasks.Append(func() (string, error) { return utils.FindAndInsert(dbV2, dbV3, conversion.Friend) })
tasks.Append(func() (string, error) { return utils.FindAndInsert(dbV2, dbV3, conversion.FriendRequest) })
tasks.Append(func() (string, error) { return utils.FindAndInsert(dbV2, dbV3, conversion.Group) })
tasks.Append(func() (string, error) { return utils.FindAndInsert(dbV2, dbV3, conversion.GroupMember) })
tasks.Append(func() (string, error) { return utils.FindAndInsert(dbV2, dbV3, conversion.GroupRequest) })
tasks.Append(func() (string, error) { return utils.FindAndInsert(dbV2, dbV3, conversion.User) })
utils.RunTask(concurrency, tasks)
}

52
tools/data-conversion/openim/mysql/cmd.go Normal file
View File

@ -0,0 +1,52 @@
package mysql
import (
"fmt"
"github.com/openimsdk/open-im-server/v3/tools/data-conversion/openim/mysql/conversion"
"github.com/openimsdk/open-im-server/v3/tools/data-conversion/utils"
"gorm.io/driver/mysql"
"gorm.io/gorm"
"gorm.io/gorm/logger"
"log"
)
func Cmd() {
var (
usernameV2 = "root"
passwordV2 = "openIM"
addrV2 = "121.5.182.23:13306"
databaseV2 = "openIM_v2"
)
var (
usernameV3 = "root"
passwordV3 = "openIM123"
addrV3 = "203.56.175.233:13306"
databaseV3 = "openIM_v3"
)
log.SetFlags(log.LstdFlags | log.Llongfile)
dsnV2 := fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8mb4&parseTime=True&loc=Local", usernameV2, passwordV2, addrV2, databaseV2)
dsnV3 := fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8mb4&parseTime=True&loc=Local", usernameV3, passwordV3, addrV3, databaseV3)
dbV2, err := gorm.Open(mysql.Open(dsnV2), &gorm.Config{Logger: logger.Discard})
if err != nil {
log.Println("open v2 db failed", err)
return
}
dbV3, err := gorm.Open(mysql.Open(dsnV3), &gorm.Config{Logger: logger.Discard})
if err != nil {
log.Println("open v3 db failed", err)
return
}
var tasks utils.TakeList
tasks.Append(func() (string, error) { return utils.FindAndInsert(dbV2, dbV3, conversion.Friend) })
tasks.Append(func() (string, error) { return utils.FindAndInsert(dbV2, dbV3, conversion.FriendRequest) })
tasks.Append(func() (string, error) { return utils.FindAndInsert(dbV2, dbV3, conversion.Group) })
tasks.Append(func() (string, error) { return utils.FindAndInsert(dbV2, dbV3, conversion.GroupMember) })
tasks.Append(func() (string, error) { return utils.FindAndInsert(dbV2, dbV3, conversion.GroupRequest) })
tasks.Append(func() (string, error) { return utils.FindAndInsert(dbV2, dbV3, conversion.User) })
utils.RunTask(4, tasks)
}

110
tools/data-conversion/openim/mysql/conversion/conversion.go Normal file
View File

@ -0,0 +1,110 @@
package conversion
import (
"github.com/OpenIMSDK/protocol/constant"
v3 "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
v2 "github.com/openimsdk/open-im-server/v3/tools/data-conversion/openim/mysql/v2"
"github.com/openimsdk/open-im-server/v3/tools/data-conversion/utils"
)
func Friend(v v2.Friend) (v3.FriendModel, bool) {
utils.InitTime(&v.CreateTime)
return v3.FriendModel{
OwnerUserID: v.OwnerUserID,
FriendUserID: v.FriendUserID,
Remark: v.Remark,
CreateTime: v.CreateTime,
AddSource: v.AddSource,
OperatorUserID: v.OperatorUserID,
Ex: v.Ex,
}, true
}
func FriendRequest(v v2.FriendRequest) (v3.FriendRequestModel, bool) {
utils.InitTime(&v.CreateTime, &v.HandleTime)
return v3.FriendRequestModel{
FromUserID: v.FromUserID,
ToUserID: v.ToUserID,
HandleResult: v.HandleResult,
ReqMsg: v.ReqMsg,
CreateTime: v.CreateTime,
HandlerUserID: v.HandlerUserID,
HandleMsg: v.HandleMsg,
HandleTime: v.HandleTime,
Ex: v.Ex,
}, true
}
func Group(v v2.Group) (v3.GroupModel, bool) {
switch v.GroupType {
case constant.WorkingGroup, constant.NormalGroup:
v.GroupType = constant.WorkingGroup
default:
return v3.GroupModel{}, false
}
utils.InitTime(&v.CreateTime, &v.NotificationUpdateTime)
return v3.GroupModel{
GroupID: v.GroupID,
GroupName: v.GroupName,
Notification: v.Notification,
Introduction: v.Introduction,
FaceURL: v.FaceURL,
CreateTime: v.CreateTime,
Ex: v.Ex,
Status: v.Status,
CreatorUserID: v.CreatorUserID,
GroupType: v.GroupType,
NeedVerification: v.NeedVerification,
LookMemberInfo: v.LookMemberInfo,
ApplyMemberFriend: v.ApplyMemberFriend,
NotificationUpdateTime: v.NotificationUpdateTime,
NotificationUserID: v.NotificationUserID,
}, true
}
func GroupMember(v v2.GroupMember) (v3.GroupMemberModel, bool) {
utils.InitTime(&v.JoinTime, &v.MuteEndTime)
return v3.GroupMemberModel{
GroupID: v.GroupID,
UserID: v.UserID,
Nickname: v.Nickname,
FaceURL: v.FaceURL,
RoleLevel: v.RoleLevel,
JoinTime: v.JoinTime,
JoinSource: v.JoinSource,
InviterUserID: v.InviterUserID,
OperatorUserID: v.OperatorUserID,
MuteEndTime: v.MuteEndTime,
Ex: v.Ex,
}, true
}
func GroupRequest(v v2.GroupRequest) (v3.GroupRequestModel, bool) {
utils.InitTime(&v.ReqTime, &v.HandledTime)
return v3.GroupRequestModel{
UserID: v.UserID,
GroupID: v.GroupID,
HandleResult: v.HandleResult,
ReqMsg: v.ReqMsg,
HandledMsg: v.HandledMsg,
ReqTime: v.ReqTime,
HandleUserID: v.HandleUserID,
HandledTime: v.HandledTime,
JoinSource: v.JoinSource,
InviterUserID: v.InviterUserID,
Ex: v.Ex,
}, true
}
func User(v v2.User) (v3.UserModel, bool) {
utils.InitTime(&v.CreateTime)
return v3.UserModel{
UserID: v.UserID,
Nickname: v.Nickname,
FaceURL: v.FaceURL,
Ex: v.Ex,
CreateTime: v.CreateTime,
AppMangerLevel: v.AppMangerLevel,
GlobalRecvMsgOpt: v.GlobalRecvMsgOpt,
}, true
}

91
tools/data-conversion/openim/mysql/v2/model_struct.go Normal file
View File

@ -0,0 +1,91 @@
package db
import "time"
type Friend struct {
OwnerUserID string `gorm:"column:owner_user_id;primary_key;size:64"`
FriendUserID string `gorm:"column:friend_user_id;primary_key;size:64"`
Remark string `gorm:"column:remark;size:255"`
CreateTime time.Time `gorm:"column:create_time"`
AddSource int32 `gorm:"column:add_source"`
OperatorUserID string `gorm:"column:operator_user_id;size:64"`
Ex string `gorm:"column:ex;size:1024"`
}
type FriendRequest struct {
FromUserID string `gorm:"column:from_user_id;primary_key;size:64"`
ToUserID string `gorm:"column:to_user_id;primary_key;size:64"`
HandleResult int32 `gorm:"column:handle_result"`
ReqMsg string `gorm:"column:req_msg;size:255"`
CreateTime time.Time `gorm:"column:create_time"`
HandlerUserID string `gorm:"column:handler_user_id;size:64"`
HandleMsg string `gorm:"column:handle_msg;size:255"`
HandleTime time.Time `gorm:"column:handle_time"`
Ex string `gorm:"column:ex;size:1024"`
}
func (FriendRequest) TableName() string {
return "friend_requests"
}
type Group struct {
GroupID string `gorm:"column:group_id;primary_key;size:64" json:"groupID" binding:"required"`
GroupName string `gorm:"column:name;size:255" json:"groupName"`
Notification string `gorm:"column:notification;size:255" json:"notification"`
Introduction string `gorm:"column:introduction;size:255" json:"introduction"`
FaceURL string `gorm:"column:face_url;size:255" json:"faceURL"`
CreateTime time.Time `gorm:"column:create_time;index:create_time"`
Ex string `gorm:"column:ex" json:"ex;size:1024" json:"ex"`
Status int32 `gorm:"column:status"`
CreatorUserID string `gorm:"column:creator_user_id;size:64"`
GroupType int32 `gorm:"column:group_type"`
NeedVerification int32 `gorm:"column:need_verification"`
LookMemberInfo int32 `gorm:"column:look_member_info" json:"lookMemberInfo"`
ApplyMemberFriend int32 `gorm:"column:apply_member_friend" json:"applyMemberFriend"`
NotificationUpdateTime time.Time `gorm:"column:notification_update_time"`
NotificationUserID string `gorm:"column:notification_user_id;size:64"`
}
type GroupMember struct {
GroupID string `gorm:"column:group_id;primary_key;size:64"`
UserID string `gorm:"column:user_id;primary_key;size:64"`
Nickname string `gorm:"column:nickname;size:255"`
FaceURL string `gorm:"column:user_group_face_url;size:255"`
RoleLevel int32 `gorm:"column:role_level"`
JoinTime time.Time `gorm:"column:join_time"`
JoinSource int32 `gorm:"column:join_source"`
InviterUserID string `gorm:"column:inviter_user_id;size:64"`
OperatorUserID string `gorm:"column:operator_user_id;size:64"`
MuteEndTime time.Time `gorm:"column:mute_end_time"`
Ex string `gorm:"column:ex;size:1024"`
}
type GroupRequest struct {
UserID string `gorm:"column:user_id;primary_key;size:64"`
GroupID string `gorm:"column:group_id;primary_key;size:64"`
HandleResult int32 `gorm:"column:handle_result"`
ReqMsg string `gorm:"column:req_msg;size:1024"`
HandledMsg string `gorm:"column:handle_msg;size:1024"`
ReqTime time.Time `gorm:"column:req_time"`
HandleUserID string `gorm:"column:handle_user_id;size:64"`
HandledTime time.Time `gorm:"column:handle_time"`
JoinSource int32 `gorm:"column:join_source"`
InviterUserID string `gorm:"column:inviter_user_id;size:64"`
Ex string `gorm:"column:ex;size:1024"`
}
type User struct {
UserID string `gorm:"column:user_id;primary_key;size:64"`
Nickname string `gorm:"column:name;size:255"`
FaceURL string `gorm:"column:face_url;size:255"`
Gender int32 `gorm:"column:gender"`
PhoneNumber string `gorm:"column:phone_number;size:32"`
Birth time.Time `gorm:"column:birth"`
Email string `gorm:"column:email;size:64"`
Ex string `gorm:"column:ex;size:1024"`
CreateTime time.Time `gorm:"column:create_time;index:create_time"`
AppMangerLevel int32 `gorm:"column:app_manger_level"`
GlobalRecvMsgOpt int32 `gorm:"column:global_recv_msg_opt"`
status int32 `gorm:"column:status"`
}

3161
tools/data-conversion/openim/proto/msg/msg.pb.go Normal file
View File

File diff suppressed because it is too large Load Diff

315
tools/data-conversion/openim/proto/msg/msg.proto Normal file
View File

@ -0,0 +1,315 @@
syntax = "proto3";
import "Open-IM-Server/pkg/proto/sdk_ws/ws.proto";
import "Open-IM-Server/pkg/proto/sdk_ws/wrappers.proto";
option go_package = "Open_IM/pkg/proto/msg;msg";
package msg;
message MsgDataToMQ{
string token =1;
string operationID = 2;
server_api_params.MsgData msgData = 3;
}
message MsgDataToDB {
server_api_params.MsgData msgData = 1;
string operationID = 2;
}
message PushMsgDataToMQ{
string OperationID = 1;
server_api_params.MsgData msgData = 2;
string pushToUserID = 3;
}
message MsgDataToMongoByMQ{
uint64 lastSeq =1;
string aggregationID = 2;
repeated MsgDataToMQ messageList = 3;
string triggerID = 4;
}
//message PullMessageReq {
// string UserID = 1;
// int64 SeqBegin = 2;
// int64 SeqEnd = 3;
// string OperationID = 4;
//}
//
//message PullMessageResp {
// int32 ErrCode = 1;
// string ErrMsg = 2;
// int64 MaxSeq = 3;
// int64 MinSeq = 4;
// repeated GatherFormat SingleUserMsg = 5;
// repeated GatherFormat GroupUserMsg = 6;
//}
//message PullMessageBySeqListReq{
// string UserID = 1;
// string OperationID = 2;
// repeated int64 seqList =3;
//}
message GetMaxAndMinSeqReq {
string UserID = 1;
string OperationID = 2;
}
message GetMaxAndMinSeqResp {
int32 ErrCode = 1;
string ErrMsg = 2;
uint32 MaxSeq = 3;
uint32 MinSeq = 4;
}
message SendMsgReq {
string token =1;
string operationID = 2;
server_api_params.MsgData msgData = 3;
}
message SendMsgResp {
int32 errCode = 1;
string errMsg = 2;
string serverMsgID = 4;
string clientMsgID = 5;
int64 sendTime = 6;
string ex = 7;
}
message ClearMsgReq{
string userID = 1;
string opUserID = 2;
string operationID = 3;
}
message ClearMsgResp{
int32 errCode = 1;
string errMsg = 2;
}
message SetMsgMinSeqReq{
string userID = 1;
string groupID = 2;
uint32 minSeq = 3;
string operationID = 4;
string opUserID = 5;
}
message SetMsgMinSeqResp{
int32 errCode = 1;
string errMsg = 2;
}
message SetSendMsgStatusReq{
string operationID = 1;
int32 status = 2;
}
message SetSendMsgStatusResp{
int32 errCode = 1;
string errMsg = 2;
}
message GetSendMsgStatusReq{
string operationID = 1;
}
message GetSendMsgStatusResp{
int32 errCode = 1;
string errMsg = 2;
int32 status = 3;
}
message DelSuperGroupMsgReq{
string opUserID = 1;
string userID = 2;
string groupID = 3;
string operationID = 4;
}
message DelSuperGroupMsgResp{
int32 errCode = 1;
string errMsg = 2;
}
message GetSuperGroupMsgReq{
string operationID = 1;
uint32 Seq = 2;
string groupID = 3;
}
message GetSuperGroupMsgResp{
int32 errCode = 1;
string errMsg = 2;
server_api_params.MsgData msgData = 3;
}
message GetWriteDiffMsgReq{
string operationID = 1;
uint32 Seq = 2;
}
message GetWriteDiffMsgResp{
int32 errCode = 1;
string errMsg = 2;
server_api_params.MsgData msgData = 3;
}
message ModifyMessageReactionExtensionsReq {
string operationID = 1;
string sourceID = 2;
string opUserID = 3;
int32 sessionType = 4;
map <string, server_api_params.KeyValue>reactionExtensionList = 5;
string clientMsgID = 6;
google.protobuf.StringValue ex = 7;
google.protobuf.StringValue attachedInfo = 8;
bool isReact = 9;
bool isExternalExtensions = 10;
int64 msgFirstModifyTime = 11;
}
message SetMessageReactionExtensionsReq {
string operationID = 1;
string sourceID = 2;
string opUserID = 3;
int32 opUserIDPlatformID = 4;
int32 sessionType = 5;
map <string, server_api_params.KeyValue>reactionExtensionList = 6;
string clientMsgID = 7;
google.protobuf.StringValue ex = 8;
google.protobuf.StringValue attachedInfo = 9;
bool isReact = 10;
bool isExternalExtensions = 11;
int64 msgFirstModifyTime = 12;
}
message SetMessageReactionExtensionsResp {
int32 errCode = 1;
string errMsg = 2;
string clientMsgID = 3;
int64 msgFirstModifyTime = 4;
bool isReact = 5;
repeated KeyValueResp result = 6;
}
message AddMessageReactionExtensionsReq {
string operationID = 1;
string sourceID = 2;
string opUserID = 3;
int32 opUserIDPlatformID = 4;
int32 sessionType = 5;
map <string, server_api_params.KeyValue>reactionExtensionList = 6;
string clientMsgID = 7;
google.protobuf.StringValue ex = 8;
google.protobuf.StringValue attachedInfo = 9;
bool isReact = 10;
bool isExternalExtensions = 11;
int64 msgFirstModifyTime = 12;
uint32 seq = 13;
}
message AddMessageReactionExtensionsResp {
int32 errCode = 1;
string errMsg = 2;
string clientMsgID = 3;
int64 msgFirstModifyTime = 4;
bool isReact = 5;
repeated KeyValueResp result = 6;
}
message GetMessageListReactionExtensionsReq {
string operationID = 1;
string opUserID = 2;
string sourceID = 3;
int32 sessionType = 4;
bool isExternalExtensions = 5;
message MessageReactionKey {
string clientMsgID = 1;
int64 msgFirstModifyTime = 2;
}
repeated string typeKeyList = 6;
repeated MessageReactionKey messageReactionKeyList = 7;
}
message GetMessageListReactionExtensionsResp{
int32 errCode = 1;
string errMsg = 2;
repeated SingleMessageExtensionResult singleMessageResult =3;
}
message SingleMessageExtensionResult {
int32 errCode = 1;
string errMsg = 2;
map <string, server_api_params.KeyValue>reactionExtensionList = 3;
string clientMsgID = 4;
}
message ModifyMessageReactionExtensionsResp {
int32 errCode = 1;
string errMsg = 2;
repeated ExtendMsgResp successList = 3;
repeated ExtendMsgResp failedList = 4;
}
message DeleteMessageListReactionExtensionsReq {
string operationID = 1;
string opUserID = 2;
string sourceID = 3;
int32 opUserIDPlatformID = 4;
int32 sessionType = 5;
string clientMsgID = 6;
bool isExternalExtensions = 7;
int64 msgFirstModifyTime = 8;
repeated server_api_params.KeyValue reactionExtensionList = 9;
}
message DeleteMessageListReactionExtensionsResp {
int32 errCode = 1;
string errMsg = 2;
repeated KeyValueResp result = 6;
}
message ExtendMsgResp {
ExtendMsg extendMsg = 1;
int32 errCode = 2;
string errMsg = 3;
}
message ExtendMsg {
map <string, KeyValueResp>reactionExtensionList = 1;
string clientMsgID = 2;
int64 msgFirstModifyTime = 3;
string attachedInfo = 4;
string ex = 5;
}
message KeyValueResp {
server_api_params.KeyValue keyValue = 1;
int32 errCode = 2;
string errMsg = 3;
}
message MsgDataToModifyByMQ{
string aggregationID = 1;
repeated MsgDataToMQ messageList = 2;
string triggerID = 3;
}
service msg {
rpc GetMaxAndMinSeq(server_api_params.GetMaxAndMinSeqReq) returns(server_api_params.GetMaxAndMinSeqResp);
rpc PullMessageBySeqList(server_api_params.PullMessageBySeqListReq) returns(server_api_params.PullMessageBySeqListResp);
rpc SendMsg(SendMsgReq) returns(SendMsgResp);
rpc DelMsgList(server_api_params.DelMsgListReq) returns(server_api_params.DelMsgListResp);
rpc DelSuperGroupMsg(DelSuperGroupMsgReq) returns(DelSuperGroupMsgResp);
rpc ClearMsg(ClearMsgReq) returns(ClearMsgResp);
rpc SetMsgMinSeq(SetMsgMinSeqReq) returns(SetMsgMinSeqResp);
rpc SetSendMsgStatus(SetSendMsgStatusReq) returns(SetSendMsgStatusResp);
rpc GetSendMsgStatus(GetSendMsgStatusReq) returns(GetSendMsgStatusResp);
rpc GetSuperGroupMsg(GetSuperGroupMsgReq) returns(GetSuperGroupMsgResp);
rpc GetWriteDiffMsg(GetWriteDiffMsgReq) returns(GetWriteDiffMsgResp);
// modify msg
rpc SetMessageReactionExtensions(SetMessageReactionExtensionsReq) returns(SetMessageReactionExtensionsResp);
rpc GetMessageListReactionExtensions(GetMessageListReactionExtensionsReq) returns(GetMessageListReactionExtensionsResp);
rpc AddMessageReactionExtensions(AddMessageReactionExtensionsReq) returns(AddMessageReactionExtensionsResp);
rpc DeleteMessageReactionExtensions(DeleteMessageListReactionExtensionsReq) returns(DeleteMessageListReactionExtensionsResp);
}

123
tools/data-conversion/openim/proto/sdk_ws/wrappers.proto Normal file
View File

@ -0,0 +1,123 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// Wrappers for primitive (non-message) types. These types are useful
// for embedding primitives in the `google.protobuf.Any` type and for places
// where we need to distinguish between the absence of a primitive
// typed field and its default value.
//
// These wrappers have no meaningful use within repeated fields as they lack
// the ability to detect presence on individual elements.
// These wrappers have no meaningful use within a map or a oneof since
// individual entries of a map or fields of a oneof can already detect presence.
syntax = "proto3";
package google.protobuf;
option csharp_namespace = "Google.Protobuf.WellKnownTypes";
option cc_enable_arenas = true;
option go_package = "google.golang.org/protobuf/types/known/wrapperspb";
option java_package = "com.google.protobuf";
option java_outer_classname = "WrappersProto";
option java_multiple_files = true;
option objc_class_prefix = "GPB";
// Wrapper message for `double`.
//
// The JSON representation for `DoubleValue` is JSON number.
message DoubleValue {
// The double value.
double value = 1;
}
// Wrapper message for `float`.
//
// The JSON representation for `FloatValue` is JSON number.
message FloatValue {
// The float value.
float value = 1;
}
// Wrapper message for `int64`.
//
// The JSON representation for `Int64Value` is JSON string.
message Int64Value {
// The int64 value.
int64 value = 1;
}
// Wrapper message for `uint64`.
//
// The JSON representation for `UInt64Value` is JSON string.
message UInt64Value {
// The uint64 value.
uint64 value = 1;
}
// Wrapper message for `int32`.
//
// The JSON representation for `Int32Value` is JSON number.
message Int32Value {
// The int32 value.
int32 value = 1;
}
// Wrapper message for `uint32`.
//
// The JSON representation for `UInt32Value` is JSON number.
message UInt32Value {
// The uint32 value.
uint32 value = 1;
}
// Wrapper message for `bool`.
//
// The JSON representation for `BoolValue` is JSON `true` and `false`.
message BoolValue {
// The bool value.
bool value = 1;
}
// Wrapper message for `string`.
//
// The JSON representation for `StringValue` is JSON string.
message StringValue {
// The string value.
string value = 1;
}
// Wrapper message for `bytes`.
//
// The JSON representation for `BytesValue` is JSON string.
message BytesValue {
// The bytes value.
bytes value = 1;
}

6622
tools/data-conversion/openim/proto/sdk_ws/ws.pb.go Normal file
View File

File diff suppressed because it is too large Load Diff

740
tools/data-conversion/openim/proto/sdk_ws/ws.proto Normal file
View File

@ -0,0 +1,740 @@
syntax = "proto3";
import "Open-IM-Server/pkg/proto/sdk_ws/wrappers.proto";
option go_package = "Open_IM/pkg/proto/sdk_ws;server_api_params";
package server_api_params;
////////////////////////////////base///////////////////////////////
message GroupInfo{
string groupID = 1;
string groupName = 2;
string notification = 3;
string introduction = 4;
string faceURL = 5;
string ownerUserID = 6;
uint32 createTime = 7;
uint32 memberCount = 8;
string ex = 9;
int32 status = 10;
string creatorUserID = 11;
int32 groupType = 12;
int32 needVerification = 13;
int32 lookMemberInfo = 14;
int32 applyMemberFriend = 15;
uint32 notificationUpdateTime = 16;
string notificationUserID = 17;
}
message GroupInfoForSet{
string groupID = 1;
string groupName = 2;
string notification = 3;
string introduction = 4;
string faceURL = 5;
string ex = 6;
google.protobuf.Int32Value needVerification = 7;
google.protobuf.Int32Value lookMemberInfo = 8;
google.protobuf.Int32Value applyMemberFriend = 9;
}
message GroupMemberFullInfo {
string groupID = 1 ;
string userID = 2 ;
int32 roleLevel = 3;
int32 joinTime = 4;
string nickname = 5;
string faceURL = 6;
int32 appMangerLevel = 7; //if >0
int32 joinSource = 8;
string operatorUserID = 9;
string ex = 10;
uint32 muteEndTime = 11;
string inviterUserID = 12;
}
message PublicUserInfo{
string userID = 1;
string nickname = 2;
string faceURL = 3;
int32 gender = 4;
string ex = 5;
}
message UserInfo{
string userID = 1;
string nickname = 2;
string faceURL = 3;
int32 gender = 4;
string phoneNumber = 5;
uint32 birth = 6;
string email = 7;
string ex = 8;
uint32 createTime = 9;
int32 appMangerLevel = 10;
int32 globalRecvMsgOpt = 11;
string birthStr = 12;
}
message FriendInfo{
string ownerUserID = 1;
string remark = 2;
uint32 createTime = 3;
UserInfo friendUser = 4;
int32 addSource = 5;
string operatorUserID = 6;
string ex = 7;
}
message BlackInfo{
string ownerUserID = 1;
uint32 createTime = 2;
PublicUserInfo blackUserInfo = 3;
int32 addSource = 4;
string operatorUserID = 5;
string ex = 6;
}
message GroupRequest{
PublicUserInfo userInfo = 1;
GroupInfo groupInfo = 2;
int32 handleResult = 3;
string reqMsg = 4;
string handleMsg = 5;
uint32 reqTime = 6;
string handleUserID = 7;
uint32 handleTime = 8;
string ex = 9;
int32 joinSource = 10;
string inviterUserID = 11;
}
message FriendRequest{
string fromUserID = 1;
string fromNickname = 2;
string fromFaceURL = 3;
int32 fromGender = 4;
string toUserID = 5;
string toNickname = 6;
string toFaceURL = 7;
int32 toGender = 8;
int32 handleResult = 9;
string reqMsg = 10;
uint32 createTime = 11;
string handlerUserID = 12;
string handleMsg = 13;
uint32 handleTime = 14;
string ex = 15;
}
///////////////////////////////////organization/////////////////////////////////////
message Department {
string departmentID = 1;
string faceURL = 2;
string name = 3;
string parentID = 4;
int32 order = 5;
int32 departmentType = 6;
uint32 createTime = 7;
uint32 subDepartmentNum = 8;
uint32 memberNum = 9;
string ex = 10;
}
message OrganizationUser {
string userID = 1;
string nickname = 2;
string englishName = 3;
string faceURL = 4;
int32 gender = 5;
string mobile = 6;
string telephone = 7;
uint32 birth = 8;
string email = 9;
uint32 createTime = 10;
string ex = 11;
string birthStr = 12;
}
message DepartmentMember {
string userID = 1;
string departmentID = 2;
int32 order = 3;
string position = 4;
int32 leader = 5;
int32 status = 6;
string ex = 7;
}
message UserDepartmentMember {
OrganizationUser organizationUser = 1;
DepartmentMember departmentMember = 2;
}
message UserInDepartment {
OrganizationUser organizationUser = 1;
repeated DepartmentMember departmentMemberList = 2;
}
///////////////////////////////////organization end//////////////////////////////////
///////////////////////////////////base end/////////////////////////////////////
message PullMessageBySeqListReq{
string userID = 1;
string operationID = 2;
repeated uint32 seqList = 3;
map <string, seqList>groupSeqList = 4;
}
message seqList {
repeated uint32 seqList = 1;
}
message MsgDataList {
repeated MsgData msgDataList = 1;
}
message PullMessageBySeqListResp {
int32 errCode = 1;
string errMsg = 2;
repeated MsgData list = 3;
map<string, MsgDataList> groupMsgDataList = 4;
}
message GetMaxAndMinSeqReq {
repeated string groupIDList = 1;
string userID = 2;
string operationID = 3;
}
message MaxAndMinSeq{
uint32 maxSeq = 1;
uint32 minSeq = 2;
}
message GetMaxAndMinSeqResp {
uint32 maxSeq = 1;
uint32 minSeq = 2;
int32 errCode = 3;
string errMsg = 4;
map<string, MaxAndMinSeq> groupMaxAndMinSeq = 5;
}
message UserSendMsgResp {
string serverMsgID = 1;
string clientMsgID = 2;
int64 sendTime = 3;
string ex = 4;
}
message MsgData {
string sendID = 1;
string recvID = 2;
string groupID = 3;
string clientMsgID = 4;
string serverMsgID = 5;
int32 senderPlatformID = 6;
string senderNickname = 7;
string senderFaceURL = 8;
int32 sessionType = 9;
int32 msgFrom = 10;
int32 contentType = 11;
bytes content = 12;
uint32 seq = 14;
int64 sendTime = 15;
int64 createTime = 16;
int32 status = 17;
map<string, bool> options = 18;
OfflinePushInfo offlinePushInfo = 19;
repeated string atUserIDList = 20;
bytes msgDataList = 21;
string attachedInfo = 22;
string ex = 23;
bool isReact = 40;
bool isExternalExtensions = 41;
int64 msgFirstModifyTime = 42;
}
message OfflinePushInfo{
string title = 1;
string desc = 2;
string ex = 3;
string iOSPushSound = 4;
bool iOSBadgeCount = 5;
}
message TipsComm{
bytes detail = 1;
string defaultTips = 2;
string jsonDetail = 3;
}
//////////////////////group/////////////////////
// OnGroupCreated()
message GroupCreatedTips{
GroupInfo group = 1;
GroupMemberFullInfo opUser = 2;
repeated GroupMemberFullInfo memberList = 3;
int64 operationTime = 4;
GroupMemberFullInfo groupOwnerUser = 5;
}
// OnGroupInfoSet()
message GroupInfoSetTips{
GroupMemberFullInfo opUser = 1; //who do this
int64 muteTime = 2;
GroupInfo group = 3;
}
// OnJoinGroupApplication()
message JoinGroupApplicationTips{
GroupInfo group = 1;
PublicUserInfo applicant = 2;
string reqMsg = 3;
}
// OnQuitGroup()
//Actively leave the group
message MemberQuitTips{
GroupInfo group = 1;
GroupMemberFullInfo quitUser = 2;
int64 operationTime = 3;
}
// OnApplicationGroupAccepted()
message GroupApplicationAcceptedTips{
GroupInfo group = 1;
GroupMemberFullInfo opUser = 2;
string handleMsg = 4;
int32 receiverAs = 5; // admin(==1) or applicant(==0)
}
// OnApplicationGroupRejected()
message GroupApplicationRejectedTips{
GroupInfo group = 1;
GroupMemberFullInfo opUser = 2;
string handleMsg = 4;
int32 receiverAs = 5; // admin(==1) or applicant(==0)
}
// OnTransferGroupOwner()
message GroupOwnerTransferredTips{
GroupInfo group = 1;
GroupMemberFullInfo opUser = 2;
GroupMemberFullInfo newGroupOwner = 3;
int64 operationTime = 4;
}
// OnMemberKicked()
message MemberKickedTips{
GroupInfo group = 1;
GroupMemberFullInfo opUser = 2;
repeated GroupMemberFullInfo kickedUserList = 3;
int64 operationTime = 4;
}
// OnMemberInvited()
message MemberInvitedTips{
GroupInfo group = 1;
GroupMemberFullInfo opUser = 2;
repeated GroupMemberFullInfo invitedUserList = 3;
int64 operationTime = 4;
}
//Actively join the group
message MemberEnterTips{
GroupInfo group = 1;
GroupMemberFullInfo entrantUser = 2;
int64 operationTime = 3;
}
message GroupDismissedTips{
GroupInfo group = 1;
GroupMemberFullInfo opUser = 2;
int64 operationTime = 3;
}
message GroupMemberMutedTips{
GroupInfo group = 1;
GroupMemberFullInfo opUser = 2;
int64 operationTime = 3;
GroupMemberFullInfo mutedUser = 4;
uint32 mutedSeconds = 5;
}
message GroupMemberCancelMutedTips{
GroupInfo group = 1;
GroupMemberFullInfo opUser = 2;
int64 operationTime = 3;
GroupMemberFullInfo mutedUser = 4;
}
message GroupMutedTips{
GroupInfo group = 1;
GroupMemberFullInfo opUser = 2;
int64 operationTime = 3;
}
message GroupCancelMutedTips{
GroupInfo group = 1;
GroupMemberFullInfo opUser = 2;
int64 operationTime = 3;
}
message GroupMemberInfoSetTips{
GroupInfo group = 1;
GroupMemberFullInfo opUser = 2;
int64 operationTime = 3;
GroupMemberFullInfo changedUser = 4;
}
message OrganizationChangedTips{
UserInfo opUser = 2;
int64 operationTime = 3;
}
//////////////////////friend/////////////////////
//message FriendInfo{
// UserInfo OwnerUser = 1;
// string Remark = 2;
// uint64 CreateTime = 3;
// UserInfo FriendUser = 4;
//}
message FriendApplication{
int64 addTime = 1;
string addSource = 2;
string addWording = 3;
}
message FromToUserID{
string fromUserID = 1;
string toUserID = 2;
}
//FromUserID apply to add ToUserID
message FriendApplicationTips{
FromToUserID fromToUserID = 1;
}
//FromUserID accept or reject ToUserID
message FriendApplicationApprovedTips{
FromToUserID fromToUserID = 1;
string handleMsg = 2;
}
//FromUserID accept or reject ToUserID
message FriendApplicationRejectedTips{
FromToUserID fromToUserID = 1;
string handleMsg = 2;
}
// FromUserID Added a friend ToUserID
message FriendAddedTips{
FriendInfo friend = 1;
int64 operationTime = 2;
PublicUserInfo opUser = 3; //who do this
}
// FromUserID deleted a friend ToUserID
message FriendDeletedTips{
FromToUserID fromToUserID = 1;
}
message BlackAddedTips{
FromToUserID fromToUserID = 1;
}
message BlackDeletedTips{
FromToUserID fromToUserID = 1;
}
message FriendInfoChangedTips{
FromToUserID fromToUserID = 1;
}
//////////////////////user/////////////////////
message UserInfoUpdatedTips{
string userID = 1;
}
//////////////////////conversation/////////////////////
message ConversationUpdateTips{
string UserID = 1;
repeated string conversationIDList = 2;
int64 updateUnreadCountTime = 3;
}
message ConversationSetPrivateTips{
string recvID = 1;
string sendID = 2;
bool isPrivate = 3;
}
////////////////////message///////////////////////
message DeleteMessageTips{
string opUserID = 1;
string userID = 2;
repeated uint32 seqList = 3;
}
///cms
message RequestPagination {
int32 pageNumber = 1;
int32 showNumber = 2;
}
message ResponsePagination {
int32 CurrentPage = 5;
int32 ShowNumber = 6;
}
///////////////////signal//////////////
message SignalReq {
oneof payload {
SignalInviteReq invite = 1;
SignalInviteInGroupReq inviteInGroup = 2;
SignalCancelReq cancel = 3;
SignalAcceptReq accept = 4;
SignalHungUpReq hungUp = 5;
SignalRejectReq reject = 6;
SignalGetRoomByGroupIDReq getRoomByGroupID = 7;
SignalOnRoomParticipantConnectedReq onRoomParticipantConnectedReq = 8;
SignalOnRoomParticipantDisconnectedReq onRoomParticipantDisconnectedReq = 9;
SignalGetTokenByRoomIDReq getTokenByRoomID = 10;
}
}
message SignalResp {
oneof payload {
SignalInviteReply invite = 1;
SignalInviteInGroupReply inviteInGroup = 2;
SignalCancelReply cancel = 3;
SignalAcceptReply accept = 4;
SignalHungUpReply hungUp = 5;
SignalRejectReply reject = 6;
SignalGetRoomByGroupIDReply getRoomByGroupID = 7;
SignalGetTokenByRoomIDReply getTokenByRoomID = 8;
}
}
message InvitationInfo {
string inviterUserID = 1;
repeated string inviteeUserIDList = 2;
string customData = 3;
string groupID = 4;
string roomID = 5;
int32 timeout = 6;
string mediaType = 7;
int32 platformID = 8;
int32 sessionType = 9;
int32 initiateTime = 10;
repeated string busyLineUserIDList = 11;
}
message ParticipantMetaData{
GroupInfo groupInfo = 1;
GroupMemberFullInfo groupMemberInfo = 2;
PublicUserInfo userInfo = 3;
}
message SignalInviteReq {
string opUserID = 1;
InvitationInfo invitation = 2;
OfflinePushInfo offlinePushInfo = 3;
ParticipantMetaData participant = 4;
}
message SignalInviteReply {
string token = 1;
string roomID = 2;
string liveURL = 3;
repeated string busyLineUserIDList = 4;
}
message SignalInviteInGroupReq {
string opUserID = 1;
InvitationInfo invitation = 2;
OfflinePushInfo offlinePushInfo = 3;
ParticipantMetaData participant = 4;
}
message SignalInviteInGroupReply {
string token = 1;
string roomID = 2;
string liveURL = 3;
repeated string busyLineUserIDList = 4;
}
message SignalCancelReq {
string opUserID = 1;
InvitationInfo invitation = 2;
OfflinePushInfo offlinePushInfo = 3;
ParticipantMetaData participant = 4;
}
message SignalCancelReply {
}
message SignalAcceptReq {
string opUserID = 1;
InvitationInfo invitation = 2;
OfflinePushInfo offlinePushInfo = 3;
ParticipantMetaData participant = 4;
int32 opUserPlatformID = 5;
}
message SignalAcceptReply {
string token = 1;
string roomID = 2;
string liveURL = 3;
}
message SignalHungUpReq {
string opUserID = 1;
InvitationInfo invitation = 2;
OfflinePushInfo offlinePushInfo = 3;
}
message SignalHungUpReply {
}
message SignalRejectReq {
string opUserID = 1;
InvitationInfo invitation = 2;
OfflinePushInfo offlinePushInfo = 3;
ParticipantMetaData participant = 4;
int32 opUserPlatformID = 5;
}
message SignalRejectReply {
}
message SignalGetRoomByGroupIDReq {
string opUserID = 1;
string groupID = 2;
ParticipantMetaData participant = 3;
}
message SignalGetRoomByGroupIDReply {
InvitationInfo invitation = 1;
repeated ParticipantMetaData participant = 2;
string roomID = 3;
}
message SignalOnRoomParticipantConnectedReq {
InvitationInfo invitation = 1;
repeated ParticipantMetaData participant = 2;
string groupID = 3;
}
message SignalOnRoomParticipantDisconnectedReq {
InvitationInfo invitation = 1;
repeated ParticipantMetaData participant = 2;
string groupID = 3;
}
message SignalGetTokenByRoomIDReq {
string roomID = 1;
string opUserID = 2;
ParticipantMetaData participant = 3;
string operationID = 4;
}
message SignalGetTokenByRoomIDReply {
string token = 1;
string liveURL = 2;
}
message DelMsgListReq{
string opUserID = 1;
string userID = 2;
repeated uint32 seqList = 3;
string operationID = 4;
}
message DelMsgListResp{
int32 errCode = 1;
string errMsg = 2;
}
message SetAppBackgroundStatusReq {
string userID = 1;
bool isBackground = 2;
}
message SetAppBackgroundStatusResp {
int32 errCode = 1;
string errMsg = 2;
}
message ExtendMsgSet {
string sourceID = 1;
int32 sessionType = 2;
map <string, ExtendMsg>extendMsgs = 3;
int64 MaxMsgUpdateTime = 4;
int32 extendMsgNum = 5;
int64 createTime = 6;
}
message ExtendMsg {
map <string, KeyValue>reactionExtensionList = 1;
string clientMsgID = 2;
int64 msgFirstModifyTime = 3;
string attachedInfo = 4;
string ex = 5;
}
message KeyValue {
string typeKey = 1;
string value = 2;
int64 latestUpdateTime = 3;
}

15
tools/data-conversion/redis-conversion.go
View File

@ -1,15 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package data_conversion

104
tools/data-conversion/utils/find_insert.go Normal file
View File

@ -0,0 +1,104 @@
package utils
import (
"fmt"
"gorm.io/gorm"
"gorm.io/gorm/schema"
"log"
"sync"
"sync/atomic"
)
func FindAndInsert[V2 any, V3 schema.Tabler](v2db *gorm.DB, v3db *gorm.DB, fn func(V2) (V3, bool)) (string, error) {
const batchSize = 100
var t V3
name := t.TableName()
if err := v3db.AutoMigrate(&t); err != nil {
return name, fmt.Errorf("auto migrate v3 %s failed %w", name, err)
}
for i := 0; ; i++ {
var v2s []V2
if err := v2db.Offset(i * batchSize).Limit(batchSize).Find(&v2s).Error; err != nil {
return name, fmt.Errorf("find v2 %s failed %w", name, err)
}
if len(v2s) == 0 {
return name, nil
}
v3s := make([]V3, 0, len(v2s))
for _, v := range v2s {
res, ok := fn(v)
if ok {
v3s = append(v3s, res)
}
}
if len(v3s) == 0 {
continue
}
if err := v3db.Create(&v3s).Error; err != nil {
return name, fmt.Errorf("insert v3 %s failed %w", name, err)
}
}
}
type TakeList []Task
func (l *TakeList) Append(fn ...Task) {
*l = append(*l, fn...)
}
type Task func() (string, error)
func RunTask(concurrency int, tasks TakeList) []string {
if len(tasks) == 0 {
return []string{}
}
if concurrency < 1 {
concurrency = 1
}
if concurrency > len(tasks) {
concurrency = len(tasks)
}
taskCh := make(chan func() (string, error), 4)
go func() {
defer close(taskCh)
for i := range tasks {
taskCh <- tasks[i]
}
}()
var lock sync.Mutex
var failedTables []string
var wg sync.WaitGroup
wg.Add(concurrency)
var count int64
for i := 0; i < concurrency; i++ {
go func() {
defer wg.Done()
for task := range taskCh {
name, err := task()
index := atomic.AddInt64(&count, 1)
if err == nil {
log.Printf("[%d/%d] %s success\n", index, len(tasks), name)
} else {
lock.Lock()
failedTables = append(failedTables, name)
lock.Unlock()
log.Printf("[%d/%d] %s failed %s\n", index, len(tasks), name, err)
return
}
}
}()
}
wg.Wait()
if len(failedTables) == 0 {
log.Println("all tables success")
} else {
log.Printf("failed tables %d: %+v\n", len(failedTables), failedTables)
}
return failedTables
}

14
tools/data-conversion/utils/time.go Normal file
View File

@ -0,0 +1,14 @@
package utils
import "time"
func InitTime(ts ...*time.Time) {
for i := range ts {
if ts[i] == nil {
continue
}
if ts[i].IsZero() || ts[i].UnixMicro() < 0 {
*ts[i] = time.UnixMicro(0)
}
}
}