blackteay/open-im-server
1
0
Fork 0
mirror of https://github.com/openimsdk/open-im-server.git synced 2025-12-03 19:02:31 +08:00
Code Issues Packages Projects Releases Wiki Activity

mysql2mongo

Browse Source
This commit is contained in:
withchao 2023-11-24 15:27:25 +08:00
parent f0d09dc4b6
commit 103fe1b654
1 changed files with 8 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
tools/mysql2mongo/main.go
View File

@ -11,7 +11,6 @@ import (
"go.mongodb.org/mongo-driver/mongo/options" "go.mongodb.org/mongo-driver/mongo/options"
"gorm.io/driver/mysql" "gorm.io/driver/mysql"
"gorm.io/gorm" "gorm.io/gorm"
"gorm.io/gorm/logger"
"log" "log"
"reflect" "reflect"
) )
@ -29,12 +28,12 @@ func main() {
var ( var (
mongoUsername = "root" // mongodb用户名 mongoUsername = "root" // mongodb用户名
mongoPassword = "openIM123" // mongodb密码 mongoPassword = "openIM123" // mongodb密码
mongoHosts = "127.0.0.1:13306" // mongodb地址 mongoHosts = "127.0.0.1:37017" // mongodb地址
mongoDatabase = "openIM_v3" // mongodb数据库名字 mongoDatabase = "openIM_v3" // mongodb数据库名字
) )
mysqlDSN := fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8mb4&parseTime=True&loc=Local", mysqlUsername, mysqlPassword, mysqlAddr, mysqlDatabase) mysqlDSN := fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8mb4&parseTime=True&loc=Local", mysqlUsername, mysqlPassword, mysqlAddr, mysqlDatabase)
mysqlDB, err := gorm.Open(mysql.Open(mysqlDSN), &gorm.Config{Logger: logger.Discard}) mysqlDB, err := gorm.Open(mysql.Open(mysqlDSN), &gorm.Config{ /* Logger: logger.Discard */ })
if err != nil { if err != nil {
log.Println("open mysql db failed", err) log.Println("open mysql db failed", err)
return return
@ -248,13 +247,13 @@ func NewTask[A interface{ TableName() string }, B any, C any](gormDB *gorm.DB, m
} }
var count int var count int
defer func() { defer func() {
log.Printf("convert %s %d records", tableName, count) log.Printf("completed convert %s total %d\n", tableName, count)
}() }()
const batch = 100 const batch = 100
for page := 0; ; page++ { for page := 0; ; page++ {
var res []A res := make([]A, 0, batch)
if err := gormDB.Find(&res).Limit(batch).Offset(page * batch).Error; err != nil { if err := gormDB.Limit(batch).Offset(page * batch).Find(&res).Error; err != nil {
return fmt.Errorf("find table %s failed, err: %w", tableName, err) return fmt.Errorf("find mysql table %s failed, err: %w", tableName, err)
} }
if len(res) == 0 { if len(res) == 0 {
return nil return nil
@ -264,12 +263,13 @@ func NewTask[A interface{ TableName() string }, B any, C any](gormDB *gorm.DB, m
temp[i] = convert(res[i]) temp[i] = convert(res[i])
} }
if _, err := coll.InsertMany(context.Background(), temp); err != nil { if _, err := coll.InsertMany(context.Background(), temp); err != nil {
return err return fmt.Errorf("insert mongo table %s failed, err: %w", tableName, err)
} }
count += len(res) count += len(res)
if len(res) < batch { if len(res) < batch {
return nil return nil
} }
log.Printf("current convert %s completed %d\n", tableName, count)
} }
} }