feat: add format set

Signed-off-by: Xinwei Xiong(cubxxw) <3293172751nss@gmail.com>
This commit is contained in:
Xinwei Xiong(cubxxw) 2023-11-29 16:27:26 +08:00
parent ec840aa0f6
commit 3b74ad56d4
42 changed files with 276 additions and 122 deletions

View File

@ -24,6 +24,7 @@ services:
restart: always restart: always
networks: networks:
server: server:
# open-im-server_server
ipv4_address: ${MYSQL_NETWORK_ADDRESS} ipv4_address: ${MYSQL_NETWORK_ADDRESS}
mongodb: mongodb:
@ -143,68 +144,68 @@ services:
server: server:
ipv4_address: ${OPENIM_WEB_NETWORK_ADDRESS} ipv4_address: ${OPENIM_WEB_NETWORK_ADDRESS}
openim-admin: # openim-admin:
image: ${IMAGE_REGISTRY}/openim-admin-front:v3.4.0 # image: ${IMAGE_REGISTRY}/openim-admin-front:v3.4.0
# image: ghcr.io/openimsdk/openim-admin-front:v3.4.0 # # image: ghcr.io/openimsdk/openim-admin-front:v3.4.0
# image: registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-admin-front:v3.4.0 # # image: registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-admin-front:v3.4.0
# image: openim/openim-admin-front:v3.4.0 # # image: openim/openim-admin-front:v3.4.0
container_name: openim-admin # container_name: openim-admin
restart: always # restart: always
ports: # ports:
- "${OPENIM_ADMIN_FRONT_PORT}:80" # - "${OPENIM_ADMIN_FRONT_PORT}:80"
networks: # networks:
server: # server:
ipv4_address: ${OPENIM_ADMIN_FRONT_NETWORK_ADDRESS} # ipv4_address: ${OPENIM_ADMIN_FRONT_NETWORK_ADDRESS}
prometheus: # prometheus:
image: prom/prometheus # image: prom/prometheus
container_name: prometheus # container_name: prometheus
hostname: prometheus # hostname: prometheus
restart: always # restart: always
volumes: # volumes:
- ./config/prometheus.yml:/etc/prometheus/prometheus.yml # - ./config/prometheus.yml:/etc/prometheus/prometheus.yml
- ./config/instance-down-rules.yml:/etc/prometheus/instance-down-rules.yml # - ./config/instance-down-rules.yml:/etc/prometheus/instance-down-rules.yml
ports: # ports:
- "${PROMETHEUS_PORT}:9090" # - "${PROMETHEUS_PORT}:9090"
networks: # networks:
server: # server:
ipv4_address: ${PROMETHEUS_NETWORK_ADDRESS} # ipv4_address: ${PROMETHEUS_NETWORK_ADDRESS}
alertmanager: # alertmanager:
image: prom/alertmanager # image: prom/alertmanager
container_name: alertmanager # container_name: alertmanager
hostname: alertmanager # hostname: alertmanager
restart: always # restart: always
volumes: # volumes:
- ./config/alertmanager.yml:/etc/alertmanager/alertmanager.yml # - ./config/alertmanager.yml:/etc/alertmanager/alertmanager.yml
- ./config/email.tmpl:/etc/alertmanager/email.tmpl # - ./config/email.tmpl:/etc/alertmanager/email.tmpl
ports: # ports:
- "${ALERT_MANAGER_PORT}:9093" # - "${ALERT_MANAGER_PORT}:9093"
networks: # networks:
server: # server:
ipv4_address: ${ALERT_MANAGER_NETWORK_ADDRESS} # ipv4_address: ${ALERT_MANAGER_NETWORK_ADDRESS}
grafana: # grafana:
image: grafana/grafana # image: grafana/grafana
container_name: grafana # container_name: grafana
hostname: grafana # hostname: grafana
user: root # user: root
restart: always # restart: always
ports: # ports:
- "${GRAFANA_PORT}:3000" # - "${GRAFANA_PORT}:3000"
volumes: # volumes:
- ${DATA_DIR}/components/grafana:/var/lib/grafana # - ${DATA_DIR}/components/grafana:/var/lib/grafana
networks: # networks:
server: # server:
ipv4_address: ${GRAFANA_NETWORK_ADDRESS} # ipv4_address: ${GRAFANA_NETWORK_ADDRESS}
node-exporter: # node-exporter:
image: quay.io/prometheus/node-exporter # image: quay.io/prometheus/node-exporter
container_name: node-exporter # container_name: node-exporter
hostname: node-exporter # hostname: node-exporter
restart: always # restart: always
ports: # ports:
- "${NODE_EXPORTER_PORT}:9100" # - "${NODE_EXPORTER_PORT}:9100"
networks: # networks:
server: # server:
ipv4_address: ${NODE_EXPORTER_NETWORK_ADDRESS} # ipv4_address: ${NODE_EXPORTER_NETWORK_ADDRESS}

View File

@ -17,7 +17,9 @@ package conversation
import ( import (
"context" "context"
"errors" "errors"
"github.com/OpenIMSDK/tools/tx" "github.com/OpenIMSDK/tools/tx"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/mgo" "github.com/openimsdk/open-im-server/v3/pkg/common/db/mgo"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/unrelation" "github.com/openimsdk/open-im-server/v3/pkg/common/db/unrelation"

View File

@ -16,6 +16,7 @@ package friend
import ( import (
"context" "context"
"github.com/OpenIMSDK/tools/tx" "github.com/OpenIMSDK/tools/tx"
"github.com/OpenIMSDK/protocol/sdkws" "github.com/OpenIMSDK/protocol/sdkws"

View File

@ -17,17 +17,19 @@ package group
import ( import (
"context" "context"
"fmt" "fmt"
pbconversation "github.com/OpenIMSDK/protocol/conversation"
"github.com/OpenIMSDK/protocol/wrapperspb"
"github.com/OpenIMSDK/tools/tx"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/mgo"
"github.com/openimsdk/open-im-server/v3/pkg/rpcclient/grouphash"
"math/big" "math/big"
"math/rand" "math/rand"
"strconv" "strconv"
"strings" "strings"
"time" "time"
pbconversation "github.com/OpenIMSDK/protocol/conversation"
"github.com/OpenIMSDK/protocol/wrapperspb"
"github.com/OpenIMSDK/tools/tx"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/mgo"
"github.com/openimsdk/open-im-server/v3/pkg/rpcclient/grouphash"
"github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/authverify"
"github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor"

View File

@ -17,6 +17,7 @@ package group
import ( import (
"context" "context"
"errors" "errors"
pbgroup "github.com/OpenIMSDK/protocol/group" pbgroup "github.com/OpenIMSDK/protocol/group"
) )

View File

@ -17,11 +17,12 @@ package third
import ( import (
"context" "context"
"fmt" "fmt"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/mgo"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/unrelation"
"net/url" "net/url"
"time" "time"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/mgo"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/unrelation"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3" "github.com/openimsdk/open-im-server/v3/pkg/common/db/s3"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3/cos" "github.com/openimsdk/open-im-server/v3/pkg/common/db/s3/cos"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3/minio" "github.com/openimsdk/open-im-server/v3/pkg/common/db/s3/minio"

View File

@ -17,11 +17,13 @@ package user
import ( import (
"context" "context"
"errors" "errors"
"github.com/OpenIMSDK/tools/tx"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/mgo"
"strings" "strings"
"time" "time"
"github.com/OpenIMSDK/tools/tx"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/mgo"
"github.com/OpenIMSDK/protocol/constant" "github.com/OpenIMSDK/protocol/constant"
"github.com/OpenIMSDK/protocol/sdkws" "github.com/OpenIMSDK/protocol/sdkws"
"github.com/OpenIMSDK/tools/errs" "github.com/OpenIMSDK/tools/errs"

View File

@ -16,10 +16,11 @@ package tools
import ( import (
"context" "context"
"github.com/OpenIMSDK/protocol/sdkws"
"math/rand" "math/rand"
"time" "time"
"github.com/OpenIMSDK/protocol/sdkws"
"github.com/OpenIMSDK/tools/log" "github.com/OpenIMSDK/tools/log"
"github.com/OpenIMSDK/tools/mcontext" "github.com/OpenIMSDK/tools/mcontext"
"github.com/OpenIMSDK/tools/utils" "github.com/OpenIMSDK/tools/utils"

View File

@ -17,10 +17,12 @@ package tools
import ( import (
"context" "context"
"fmt" "fmt"
"math"
"github.com/OpenIMSDK/protocol/sdkws" "github.com/OpenIMSDK/protocol/sdkws"
"github.com/OpenIMSDK/tools/tx" "github.com/OpenIMSDK/tools/tx"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/mgo" "github.com/openimsdk/open-im-server/v3/pkg/common/db/mgo"
"math"
"github.com/redis/go-redis/v9" "github.com/redis/go-redis/v9"
"google.golang.org/grpc" "google.golang.org/grpc"

View File

@ -46,7 +46,7 @@ func (a *RpcCmd) Exec() error {
return a.Execute() return a.Execute()
} }
func (a *RpcCmd) StartSvr(name string, rpcFn func(discov discoveryregistry.SvcDiscoveryRegistry, server *grpc.Server) error,) error { func (a *RpcCmd) StartSvr(name string, rpcFn func(discov discoveryregistry.SvcDiscoveryRegistry, server *grpc.Server) error) error {
if a.GetPortFlag() == 0 { if a.GetPortFlag() == 0 {
return errors.New("port is required") return errors.New("port is required")
} }

View File

@ -19,6 +19,7 @@ import (
"testing" "testing"
"github.com/OpenIMSDK/protocol/sdkws" "github.com/OpenIMSDK/protocol/sdkws"
relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
) )

View File

@ -17,11 +17,12 @@ package cache
import ( import (
"context" "context"
"fmt" "fmt"
"github.com/OpenIMSDK/protocol/constant"
"github.com/OpenIMSDK/tools/errs"
"strconv" "strconv"
"time" "time"
"github.com/OpenIMSDK/protocol/constant"
"github.com/OpenIMSDK/tools/errs"
"github.com/OpenIMSDK/tools/log" "github.com/OpenIMSDK/tools/log"
"github.com/dtm-labs/rockscache" "github.com/dtm-labs/rockscache"
@ -318,7 +319,12 @@ func (g *GroupCacheRedis) GetGroupMembersInfo(ctx context.Context, groupID strin
}) })
} }
func (g *GroupCacheRedis) GetGroupMembersPage(ctx context.Context, groupID string, userIDs []string, showNumber, pageNumber int32) (total uint32, groupMembers []*relationtb.GroupMemberModel, err error) { func (g *GroupCacheRedis) GetGroupMembersPage(
ctx context.Context,
groupID string,
userIDs []string,
showNumber, pageNumber int32,
) (total uint32, groupMembers []*relationtb.GroupMemberModel, err error) {
groupMemberIDs, err := g.GetGroupMemberIDs(ctx, groupID) groupMemberIDs, err := g.GetGroupMemberIDs(ctx, groupID)
if err != nil { if err != nil {
return 0, nil, err return 0, nil, err

View File

@ -18,11 +18,12 @@ import (
"context" "context"
"encoding/json" "encoding/json"
"errors" "errors"
relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"hash/crc32" "hash/crc32"
"strconv" "strconv"
"time" "time"
relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/OpenIMSDK/tools/log" "github.com/OpenIMSDK/tools/log"
"github.com/OpenIMSDK/protocol/constant" "github.com/OpenIMSDK/protocol/constant"

View File

@ -16,6 +16,7 @@ package controller
import ( import (
"context" "context"
"github.com/OpenIMSDK/tools/pagination" "github.com/OpenIMSDK/tools/pagination"
"github.com/OpenIMSDK/tools/log" "github.com/OpenIMSDK/tools/log"

View File

@ -16,9 +16,10 @@ package controller
import ( import (
"context" "context"
"github.com/OpenIMSDK/tools/pagination"
"time" "time"
"github.com/OpenIMSDK/tools/pagination"
"github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor"
"github.com/OpenIMSDK/protocol/constant" "github.com/OpenIMSDK/protocol/constant"
@ -237,7 +238,9 @@ func (c *conversationDatabase) SetUserConversations(ctx context.Context, ownerUs
if err != nil { if err != nil {
return err return err
} }
cache = cache.DelConversationIDs(ownerUserID).DelUserConversationIDsHash(ownerUserID).DelConversationNotReceiveMessageUserIDs(utils.Slice(notExistConversations, func(e *relationtb.ConversationModel) string { return e.ConversationID })...) cache = cache.DelConversationIDs(ownerUserID).
DelUserConversationIDsHash(ownerUserID).
DelConversationNotReceiveMessageUserIDs(utils.Slice(notExistConversations, func(e *relationtb.ConversationModel) string { return e.ConversationID })...)
} }
return cache.ExecDel(ctx) return cache.ExecDel(ctx)
}) })

View File

@ -16,9 +16,10 @@ package controller
import ( import (
"context" "context"
"github.com/OpenIMSDK/tools/pagination"
"time" "time"
"github.com/OpenIMSDK/tools/pagination"
"github.com/OpenIMSDK/protocol/constant" "github.com/OpenIMSDK/protocol/constant"
"github.com/OpenIMSDK/tools/errs" "github.com/OpenIMSDK/tools/errs"
"github.com/OpenIMSDK/tools/log" "github.com/OpenIMSDK/tools/log"
@ -97,7 +98,10 @@ func (f *friendDatabase) AddFriendRequest(ctx context.Context, fromUserID, toUse
m["create_time"] = time.Now() m["create_time"] = time.Now()
return f.friendRequest.UpdateByMap(ctx, fromUserID, toUserID, m) return f.friendRequest.UpdateByMap(ctx, fromUserID, toUserID, m)
case relation.IsNotFound(err): case relation.IsNotFound(err):
return f.friendRequest.Create(ctx, []*relation.FriendRequestModel{{FromUserID: fromUserID, ToUserID: toUserID, ReqMsg: reqMsg, Ex: ex, CreateTime: time.Now(), HandleTime: time.Unix(0, 0)}}) return f.friendRequest.Create(
ctx,
[]*relation.FriendRequestModel{{FromUserID: fromUserID, ToUserID: toUserID, ReqMsg: reqMsg, Ex: ex, CreateTime: time.Now(), HandleTime: time.Unix(0, 0)}},
)
default: default:
return err return err
} }

View File

@ -16,9 +16,10 @@ package controller
import ( import (
"context" "context"
"time"
"github.com/OpenIMSDK/tools/pagination" "github.com/OpenIMSDK/tools/pagination"
"github.com/dtm-labs/rockscache" "github.com/dtm-labs/rockscache"
"time"
"github.com/OpenIMSDK/protocol/constant" "github.com/OpenIMSDK/protocol/constant"
"github.com/OpenIMSDK/tools/tx" "github.com/OpenIMSDK/tools/tx"
@ -74,7 +75,14 @@ type GroupDatabase interface {
DeleteGroupMemberHash(ctx context.Context, groupIDs []string) error DeleteGroupMemberHash(ctx context.Context, groupIDs []string) error
} }
func NewGroupDatabase(rdb redis.UniversalClient, groupDB relationtb.GroupModelInterface, groupMemberDB relationtb.GroupMemberModelInterface, groupRequestDB relationtb.GroupRequestModelInterface, ctxTx tx.CtxTx, groupHash cache.GroupHash) GroupDatabase { func NewGroupDatabase(
rdb redis.UniversalClient,
groupDB relationtb.GroupModelInterface,
groupMemberDB relationtb.GroupMemberModelInterface,
groupRequestDB relationtb.GroupRequestModelInterface,
ctxTx tx.CtxTx,
groupHash cache.GroupHash,
) GroupDatabase {
rcOptions := rockscache.NewDefaultOptions() rcOptions := rockscache.NewDefaultOptions()
rcOptions.StrongConsistency = true rcOptions.StrongConsistency = true
rcOptions.RandomExpireAdjustment = 0.2 rcOptions.RandomExpireAdjustment = 0.2

View File

@ -16,9 +16,10 @@ package controller
import ( import (
"context" "context"
"github.com/OpenIMSDK/tools/pagination"
"time" "time"
"github.com/OpenIMSDK/tools/pagination"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache" "github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
) )

View File

@ -16,10 +16,12 @@ package controller
import ( import (
"context" "context"
"time"
"github.com/OpenIMSDK/tools/pagination" "github.com/OpenIMSDK/tools/pagination"
"github.com/OpenIMSDK/tools/tx" "github.com/OpenIMSDK/tools/tx"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"time"
"github.com/OpenIMSDK/protocol/user" "github.com/OpenIMSDK/protocol/user"

View File

@ -2,12 +2,14 @@ package mgo
import ( import (
"context" "context"
"github.com/OpenIMSDK/tools/mgoutil" "github.com/OpenIMSDK/tools/mgoutil"
"github.com/OpenIMSDK/tools/pagination" "github.com/OpenIMSDK/tools/pagination"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options" "go.mongodb.org/mongo-driver/mongo/options"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
) )
func NewBlackMongo(db *mongo.Database) (relation.BlackModelInterface, error) { func NewBlackMongo(db *mongo.Database) (relation.BlackModelInterface, error) {

View File

@ -2,14 +2,16 @@ package mgo
import ( import (
"context" "context"
"time"
"github.com/OpenIMSDK/protocol/constant" "github.com/OpenIMSDK/protocol/constant"
"github.com/OpenIMSDK/tools/mgoutil" "github.com/OpenIMSDK/tools/mgoutil"
"github.com/OpenIMSDK/tools/pagination" "github.com/OpenIMSDK/tools/pagination"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options" "go.mongodb.org/mongo-driver/mongo/options"
"time"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
) )
func NewConversationMongo(db *mongo.Database) (*ConversationMgo, error) { func NewConversationMongo(db *mongo.Database) (*ConversationMgo, error) {
@ -56,7 +58,12 @@ func (c *ConversationMgo) Find(ctx context.Context, ownerUserID string, conversa
} }
func (c *ConversationMgo) FindUserID(ctx context.Context, userIDs []string, conversationIDs []string) ([]string, error) { func (c *ConversationMgo) FindUserID(ctx context.Context, userIDs []string, conversationIDs []string) ([]string, error) {
return mgoutil.Find[string](ctx, c.coll, bson.M{"owner_user_id": bson.M{"$in": userIDs}, "conversation_id": bson.M{"$in": conversationIDs}}, options.Find().SetProjection(bson.M{"_id": 0, "owner_user_id": 1})) return mgoutil.Find[string](
ctx,
c.coll,
bson.M{"owner_user_id": bson.M{"$in": userIDs}, "conversation_id": bson.M{"$in": conversationIDs}},
options.Find().SetProjection(bson.M{"_id": 0, "owner_user_id": 1}),
)
} }
func (c *ConversationMgo) FindUserIDAllConversationID(ctx context.Context, userID string) ([]string, error) { func (c *ConversationMgo) FindUserIDAllConversationID(ctx context.Context, userID string) ([]string, error) {
@ -134,5 +141,10 @@ func (c *ConversationMgo) GetConversationIDsNeedDestruct(ctx context.Context) ([
} }
func (c *ConversationMgo) GetConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) ([]string, error) { func (c *ConversationMgo) GetConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) ([]string, error) {
return mgoutil.Find[string](ctx, c.coll, bson.M{"conversation_id": conversationID, "recv_msg_opt": bson.M{"$ne": constant.ReceiveMessage}}, options.Find().SetProjection(bson.M{"_id": 0, "owner_user_id": 1})) return mgoutil.Find[string](
ctx,
c.coll,
bson.M{"conversation_id": conversationID, "recv_msg_opt": bson.M{"$ne": constant.ReceiveMessage}},
options.Find().SetProjection(bson.M{"_id": 0, "owner_user_id": 1}),
)
} }

View File

@ -2,13 +2,15 @@ package mgo
import ( import (
"context" "context"
"github.com/OpenIMSDK/tools/mgoutil" "github.com/OpenIMSDK/tools/mgoutil"
"github.com/OpenIMSDK/tools/pagination" "github.com/OpenIMSDK/tools/pagination"
"go.mongodb.org/mongo-driver/mongo/options" "go.mongodb.org/mongo-driver/mongo/options"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
) )
// FriendMgo implements FriendModelInterface using MongoDB as the storage backend. // FriendMgo implements FriendModelInterface using MongoDB as the storage backend.

View File

@ -2,13 +2,15 @@ package mgo
import ( import (
"context" "context"
"github.com/OpenIMSDK/tools/mgoutil" "github.com/OpenIMSDK/tools/mgoutil"
"github.com/OpenIMSDK/tools/pagination" "github.com/OpenIMSDK/tools/pagination"
"go.mongodb.org/mongo-driver/mongo/options" "go.mongodb.org/mongo-driver/mongo/options"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
) )
func NewFriendRequestMongo(db *mongo.Database) (relation.FriendRequestModelInterface, error) { func NewFriendRequestMongo(db *mongo.Database) (relation.FriendRequestModelInterface, error) {

View File

@ -2,13 +2,15 @@ package mgo
import ( import (
"context" "context"
"time"
"github.com/OpenIMSDK/tools/mgoutil" "github.com/OpenIMSDK/tools/mgoutil"
"github.com/OpenIMSDK/tools/pagination" "github.com/OpenIMSDK/tools/pagination"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options" "go.mongodb.org/mongo-driver/mongo/options"
"time"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
) )
func NewGroupMongo(db *mongo.Database) (relation.GroupModelInterface, error) { func NewGroupMongo(db *mongo.Database) (relation.GroupModelInterface, error) {

View File

@ -2,13 +2,15 @@ package mgo
import ( import (
"context" "context"
"github.com/OpenIMSDK/protocol/constant" "github.com/OpenIMSDK/protocol/constant"
"github.com/OpenIMSDK/tools/mgoutil" "github.com/OpenIMSDK/tools/mgoutil"
"github.com/OpenIMSDK/tools/pagination" "github.com/OpenIMSDK/tools/pagination"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options" "go.mongodb.org/mongo-driver/mongo/options"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
) )
func NewGroupMember(db *mongo.Database) (relation.GroupMemberModelInterface, error) { func NewGroupMember(db *mongo.Database) (relation.GroupMemberModelInterface, error) {

View File

@ -2,12 +2,14 @@ package mgo
import ( import (
"context" "context"
"github.com/OpenIMSDK/tools/mgoutil" "github.com/OpenIMSDK/tools/mgoutil"
"github.com/OpenIMSDK/tools/pagination" "github.com/OpenIMSDK/tools/pagination"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options" "go.mongodb.org/mongo-driver/mongo/options"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
) )
func NewGroupRequestMgo(db *mongo.Database) (relation.GroupRequestModelInterface, error) { func NewGroupRequestMgo(db *mongo.Database) (relation.GroupRequestModelInterface, error) {

View File

@ -2,13 +2,15 @@ package mgo
import ( import (
"context" "context"
"time"
"github.com/OpenIMSDK/tools/mgoutil" "github.com/OpenIMSDK/tools/mgoutil"
"github.com/OpenIMSDK/tools/pagination" "github.com/OpenIMSDK/tools/pagination"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options" "go.mongodb.org/mongo-driver/mongo/options"
"time"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
) )
func NewLogMongo(db *mongo.Database) (relation.LogInterface, error) { func NewLogMongo(db *mongo.Database) (relation.LogInterface, error) {

View File

@ -2,11 +2,13 @@ package mgo
import ( import (
"context" "context"
"github.com/OpenIMSDK/tools/mgoutil" "github.com/OpenIMSDK/tools/mgoutil"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options" "go.mongodb.org/mongo-driver/mongo/options"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
) )
func NewS3Mongo(db *mongo.Database) (relation.ObjectInfoModelInterface, error) { func NewS3Mongo(db *mongo.Database) (relation.ObjectInfoModelInterface, error) {

View File

@ -2,13 +2,15 @@ package mgo
import ( import (
"context" "context"
"time"
"github.com/OpenIMSDK/tools/mgoutil" "github.com/OpenIMSDK/tools/mgoutil"
"github.com/OpenIMSDK/tools/pagination" "github.com/OpenIMSDK/tools/pagination"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options" "go.mongodb.org/mongo-driver/mongo/options"
"time"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
) )
func NewUserMongo(db *mongo.Database) (relation.UserModelInterface, error) { func NewUserMongo(db *mongo.Database) (relation.UserModelInterface, error) {

View File

@ -16,8 +16,9 @@ package relation
import ( import (
"context" "context"
"github.com/OpenIMSDK/tools/pagination"
"time" "time"
"github.com/OpenIMSDK/tools/pagination"
) )
type BlackModel struct { type BlackModel struct {

View File

@ -16,8 +16,9 @@ package relation
import ( import (
"context" "context"
"github.com/OpenIMSDK/tools/pagination"
"time" "time"
"github.com/OpenIMSDK/tools/pagination"
) )
type ConversationModel struct { type ConversationModel struct {

View File

@ -16,8 +16,9 @@ package relation
import ( import (
"context" "context"
"github.com/OpenIMSDK/tools/pagination"
"time" "time"
"github.com/OpenIMSDK/tools/pagination"
) )
// FriendModel represents the data structure for a friend relationship in MongoDB. // FriendModel represents the data structure for a friend relationship in MongoDB.

View File

@ -16,8 +16,9 @@ package relation
import ( import (
"context" "context"
"github.com/OpenIMSDK/tools/pagination"
"time" "time"
"github.com/OpenIMSDK/tools/pagination"
) )
type FriendRequestModel struct { type FriendRequestModel struct {

View File

@ -16,8 +16,9 @@ package relation
import ( import (
"context" "context"
"github.com/OpenIMSDK/tools/pagination"
"time" "time"
"github.com/OpenIMSDK/tools/pagination"
) )
type GroupModel struct { type GroupModel struct {

View File

@ -16,8 +16,9 @@ package relation
import ( import (
"context" "context"
"github.com/OpenIMSDK/tools/pagination"
"time" "time"
"github.com/OpenIMSDK/tools/pagination"
) )
type GroupMemberModel struct { type GroupMemberModel struct {

View File

@ -16,8 +16,9 @@ package relation
import ( import (
"context" "context"
"github.com/OpenIMSDK/tools/pagination"
"time" "time"
"github.com/OpenIMSDK/tools/pagination"
) )
type GroupRequestModel struct { type GroupRequestModel struct {

View File

@ -2,8 +2,9 @@ package relation
import ( import (
"context" "context"
"github.com/OpenIMSDK/tools/pagination"
"time" "time"
"github.com/OpenIMSDK/tools/pagination"
) )
type LogModel struct { type LogModel struct {

View File

@ -16,8 +16,9 @@ package relation
import ( import (
"context" "context"
"github.com/OpenIMSDK/tools/pagination"
"time" "time"
"github.com/OpenIMSDK/tools/pagination"
) )
type UserModel struct { type UserModel struct {

View File

@ -5,6 +5,7 @@ import (
"crypto/md5" "crypto/md5"
"encoding/binary" "encoding/binary"
"encoding/json" "encoding/json"
"github.com/OpenIMSDK/protocol/group" "github.com/OpenIMSDK/protocol/group"
"github.com/OpenIMSDK/protocol/sdkws" "github.com/OpenIMSDK/protocol/sdkws"
"github.com/OpenIMSDK/tools/utils" "github.com/OpenIMSDK/tools/utils"

View File

@ -20,9 +20,10 @@ import (
"testing" "testing"
"time" "time"
"github.com/openimsdk/open-im-server/v3/pkg/common/config"
"github.com/redis/go-redis/v9" "github.com/redis/go-redis/v9"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/openimsdk/open-im-server/v3/pkg/common/config"
) )
func TestCheckMysql(t *testing.T) { func TestCheckMysql(t *testing.T) {

67
tools/up35/README.md Normal file
View File

@ -0,0 +1,67 @@
# README for OpenIM Server Data Conversion Tool
## Overview
This tool is part of the OpenIM Server suite, specifically designed for data conversion between MySQL and MongoDB databases. It handles the migration of various data types, including user information, friendships, group memberships, and more from a MySQL database to MongoDB, ensuring data consistency and integrity during the transition.
## Features
+ **Configurable Database Connections:** Supports connections to both MySQL and MongoDB, configurable through a YAML file.
+ **Data Conversion Tasks:** Converts a range of data models, including user profiles, friend requests, group memberships, and logs.
+ **Version Control:** Maintains data versioning, ensuring only necessary migrations are performed.
+ **Error Handling:** Robust error handling for database connectivity and query execution.
## Requirements
+ Go programming environment
+ MySQL and MongoDB servers
+ OpenIM Server dependencies installed
## Installation
1. Ensure Go is installed and set up on your system.
2. Clone the OpenIM Server repository.
3. Navigate to the directory containing this tool.
4. Install required dependencies.
## Configuration
+ Configuration is managed through a YAML file specified at runtime.
+ Set up the MySQL and MongoDB connection parameters in the config file.
## Usage
To run the tool, use the following command from the terminal:
```go
make build BINS="up35"
```
Where `path/to/config.yaml` is the path to your configuration file.
## Functionality
The main functions of the script include:
+ `InitConfig(path string)`: Reads and parses the YAML configuration file.
+ `GetMysql()`: Establishes a connection to the MySQL database.
+ `GetMongo()`: Establishes a connection to the MongoDB database.
+ `Main(path string)`: Orchestrates the data migration process.
+ `SetMongoDataVersion(db *mongo.Database, curver string)`: Updates the data version in MongoDB after migration.
+ `NewTask(...)`: Generic function to handle the migration of different data types.
+ `insertMany(coll *mongo.Collection, objs []any)`: Inserts multiple records into a MongoDB collection.
+ `getColl(obj any)`: Retrieves the MongoDB collection associated with a given object.
+ `convert struct`: Contains methods for converting MySQL models to MongoDB models.
## Notes
+ Ensure that the MySQL and MongoDB instances are accessible and that the credentials provided in the config file are correct.
+ It is advisable to backup databases before running the migration to prevent data loss.
## Contributing
Contributions to improve the tool or address issues are welcome. Please follow the project's contribution guidelines.
## License
Refer to the project's license document for usage and distribution rights.

View File

@ -5,12 +5,13 @@ import (
"errors" "errors"
"flag" "flag"
"fmt" "fmt"
"log"
"os"
"path/filepath"
"reflect"
"strconv"
"github.com/go-sql-driver/mysql" "github.com/go-sql-driver/mysql"
"github.com/openimsdk/open-im-server/v3/pkg/common/config"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/mgo"
mongoModel "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/unrelation"
mysqlModel "github.com/openimsdk/open-im-server/v3/tools/data-conversion/openim/mysql/v3"
"go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options" "go.mongodb.org/mongo-driver/mongo/options"
@ -18,10 +19,12 @@ import (
gormMysql "gorm.io/driver/mysql" gormMysql "gorm.io/driver/mysql"
"gorm.io/gorm" "gorm.io/gorm"
"gorm.io/gorm/logger" "gorm.io/gorm/logger"
"log"
"os" "github.com/openimsdk/open-im-server/v3/pkg/common/config"
"reflect" "github.com/openimsdk/open-im-server/v3/pkg/common/db/mgo"
"strconv" mongoModel "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/unrelation"
mysqlModel "github.com/openimsdk/open-im-server/v3/tools/data-conversion/openim/mysql/v3"
) )
const ( const (
@ -31,9 +34,12 @@ const (
) )
func main() { func main() {
var path string var dir string
flag.StringVar(&path, "c", "", "path config file") flag.StringVar(&dir, "c", "../../../../../", "path to config directory")
flag.Parse() flag.Parse()
path := filepath.Join(dir, "config.yaml")
if err := Main(path); err != nil { if err := Main(path); err != nil {
log.Fatal(err) log.Fatal(err)
return return