From 7ef37a9fa28250812a31487be2ec1b09e31c7806 Mon Sep 17 00:00:00 2001 From: Gordon <1432970085@qq.com> Date: Mon, 4 Jul 2022 11:53:11 +0800 Subject: [PATCH 1/7] add docker-compose shard cluster --- docker-compose-mongos.yaml | 139 ++++++++++++++++++++++++++++++ docker-compose-shard-cluster.yaml | 79 +++++++++++++++++ 2 files changed, 218 insertions(+) create mode 100644 docker-compose-mongos.yaml create mode 100644 docker-compose-shard-cluster.yaml diff --git a/docker-compose-mongos.yaml b/docker-compose-mongos.yaml new file mode 100644 index 000000000..309ccb98b --- /dev/null +++ b/docker-compose-mongos.yaml @@ -0,0 +1,139 @@ +version: "3" +#fixme Clone openIM Server project before using docker-compose,project address：https://github.com/OpenIMSDK/Open-IM-Server.git + +services: + mysql: + image: mysql:5.7 + ports: + - 13306:3306 + - 23306:33060 + container_name: mysql + volumes: + - ./components/mysql/data:/var/lib/mysql + - /etc/localtime:/etc/localtime + environment: + MYSQL_ROOT_PASSWORD: openIM + restart: always + + mongos: + image: mongo:latest + ports: + - 37017:27017 + container_name: mongos + command: mongos --configdb config/103.116.45.174:27019 --bind_ip_all + volumes: + - ./components/mongos_mongodb/data/db:/data/db + - ./components/mongos_mongodb/data/logs:/data/logs + - ./components/mongos_mongodb/data/conf:/etc/mongo + - ./components/mongos_mongodb/data/configdb:/data/configdb + environment: + - TZ=Asia/Shanghai + # cache + - wiredTigerCacheSizeGB=1 + # environment: + # - MONGO_INITDB_ROOT_USERNAME=openIM + # - MONGO_INITDB_ROOT_PASSWORD=openIM + + + #TZ: Asia/Shanghai + restart: always + + + redis: + image: redis + ports: + - 16379:6379 + container_name: redis + volumes: + - ./components/redis/data:/data + #redis config file + - ./components/redis/config/redis.conf:/usr/local/redis/config/redis.conf + environment: + TZ: Asia/Shanghai + restart: always + sysctls: + net.core.somaxconn: 1024 + command: redis-server --requirepass openIM --appendonly yes + + + zookeeper: + image: wurstmeister/zookeeper + ports: + - 2181:2181 + container_name: zookeeper + volumes: + - /etc/localtime:/etc/localtime + environment: + TZ: Asia/Shanghai + restart: always + + + kafka: + image: wurstmeister/kafka + container_name: kafka + restart: always + ports: + - 9092:9092 + environment: + TZ: Asia/Shanghai + KAFKA_BROKER_ID: 0 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_CREATE_TOPICS: "ws2ms_chat:8:1,ms2ps_chat:8:1,msg_to_mongo:8:1" + KAFKA_ADVERTISED_LISTENERS: INSIDE://127.0.0.1:9092,OUTSIDE://103.116.45.174:9092 + KAFKA_LISTENERS: INSIDE://:9092,OUTSIDE://:9093 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT" + KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE + depends_on: + - zookeeper + + etcd: + image: quay.io/coreos/etcd + ports: + - 2379:2379 + - 2380:2380 + container_name: etcd + volumes: + - /etc/timezone:/etc/timezone + - /etc/localtime:/etc/localtime + environment: + ETCDCTL_API: 3 + restart: always + command: /usr/local/bin/etcd --name etcd0 --data-dir /etcd-data --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://0.0.0.0:2379 --listen-peer-urls http://0.0.0.0:2380 --initial-advertise-peer-urls http://0.0.0.0:2380 --initial-cluster etcd0=http://0.0.0.0:2380 --initial-cluster-token tkn --initial-cluster-state new + + minio: + image: minio/minio + ports: + - 10005:9000 + - 9090:9090 + container_name: minio + volumes: + - /mnt/data:/data + - /mnt/config:/root/.minio + environment: + MINIO_ROOT_USER: user12345 + MINIO_ROOT_PASSWORD: key12345 + restart: always + command: minio server /data --console-address ':9090' + + open_im_server: + image: openim/open_im_server:v2.2.0 + container_name: open_im_server + volumes: + - ./logs:/Open-IM-Server/logs + - ./config/config.yaml:/Open-IM-Server/config/config.yaml + - ./db/sdk:/Open-IM-Server/db/sdk + - ./script:/Open-IM-Server/script + restart: always + depends_on: + - kafka + - mysql + - mongodb + - redis + - etcd + - minio + network_mode: "host" + logging: + driver: json-file + options: + max-size: "1g" + max-file: "2" diff --git a/docker-compose-shard-cluster.yaml b/docker-compose-shard-cluster.yaml new file mode 100644 index 000000000..8bc56b83d --- /dev/null +++ b/docker-compose-shard-cluster.yaml @@ -0,0 +1,79 @@ +version: '3' +networks: + mongo-network: + external: false +services: + # 配置服务器configsvr + config: + image: mongo:latest + networks: + - mongo-network + container_name: config + restart: always + ports: + - 27019:27019 + command: --configsvr --replSet "config" --bind_ip_all + volumes: + - ./components/config_mongodb/data/db:/data/db + - ./components/config_mongodb/data/logs:/data/logs + - ./components/config_mongodb/data/conf:/etc/mongo + - ./components/config_mongodb/data/configdb:/data/configdb + environment: + - TZ=Asia/Shanghai + # cache + - wiredTigerCacheSizeGB=1 + + # shard分片1 + shard1: + image: mongo:latest + networks: + - mongo-network + container_name: shard1 + restart: always + ports: + - 27118:27018 + command: --shardsvr --replSet "shard1" --bind_ip_all + volumes: + - /data01/mongo/shard1/db:/data/db + - /data01/mongo/shard1/configdb:/data/configdb + environment: + - TZ=Asia/Shanghai + # cache + - wiredTigerCacheSizeGB=1 + + # shard分片2 + shard2: + image: mongo:latest + networks: + - mongo-network + container_name: shard2 + restart: always + ports: + - 27218:27018 + command: --shardsvr --replSet "shard2" --bind_ip_all + volumes: + - /data02/mongo/shard2/db:/data/db + - /data02/mongo/shard2/configdb:/data/configdb + environment: + - TZ=Asia/Shanghai + # cache + - wiredTigerCacheSizeGB=1 + # - ${PWD}/key.file:/data/mongodb/key.file + + # shard分片3 + shard3: + image: mongo:latest + networks: + - mongo-network + container_name: shard3 + restart: always + ports: + - 27318:27018 + command: --shardsvr --replSet "shard3" --bind_ip_all + volumes: + - /data03/mongo/shard3/db:/data/db + - /data03/mongo/shard3/configdb:/data/configdb + environment: + - TZ=Asia/Shanghai + # cache + - wiredTigerCacheSizeGB=1 From b14bcf966df50fee024c9c524b9a32183beebb22 Mon Sep 17 00:00:00 2001 From: Gordon <1432970085@qq.com> Date: Mon, 4 Jul 2022 12:09:29 +0800 Subject: [PATCH 2/7] add docker-compose shard cluster --- docker-compose-mongos.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose-mongos.yaml b/docker-compose-mongos.yaml index 309ccb98b..1ad89204d 100644 --- a/docker-compose-mongos.yaml +++ b/docker-compose-mongos.yaml @@ -127,7 +127,7 @@ services: depends_on: - kafka - mysql - - mongodb + - mongos - redis - etcd - minio From a504961e93f96b49494ead7e8b95333958b27896 Mon Sep 17 00:00:00 2001 From: Gordon <1432970085@qq.com> Date: Mon, 4 Jul 2022 13:02:35 +0800 Subject: [PATCH 3/7] add docker-compose shard cluster --- docker-compose-shard-cluster.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docker-compose-shard-cluster.yaml b/docker-compose-shard-cluster.yaml index 8bc56b83d..5bfa4138a 100644 --- a/docker-compose-shard-cluster.yaml +++ b/docker-compose-shard-cluster.yaml @@ -77,3 +77,20 @@ services: - TZ=Asia/Shanghai # cache - wiredTigerCacheSizeGB=1 + # shard分片4 + shard4: + image: mongo:latest + networks: + - mongo-network + container_name: shard4 + restart: always + ports: + - 27418:27018 + command: --shardsvr --replSet "shard4" --bind_ip_all + volumes: + - /data04/mongo/shard4/db:/data/db + - /data04/mongo/shard4/configdb:/data/configdb + environment: + - TZ=Asia/Shanghai + # cache + - wiredTigerCacheSizeGB=1 \ No newline at end of file From b0669b7eea9fe7a1d01f21e23144adcba55f51b0 Mon Sep 17 00:00:00 2001 From: Gordon <1432970085@qq.com> Date: Mon, 4 Jul 2022 16:30:37 +0800 Subject: [PATCH 4/7] add docker-compose shard cluster --- docker-compose-shard-cluster.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docker-compose-shard-cluster.yaml b/docker-compose-shard-cluster.yaml index 5bfa4138a..e3244a4ba 100644 --- a/docker-compose-shard-cluster.yaml +++ b/docker-compose-shard-cluster.yaml @@ -90,6 +90,23 @@ services: volumes: - /data04/mongo/shard4/db:/data/db - /data04/mongo/shard4/configdb:/data/configdb + environment: + - TZ=Asia/Shanghai + # cache + - wiredTigerCacheSizeGB=1 + # shard分片5 + shard5: + image: mongo:latest + networks: + - mongo-network + container_name: shard5 + restart: always + ports: + - 27518:27018 + command: --shardsvr --replSet "shard5" --bind_ip_all + volumes: + - /data05/mongo/shard5/db:/data/db + - /data05/mongo/shard5/configdb:/data/configdb environment: - TZ=Asia/Shanghai # cache From d0e7147911d3ed6d8b27c4b011082164724b23b2 Mon Sep 17 00:00:00 2001 From: wangchuxiao Date: Tue, 5 Jul 2022 11:01:10 +0800 Subject: [PATCH 5/7] demo --- internal/demo/register/login.go | 7 +++++-- internal/demo/register/reset_password.go | 2 +- internal/demo/register/send_code.go | 2 +- pkg/common/db/mysql_model/im_mysql_model/demo_model.go | 6 +++--- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/internal/demo/register/login.go b/internal/demo/register/login.go index 4c475f57d..28852a83d 100644 --- a/internal/demo/register/login.go +++ b/internal/demo/register/login.go @@ -15,6 +15,7 @@ import ( ) type ParamsLogin struct { + UserID string `json:"userID"` Email string `json:"email"` PhoneNumber string `json:"phoneNumber"` Password string `json:"password"` @@ -32,11 +33,13 @@ func Login(c *gin.Context) { var account string if params.Email != "" { account = params.Email - } else { + } else if params.PhoneNumber != "" { account = params.PhoneNumber + } else { + account = params.UserID } - r, err := im_mysql_model.GetRegister(account, params.AreaCode) + r, err := im_mysql_model.GetRegister(account, params.AreaCode, params.UserID) if err != nil { log.NewError(params.OperationID, "user have not register", params.Password, account, err.Error()) c.JSON(http.StatusOK, gin.H{"errCode": constant.NotRegistered, "errMsg": "Mobile phone number is not registered"}) diff --git a/internal/demo/register/reset_password.go b/internal/demo/register/reset_password.go index 6481a892e..e3b1032c5 100644 --- a/internal/demo/register/reset_password.go +++ b/internal/demo/register/reset_password.go @@ -44,7 +44,7 @@ func ResetPassword(c *gin.Context) { return } } - user, err := im_mysql_model.GetRegister(account, req.AreaCode) + user, err := im_mysql_model.GetRegister(account, req.AreaCode, "") if err != nil || user.Account == "" { if err != nil { log.NewError(req.OperationID, utils.GetSelfFuncName(), "get register error", err.Error()) diff --git a/internal/demo/register/send_code.go b/internal/demo/register/send_code.go index 6bb41f153..cd67c0c04 100644 --- a/internal/demo/register/send_code.go +++ b/internal/demo/register/send_code.go @@ -66,7 +66,7 @@ func SendVerificationCode(c *gin.Context) { } switch params.UsedFor { case constant.VerificationCodeForRegister: - _, err := im_mysql_model.GetRegister(account, params.AreaCode) + _, err := im_mysql_model.GetRegister(account, params.AreaCode, "") if err == nil { log.NewError(params.OperationID, "The phone number has been registered", params) c.JSON(http.StatusOK, gin.H{"errCode": constant.HasRegistered, "errMsg": "The phone number has been registered"}) diff --git a/pkg/common/db/mysql_model/im_mysql_model/demo_model.go b/pkg/common/db/mysql_model/im_mysql_model/demo_model.go index 09d04b46d..fcfc2705f 100644 --- a/pkg/common/db/mysql_model/im_mysql_model/demo_model.go +++ b/pkg/common/db/mysql_model/im_mysql_model/demo_model.go @@ -5,14 +5,14 @@ import ( _ "github.com/jinzhu/gorm" ) -func GetRegister(account, areaCode string) (*db.Register, error) { +func GetRegister(account, areaCode, userID string) (*db.Register, error) { dbConn, err := db.DB.MysqlDB.DefaultGormDB() if err != nil { return nil, err } var r db.Register - return &r, dbConn.Table("registers").Where("account = ? or account =? and area_code=?", - account, account, areaCode).Take(&r).Error + return &r, dbConn.Table("registers").Where("user_id = ? and user_id != ? or account = ? or account =? and area_code=?", + userID, "", account, account, areaCode).Take(&r).Error } func SetPassword(account, password, ex, userID, areaCode string) error { From 3ad3b30d97092ba8340f2f124503079029780e06 Mon Sep 17 00:00:00 2001 From: wangchuxiao Date: Tue, 5 Jul 2022 14:30:53 +0800 Subject: [PATCH 6/7] demo --- internal/demo/register/set_password.go | 43 ++++++++++++------- .../im_mysql_model/organization_model.go | 2 +- 2 files changed, 28 insertions(+), 17 deletions(-) diff --git a/internal/demo/register/set_password.go b/internal/demo/register/set_password.go index 19831399b..2f928efda 100644 --- a/internal/demo/register/set_password.go +++ b/internal/demo/register/set_password.go @@ -18,6 +18,7 @@ import ( ) type ParamsSetPassword struct { + UserID string `json:"userID"` Email string `json:"email"` Nickname string `json:"nickname"` PhoneNumber string `json:"phoneNumber"` @@ -40,29 +41,37 @@ func SetPassword(c *gin.Context) { var account string if params.Email != "" { account = params.Email - } else { + } else if params.PhoneNumber != "" { account = params.PhoneNumber + } else { + account = params.UserID } if params.Nickname == "" { params.Nickname = account } - if (config.Config.Demo.UseSuperCode && params.VerificationCode != config.Config.Demo.SuperCode) || !config.Config.Demo.UseSuperCode { - accountKey := params.AreaCode + account + "_" + constant.VerificationCodeForRegisterSuffix - v, err := db.DB.GetAccountCode(accountKey) - if err != nil || v != params.VerificationCode { - log.NewError(params.OperationID, "password Verification code error", account, params.VerificationCode) - data := make(map[string]interface{}) - data["PhoneNumber"] = account - c.JSON(http.StatusOK, gin.H{"errCode": constant.CodeInvalidOrExpired, "errMsg": "Verification code error!", "data": data}) - return + if params.UserID == "" { + if (config.Config.Demo.UseSuperCode && params.VerificationCode != config.Config.Demo.SuperCode) || !config.Config.Demo.UseSuperCode { + accountKey := params.AreaCode + account + "_" + constant.VerificationCodeForRegisterSuffix + v, err := db.DB.GetAccountCode(accountKey) + if err != nil || v != params.VerificationCode { + log.NewError(params.OperationID, "password Verification code error", account, params.VerificationCode) + data := make(map[string]interface{}) + data["PhoneNumber"] = account + c.JSON(http.StatusOK, gin.H{"errCode": constant.CodeInvalidOrExpired, "errMsg": "Verification code error!", "data": data}) + return + } } } //userID := utils.Base64Encode(account) - - userID := utils.Md5(params.OperationID + strconv.FormatInt(time.Now().UnixNano(), 10)) - bi := big.NewInt(0) - bi.SetString(userID[0:8], 16) - userID = bi.String() + var userID string + if params.UserID == "" { + userID := utils.Md5(params.OperationID + strconv.FormatInt(time.Now().UnixNano(), 10)) + bi := big.NewInt(0) + bi.SetString(userID[0:8], 16) + userID = bi.String() + } else { + userID = params.UserID + } url := config.Config.Demo.ImAPIURL + "/auth/user_register" openIMRegisterReq := api.UserRegisterReq{} @@ -97,7 +106,9 @@ func SetPassword(c *gin.Context) { } log.Info(params.OperationID, "end setPassword", account, params.Password) // demo onboarding - onboardingProcess(params.OperationID, userID, params.Nickname, params.FaceURL, params.AreaCode+params.PhoneNumber, params.Email) + if params.UserID == "" { + onboardingProcess(params.OperationID, userID, params.Nickname, params.FaceURL, params.AreaCode+params.PhoneNumber, params.Email) + } c.JSON(http.StatusOK, gin.H{"errCode": constant.NoError, "errMsg": "", "data": openIMRegisterResp.UserToken}) return } diff --git a/pkg/common/db/mysql_model/im_mysql_model/organization_model.go b/pkg/common/db/mysql_model/im_mysql_model/organization_model.go index 52b5bf245..6ac231032 100644 --- a/pkg/common/db/mysql_model/im_mysql_model/organization_model.go +++ b/pkg/common/db/mysql_model/im_mysql_model/organization_model.go @@ -302,6 +302,6 @@ func GetRandomDepartmentID() (string, error) { return "", err } department := &db.Department{} - err = dbConn.Model(department).Order("RAND()").Where("related_group_id != ? AND department_id != ?", "", "0").First(department).Error + err = dbConn.Model(department).Order("RAND()").Where("related_group_id != ? AND department_id != ? AND department_type = ?", "", "0", 1).First(department).Error return department.DepartmentID, err } From 04b7dd719c05094cac16bd8a69c96510b6eef735 Mon Sep 17 00:00:00 2001 From: Gordon <1432970085@qq.com> Date: Tue, 5 Jul 2022 16:33:05 +0800 Subject: [PATCH 7/7] test group --- pkg/common/db/redisModel_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/common/db/redisModel_test.go b/pkg/common/db/redisModel_test.go index cd82aca03..70c7f4d22 100644 --- a/pkg/common/db/redisModel_test.go +++ b/pkg/common/db/redisModel_test.go @@ -119,3 +119,9 @@ func Test_GetAccountCode(t *testing.T) { assert.Nil(t, err) fmt.Println(code) } +func Test_GetGroupMemberList(t *testing.T) { + groupID := "3791742301" + list, err := DB.GetGroupMemberIDListFromCache(groupID) + assert.Nil(t, err) + fmt.Println(list) +}