feat: add openim mongo and redis env

Signed-off-by: Xinwei Xiong (cubxxw) <3293172751nss@gmail.com>

config/templates/config.yaml.template

@@ -115,14 +115,14 @@ api:
 # minio.signEndpoint is minio public network address
 object:
   enable: "minio"
-  apiURL: "http://14.155.64.202:10002"
+  apiURL: "http://172.28.0.1:10002"
   minio:
     bucket: "openim"
     endpoint: "http://172.28.0.1:10005"
     accessKeyID: "root"
     secretAccessKey: "openIM123"
     sessionToken: ''
-    signEndpoint: "http://14.155.64.202:10005"
+    signEndpoint: "http://172.28.0.1:10005"
     publicRead: false
   cos:
     bucketURL: https://temp-1252357374.cos.ap-chengdu.myqcloud.com
@@ -186,7 +186,7 @@ rpcRegisterName:
 # Whether to output in json format
 # Whether to include stack trace in logs
 log:
-  storageLocation: /data/workspaces/open-im-server/logs/
+  storageLocation: ../logs/
   rotationTime: 24
   remainRotationCount: 2
   remainLogLevel: 6

config/templates/env.template

@@ -16,11 +16,11 @@ MINIO_ENDPOINT=http://172.28.0.1:10005
 
 # Base URL for the application programming interface (API).
 # Default: API_URL=http://172.28.0.1:10002
-API_URL=http://14.155.64.202:10002
+API_URL=http://172.28.0.1:10002
 
 # Directory path for storing data files or related information.
 # Default: DATA_DIR=./
-DATA_DIR=/data/workspaces/open-im-server
+DATA_DIR=./
 
 # Choose the appropriate image address, the default is GITHUB image, 
 # you can choose docker hub, for Chinese users can choose Ali Cloud

deployments/templates/openim.yaml

@@ -37,13 +37,20 @@ zookeeper:
 
 ###################### Mongo ######################
 # MongoDB configuration
-# If uri is not empty, it will be used directly
+
-#
+# If uri is not empty, it will be used directly for the MongoDB connection.
-# MongoDB address for standalone setup, Mongos address for sharded cluster setup
+# This is a complete MongoDB URI string.
-# Default MongoDB database name
+# Example: mongodb://user:password@host1:port1,host2:port2/dbname?options
-# Maximum connection pool size
 mongo:
   uri: ${MONGO_URI}
+
+# List of MongoDB server addresses.
+# Used for constructing the MongoDB URI if 'uri' above is empty.
+# For a standalone setup, specify the address of the single server.
+# For a sharded cluster, specify the addresses of the Mongos servers.
+# Example: [ '172.28.0.1:37017', '172.28.0.2:37017' ]
+# Default MongoDB database name
+# Maximum connection pool size
   address: [ ${MONGO_ADDRESS}:${MONGO_PORT} ]
   database: ${MONGO_DATABASE}
   username: ${MONGO_USERNAME}

pkg/common/db/cache/init_redis.go

@@ -18,6 +18,8 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"os"
+	"strings"
 	"time"
 
 	"github.com/redis/go-redis/v9"
@@ -43,6 +45,9 @@ func NewRedis() (redis.UniversalClient, error) {
 		return redisClient, nil
 	}
 
+	// Read configuration from environment variables
+	overrideConfigFromEnv()
+
 	if len(config.Config.Redis.Address) == 0 {
 		return nil, errors.New("redis address is empty")
 	}
@@ -60,7 +65,7 @@ func NewRedis() (redis.UniversalClient, error) {
 		rdb = redis.NewClient(&redis.Options{
 			Addr:       config.Config.Redis.Address[0],
 			Username:   config.Config.Redis.Username,
-			Password:   config.Config.Redis.Password, // no password set
+			Password:   config.Config.Redis.Password,
 			DB:         0,                            // use default DB
 			PoolSize:   100,                          // connection pool size
 			MaxRetries: maxRetry,
@@ -78,3 +83,16 @@ func NewRedis() (redis.UniversalClient, error) {
 	redisClient = rdb
 	return rdb, err
 }
+
+// overrideConfigFromEnv overrides configuration fields with environment variables if present.
+func overrideConfigFromEnv() {
+	if envAddr := os.Getenv("REDIS_ADDRESS"); envAddr != "" {
+		config.Config.Redis.Address = strings.Split(envAddr, ",") // Assuming addresses are comma-separated
+	}
+	if envUser := os.Getenv("REDIS_USERNAME"); envUser != "" {
+		config.Config.Redis.Username = envUser
+	}
+	if envPass := os.Getenv("REDIS_PASSWORD"); envPass != "" {
+		config.Config.Redis.Password = envPass
+	}
+}

pkg/common/db/unrelation/mongo.go

@@ -1,27 +1,13 @@
-// Copyright © 2023 OpenIM. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
 package unrelation
 
 import (
 	"context"
 	"fmt"
+	"os"
 	"strings"
 	"time"
 
 	"go.mongodb.org/mongo-driver/bson"
-
 	"go.mongodb.org/mongo-driver/mongo"
 	"go.mongodb.org/mongo-driver/mongo/options"
 
@@ -34,7 +20,8 @@ import (
 )
 
 const (
-	maxRetry = 10 // number of retries
+	maxRetry         = 10 // number of retries
+	mongoConnTimeout = 10 * time.Second
 )
 
 type Mongo struct {
@@ -44,90 +31,122 @@ type Mongo struct {
 // NewMongo Initialize MongoDB connection.
 func NewMongo() (*Mongo, error) {
 	specialerror.AddReplace(mongo.ErrNoDocuments, errs.ErrRecordNotFound)
-	uri := "mongodb://sample.host:27017/?maxPoolSize=20&w=majority"
+	uri := buildMongoURI()
-	if config.Config.Mongo.Uri != "" {
-		uri = config.Config.Mongo.Uri
-	} else {
-		mongodbHosts := ""
-		for i, v := range config.Config.Mongo.Address {
-			if i == len(config.Config.Mongo.Address)-1 {
-				mongodbHosts += v
-			} else {
-				mongodbHosts += v + ","
-			}
-		}
-		if config.Config.Mongo.Password != "" && config.Config.Mongo.Username != "" {
-			uri = fmt.Sprintf("mongodb://%s:%s@%s/%s?maxPoolSize=%d&authSource=admin",
-				config.Config.Mongo.Username, config.Config.Mongo.Password, mongodbHosts,
-				config.Config.Mongo.Database, config.Config.Mongo.MaxPoolSize)
-		} else {
-			uri = fmt.Sprintf("mongodb://%s/%s/?maxPoolSize=%d&authSource=admin",
-				mongodbHosts, config.Config.Mongo.Database,
-				config.Config.Mongo.MaxPoolSize)
-		}
-	}
 	fmt.Println("mongo:", uri)
+
 	var mongoClient *mongo.Client
-	var err error = nil
+	var err error
+
 	for i := 0; i <= maxRetry; i++ {
-		ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
+		ctx, cancel := context.WithTimeout(context.Background(), mongoConnTimeout)
 		defer cancel()
 		mongoClient, err = mongo.Connect(ctx, options.Client().ApplyURI(uri))
 		if err == nil {
 			return &Mongo{db: mongoClient}, nil
 		}
-		if cmdErr, ok := err.(mongo.CommandError); ok {
+		if shouldRetry(err) {
-			if cmdErr.Code == 13 || cmdErr.Code == 18 {
+			fmt.Printf("Failed to connect to MongoDB, retrying: %s\n", err)
-				return nil, err
+			time.Sleep(time.Second) // exponential backoff could be implemented here
-			} else {
+			continue
-				fmt.Printf("Failed to connect to MongoDB: %s\n", err)
-			}
 		}
+		return nil, err
 	}
 	return nil, err
 }
 
+func buildMongoURI() string {
+	uri := os.Getenv("MONGO_URI")
+	if uri != "" {
+		return uri
+	}
+
+	username := os.Getenv("MONGO_USERNAME")
+	password := os.Getenv("MONGO_PASSWORD")
+	address := os.Getenv("MONGO_ADDRESS")
+	port := os.Getenv("MONGO_PORT")
+	database := os.Getenv("MONGO_DATABASE")
+	maxPoolSize := os.Getenv("MONGO_MAX_POOL_SIZE")
+
+	if username == "" {
+		username = config.Config.Mongo.Username
+	}
+	if password == "" {
+		password = config.Config.Mongo.Password
+	}
+	if address == "" {
+		address = strings.Join(config.Config.Mongo.Address, ",")
+	} else if port != "" {
+		address = fmt.Sprintf("%s:%s", address, port)
+	}
+	if database == "" {
+		database = config.Config.Mongo.Database
+	}
+	if maxPoolSize == "" {
+		maxPoolSize = fmt.Sprint(config.Config.Mongo.MaxPoolSize)
+	}
+
+	uriFormat := "mongodb://%s/%s?maxPoolSize=%s&authSource=admin"
+	if username != "" && password != "" {
+		uriFormat = "mongodb://%s:%s@%s/%s?maxPoolSize=%s&authSource=admin"
+		return fmt.Sprintf(uriFormat, username, password, address, database, maxPoolSize)
+	}
+	return fmt.Sprintf(uriFormat, address, database, maxPoolSize)
+}
+
+func shouldRetry(err error) bool {
+	if cmdErr, ok := err.(mongo.CommandError); ok {
+		return cmdErr.Code != 13 && cmdErr.Code != 18
+	}
+	return true
+}
+
+// GetClient returns the MongoDB client.
 func (m *Mongo) GetClient() *mongo.Client {
 	return m.db
 }
 
+// GetDatabase returns the specific database from MongoDB.
 func (m *Mongo) GetDatabase() *mongo.Database {
 	return m.db.Database(config.Config.Mongo.Database)
 }
 
+// CreateMsgIndex creates an index for messages in MongoDB.
 func (m *Mongo) CreateMsgIndex() error {
 	return m.createMongoIndex(unrelation.Msg, true, "doc_id")
 }
 
+// createMongoIndex creates an index in a MongoDB collection.
 func (m *Mongo) createMongoIndex(collection string, isUnique bool, keys ...string) error {
-	db := m.db.Database(config.Config.Mongo.Database).Collection(collection)
+	db := m.GetDatabase().Collection(collection)
 	opts := options.CreateIndexes().SetMaxTime(10 * time.Second)
 	indexView := db.Indexes()
-	keysDoc := bson.D{}
+
-	// create composite indexes
+	keysDoc := buildIndexKeys(keys)
-	for _, key := range keys {
+
-		if strings.HasPrefix(key, "-") {
-			keysDoc = append(keysDoc, bson.E{Key: strings.TrimLeft(key, "-"), Value: -1})
-			// keysDoc = keysDoc.Append(strings.TrimLeft(key, "-"), bsonx.Int32(-1))
-		} else {
-			keysDoc = append(keysDoc, bson.E{Key: key, Value: 1})
-			// keysDoc = keysDoc.Append(key, bsonx.Int32(1))
-		}
-	}
-	// create index
 	index := mongo.IndexModel{
 		Keys: keysDoc,
 	}
 	if isUnique {
 		index.Options = options.Index().SetUnique(true)
 	}
-	result, err := indexView.CreateOne(
+
-		context.Background(),
+	_, err := indexView.CreateOne(context.Background(), index, opts)
-		index,
-		opts,
-	)
 	if err != nil {
-		return utils.Wrap(err, result)
+		return utils.Wrap(err, "CreateIndex")
 	}
 	return nil
 }
+
+// buildIndexKeys builds the BSON document for index keys.
+func buildIndexKeys(keys []string) bson.D {
+	keysDoc := bson.D{}
+	for _, key := range keys {
+		direction := 1 // default direction is ascending
+		if strings.HasPrefix(key, "-") {
+			direction = -1 // descending order for prefixed with "-"
+			key = strings.TrimLeft(key, "-")
+		}
+		keysDoc = append(keysDoc, bson.E{Key: key, Value: direction})
+	}
+	return keysDoc
+}

scripts/init-config.sh

@@ -88,8 +88,19 @@ generate_config_files() {
   done
 }
 
+declare -A env_vars=(
+    ["OPENIM_IP"]="172.28.0.1"
+    ["DATA_DIR"]="./"
+    ["LOG_STORAGE_LOCATION"]="../logs/"
+)
+
 # Function to generate example files
 generate_example_files() {
+  env_cmd="env -i"
+  for var in "${!env_vars[@]}"; do
+      env_cmd+=" $var='${env_vars[$var]}'"
+  done
+
   for template in "${!EXAMPLES[@]}"; do
     local example_file="${EXAMPLES[$template]}"
     if [[ -f "${example_file}" ]]; then
@@ -116,7 +127,7 @@ generate_example_files() {
       openim::log::error "genconfig.sh script not found"
       exit 1
     fi
-    "${OPENIM_ROOT}/scripts/genconfig.sh" "${ENV_FILE}" "${template}" > "${example_file}" || {
+    eval "$env_cmd ${OPENIM_ROOT}/scripts/genconfig.sh '${ENV_FILE}' '${template}' > '${example_file}'" || {
       openim::log::error "Error processing template file ${template}"
       exit 1
     }