From e937a6582f210801844cdad55cf334d23d9cf1e6 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Mon, 16 Jun 2025 17:16:16 +0800 Subject: [PATCH] feat: support redis sentinel. --- config/redis.yml | 8 +- docker-compose.yml | 208 ++++++++++++++++++++++++++++++++++-- go.mod | 2 +- go.sum | 4 +- pkg/common/config/config.go | 28 +++-- tools/seq/internal/seq.go | 12 ++- 6 files changed, 234 insertions(+), 28 deletions(-) diff --git a/config/redis.yml b/config/redis.yml index 2448bcb5c..c49129f80 100644 --- a/config/redis.yml +++ b/config/redis.yml @@ -1,7 +1,11 @@ -address: [ localhost:16379 ] -username: +address: [localhost:16379] +username: password: openIM123 clusterMode: false db: 0 maxRetry: 10 poolSize: 100 +sentinelMode: + enable: true + masterName: "redis-master" + sentinelsAddrs: ["127.0.0.1:26379", "127.0.0.1:26380", "127.0.0.1:26381"] diff --git a/docker-compose.yml b/docker-compose.yml index 65b4e6625..497fcdd86 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -64,6 +64,193 @@ services: sysctls: net.core.somaxconn: 1024 command: redis-server /usr/local/redis/config/redis.conf --requirepass openIM123 --appendonly yes + healthcheck: + test: ["CMD", "redis-cli", "-a", "openIM123", "ping"] + interval: 10s + timeout: 3s + retries: 3 + start_period: 30s + networks: + - openim + + redis-slave-1: + image: "${REDIS_IMAGE}" + container_name: redis-slave-1 + ports: + - "16380:6379" + volumes: + - "${DATA_DIR}/components/redis/slave1:/data" + environment: + TZ: Asia/Shanghai + REDIS_MASTER_HOST: redis + REDIS_MASTER_PORT: 6379 + REDIS_MASTER_PASSWORD: openIM123 + REDIS_SLAVE_PASSWORD: openIM123 + restart: always + command: > + bash -c ' + echo "port 6379" > /data/redis-slave.conf && + echo "replicaof $${REDIS_MASTER_HOST} $${REDIS_MASTER_PORT}" >> /data/redis-slave.conf && + echo "masterauth $${REDIS_MASTER_PASSWORD}" >> /data/redis-slave.conf && + echo "requirepass $${REDIS_SLAVE_PASSWORD}" >> /data/redis-slave.conf && + echo "appendonly yes" >> /data/redis-slave.conf && + echo "replica-read-only yes" >> /data/redis-slave.conf && + redis-server /data/redis-slave.conf + ' + depends_on: + - redis + networks: + - openim + + redis-slave-2: + image: "${REDIS_IMAGE}" + container_name: redis-slave-2 + ports: + - "16381:6379" + volumes: + - "${DATA_DIR}/components/redis/slave2:/data" + environment: + TZ: Asia/Shanghai + REDIS_MASTER_HOST: redis + REDIS_MASTER_PORT: 6379 + REDIS_MASTER_PASSWORD: openIM123 + REDIS_SLAVE_PASSWORD: openIM123 + restart: always + command: > + bash -c ' + echo "port 6379" > /data/redis-slave.conf && + echo "replicaof $${REDIS_MASTER_HOST} $${REDIS_MASTER_PORT}" >> /data/redis-slave.conf && + echo "masterauth $${REDIS_MASTER_PASSWORD}" >> /data/redis-slave.conf && + echo "requirepass $${REDIS_SLAVE_PASSWORD}" >> /data/redis-slave.conf && + echo "appendonly yes" >> /data/redis-slave.conf && + echo "replica-read-only yes" >> /data/redis-slave.conf && + redis-server /data/redis-slave.conf + ' + depends_on: + - redis + networks: + - openim + + redis-sentinel-1: + image: "${REDIS_IMAGE}" + container_name: redis-sentinel-1 + ports: + - "26379:26379" + volumes: + - "${DATA_DIR}/components/redis/sentinel1:/data" + environment: + TZ: Asia/Shanghai + SENTINEL_PORT: 26379 + REDIS_MASTER_NAME: redis-master + REDIS_MASTER_HOST: 127.0.0.1 + REDIS_MASTER_PORT: 16379 + REDIS_MASTER_PASSWORD: openIM123 + SENTINEL_QUORUM: 2 + SENTINEL_DOWN_AFTER: 30000 + SENTINEL_PARALLEL_SYNCS: 1 + SENTINEL_FAILOVER_TIMEOUT: 180000 + restart: always + command: > + bash -c ' + echo "port $${SENTINEL_PORT}" > /data/sentinel.conf && + echo "sentinel monitor $${REDIS_MASTER_NAME} $${REDIS_MASTER_HOST} $${REDIS_MASTER_PORT} $${SENTINEL_QUORUM}" >> /data/sentinel.conf && + echo "sentinel auth-pass $${REDIS_MASTER_NAME} $${REDIS_MASTER_PASSWORD}" >> /data/sentinel.conf && + echo "sentinel down-after-milliseconds $${REDIS_MASTER_NAME} $${SENTINEL_DOWN_AFTER}" >> /data/sentinel.conf && + echo "sentinel parallel-syncs $${REDIS_MASTER_NAME} $${SENTINEL_PARALLEL_SYNCS}" >> /data/sentinel.conf && + echo "sentinel failover-timeout $${REDIS_MASTER_NAME} $${SENTINEL_FAILOVER_TIMEOUT}" >> /data/sentinel.conf && + echo "sentinel resolve-hostnames yes" >> /data/sentinel.conf && + echo "sentinel announce-hostnames yes" >> /data/sentinel.conf && + redis-sentinel /data/sentinel.conf + ' + depends_on: + redis: + condition: service_healthy + redis-slave-1: + condition: service_started + redis-slave-2: + condition: service_started + networks: + - openim + + redis-sentinel-2: + image: "${REDIS_IMAGE}" + container_name: redis-sentinel-2 + ports: + - "26380:26379" + volumes: + - "${DATA_DIR}/components/redis/sentinel2:/data" + environment: + TZ: Asia/Shanghai + SENTINEL_PORT: 26379 + REDIS_MASTER_NAME: redis-master + REDIS_MASTER_HOST: 127.0.0.1 + REDIS_MASTER_PORT: 16379 + REDIS_MASTER_PASSWORD: openIM123 + SENTINEL_QUORUM: 2 + SENTINEL_DOWN_AFTER: 30000 + SENTINEL_PARALLEL_SYNCS: 1 + SENTINEL_FAILOVER_TIMEOUT: 180000 + restart: always + command: > + bash -c ' + echo "port $${SENTINEL_PORT}" > /data/sentinel.conf && + echo "sentinel monitor $${REDIS_MASTER_NAME} $${REDIS_MASTER_HOST} $${REDIS_MASTER_PORT} $${SENTINEL_QUORUM}" >> /data/sentinel.conf && + echo "sentinel auth-pass $${REDIS_MASTER_NAME} $${REDIS_MASTER_PASSWORD}" >> /data/sentinel.conf && + echo "sentinel down-after-milliseconds $${REDIS_MASTER_NAME} $${SENTINEL_DOWN_AFTER}" >> /data/sentinel.conf && + echo "sentinel parallel-syncs $${REDIS_MASTER_NAME} $${SENTINEL_PARALLEL_SYNCS}" >> /data/sentinel.conf && + echo "sentinel failover-timeout $${REDIS_MASTER_NAME} $${SENTINEL_FAILOVER_TIMEOUT}" >> /data/sentinel.conf && + echo "sentinel resolve-hostnames yes" >> /data/sentinel.conf && + echo "sentinel announce-hostnames yes" >> /data/sentinel.conf && + redis-sentinel /data/sentinel.conf + ' + depends_on: + redis: + condition: service_healthy + redis-slave-1: + condition: service_started + redis-slave-2: + condition: service_started + networks: + - openim + + redis-sentinel-3: + image: "${REDIS_IMAGE}" + container_name: redis-sentinel-3 + ports: + - "26381:26379" + volumes: + - "${DATA_DIR}/components/redis/sentinel3:/data" + environment: + TZ: Asia/Shanghai + SENTINEL_PORT: 26379 + REDIS_MASTER_NAME: redis-master + REDIS_MASTER_HOST: 127.0.0.1 + REDIS_MASTER_PORT: 16379 + REDIS_MASTER_PASSWORD: openIM123 + SENTINEL_QUORUM: 2 + SENTINEL_DOWN_AFTER: 30000 + SENTINEL_PARALLEL_SYNCS: 1 + SENTINEL_FAILOVER_TIMEOUT: 180000 + restart: always + command: > + bash -c ' + echo "port $${SENTINEL_PORT}" > /data/sentinel.conf && + echo "sentinel monitor $${REDIS_MASTER_NAME} $${REDIS_MASTER_HOST} $${REDIS_MASTER_PORT} $${SENTINEL_QUORUM}" >> /data/sentinel.conf && + echo "sentinel auth-pass $${REDIS_MASTER_NAME} $${REDIS_MASTER_PASSWORD}" >> /data/sentinel.conf && + echo "sentinel down-after-milliseconds $${REDIS_MASTER_NAME} $${SENTINEL_DOWN_AFTER}" >> /data/sentinel.conf && + echo "sentinel parallel-syncs $${REDIS_MASTER_NAME} $${SENTINEL_PARALLEL_SYNCS}" >> /data/sentinel.conf && + echo "sentinel failover-timeout $${REDIS_MASTER_NAME} $${SENTINEL_FAILOVER_TIMEOUT}" >> /data/sentinel.conf && + echo "sentinel resolve-hostnames yes" >> /data/sentinel.conf && + echo "sentinel announce-hostnames yes" >> /data/sentinel.conf && + redis-sentinel /data/sentinel.conf + ' + depends_on: + redis: + condition: service_healthy + redis-slave-1: + condition: service_started + redis-slave-2: + condition: service_started networks: - openim @@ -148,7 +335,7 @@ services: - "11002:80" networks: - openim - + prometheus: image: ${PROMETHEUS_IMAGE} container_name: prometheus @@ -161,9 +348,9 @@ services: - ./config/instance-down-rules.yml:/etc/prometheus/instance-down-rules.yml - ${DATA_DIR}/components/prometheus/data:/prometheus command: - - '--config.file=/etc/prometheus/prometheus.yml' - - '--storage.tsdb.path=/prometheus' - - '--web.listen-address=:${PROMETHEUS_PORT}' + - "--config.file=/etc/prometheus/prometheus.yml" + - "--storage.tsdb.path=/prometheus" + - "--web.listen-address=:${PROMETHEUS_PORT}" network_mode: host alertmanager: @@ -176,8 +363,8 @@ services: - ./config/alertmanager.yml:/etc/alertmanager/alertmanager.yml - ./config/email.tmpl:/etc/alertmanager/email.tmpl command: - - '--config.file=/etc/alertmanager/alertmanager.yml' - - '--web.listen-address=:${ALERTMANAGER_PORT}' + - "--config.file=/etc/alertmanager/alertmanager.yml" + - "--web.listen-address=:${ALERTMANAGER_PORT}" network_mode: host grafana: @@ -209,9 +396,8 @@ services: - /sys:/host/sys:ro - /:/rootfs:ro command: - - '--path.procfs=/host/proc' - - '--path.sysfs=/host/sys' - - '--path.rootfs=/rootfs' - - '--web.listen-address=:19100' + - "--path.procfs=/host/proc" + - "--path.sysfs=/host/sys" + - "--path.rootfs=/rootfs" + - "--web.listen-address=:19100" network_mode: host - diff --git a/go.mod b/go.mod index 845f75bb2..6b85eb89a 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.73-alpha.12 - github.com/openimsdk/tools v0.0.50-alpha.85 + github.com/openimsdk/tools v0.0.50-alpha.87 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index b775a1056..f179f7ecf 100644 --- a/go.sum +++ b/go.sum @@ -349,8 +349,8 @@ github.com/openimsdk/gomake v0.0.15-alpha.5 h1:eEZCEHm+NsmcO3onXZPIUbGFCYPYbsX5b github.com/openimsdk/gomake v0.0.15-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.73-alpha.12 h1:2NYawXeHChYUeSme6QJ9pOLh+Empce2WmwEtbP4JvKk= github.com/openimsdk/protocol v0.0.73-alpha.12/go.mod h1:WF7EuE55vQvpyUAzDXcqg+B+446xQyEba0X35lTINmw= -github.com/openimsdk/tools v0.0.50-alpha.85 h1:OqTUYx6r7Zp/eH8FKB08XeNjPV405TUIG9QT6QQ+F+s= -github.com/openimsdk/tools v0.0.50-alpha.85/go.mod h1:n2poR3asX1e1XZce4O+MOWAp+X02QJRFvhcLCXZdzRo= +github.com/openimsdk/tools v0.0.50-alpha.87 h1:42jaKZZ5874o+1qup9KKHYmCQJWO7ibOKppxr8rhIS0= +github.com/openimsdk/tools v0.0.50-alpha.87/go.mod h1:n2poR3asX1e1XZce4O+MOWAp+X02QJRFvhcLCXZdzRo= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index cd57a11cf..9d9f500a1 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -323,14 +323,21 @@ type RPC struct { } type Redis struct { - Disable bool `yaml:"-"` - Address []string `yaml:"address"` - Username string `yaml:"username"` - Password string `yaml:"password"` - ClusterMode bool `yaml:"clusterMode"` - DB int `yaml:"db"` - MaxRetry int `yaml:"maxRetry"` - PoolSize int `yaml:"poolSize"` + Disable bool `yaml:"-"` + Address []string `yaml:"address"` + Username string `yaml:"username"` + Password string `yaml:"password"` + ClusterMode bool `yaml:"clusterMode"` + DB int `yaml:"db"` + MaxRetry int `yaml:"maxRetry"` + PoolSize int `yaml:"poolSize"` + SentinelMode Sentinel `yaml:"sentinelMode"` +} + +type Sentinel struct { + Enable bool `yaml:"enable"` + MasterName string `yaml:"masterName"` + SentinelAddrs []string `yaml:"sentinelsAddrs"` } type BeforeConfig struct { @@ -494,6 +501,11 @@ func (r *Redis) Build() *redisutil.Config { DB: r.DB, MaxRetry: r.MaxRetry, PoolSize: r.PoolSize, + Sentinel: &redisutil.Sentinel{ + Enable: r.SentinelMode.Enable, + MasterName: r.SentinelMode.MasterName, + SentinelAddrs: r.SentinelMode.SentinelAddrs, + }, } } diff --git a/tools/seq/internal/seq.go b/tools/seq/internal/seq.go index 9fd352a96..c78b1b644 100644 --- a/tools/seq/internal/seq.go +++ b/tools/seq/internal/seq.go @@ -28,6 +28,8 @@ import ( "go.mongodb.org/mongo-driver/mongo/options" ) +const StructTagName = "yaml" + const ( MaxSeq = "MAX_SEQ:" MinSeq = "MIN_SEQ:" @@ -54,13 +56,14 @@ func readConfig[T any](dir string, name string) (*T, error) { if err := v.ReadInConfig(); err != nil { return nil, err } - fn := func(config *mapstructure.DecoderConfig) { - config.TagName = "mapstructure" - } + var conf T - if err := v.Unmarshal(&conf, fn); err != nil { + if err := v.Unmarshal(&conf, func(config *mapstructure.DecoderConfig) { + config.TagName = StructTagName + }); err != nil { return nil, err } + return &conf, nil } @@ -69,6 +72,7 @@ func Main(conf string, del time.Duration) error { if err != nil { return err } + fmt.Println("[seq] redis config", redisConfig) mongodbConfig, err := readConfig[config.Mongo](conf, config.MongodbConfigFileName) if err != nil { return err