From da7943cc649642c0a284f74a50f233d433ec933c Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Wed, 28 May 2025 17:38:22 +0800 Subject: [PATCH 1/5] feat: Implement etcd and kafka auth. --- .env | 2 +- config/discovery.yml | 8 ++++---- config/kafka.yml | 16 ++++++++-------- docker-compose.yml | 30 +++++++++++++++++++++++++++--- 4 files changed, 40 insertions(+), 16 deletions(-) diff --git a/.env b/.env index 0ab998037..2d4dfd4c7 100644 --- a/.env +++ b/.env @@ -2,7 +2,7 @@ MONGO_IMAGE=mongo:7.0 REDIS_IMAGE=redis:7.0.0 KAFKA_IMAGE=bitnami/kafka:3.5.1 MINIO_IMAGE=minio/minio:RELEASE.2024-01-11T07-46-16Z -ETCD_IMAGE=quay.io/coreos/etcd:v3.5.13 +ETCD_IMAGE=bitnami/etcd:3.5.13 PROMETHEUS_IMAGE=prom/prometheus:v2.45.6 ALERTMANAGER_IMAGE=prom/alertmanager:v0.27.0 GRAFANA_IMAGE=grafana/grafana:11.0.1 diff --git a/config/discovery.yml b/config/discovery.yml index e8d733e9f..f44162022 100644 --- a/config/discovery.yml +++ b/config/discovery.yml @@ -1,9 +1,9 @@ enable: etcd etcd: rootDirectory: openim - address: [ localhost:12379 ] - username: '' - password: '' + address: [localhost:12379] + username: "openIM" + password: "openIM123" kubernetes: namespace: default @@ -17,4 +17,4 @@ rpcService: group: group-rpc-service auth: auth-rpc-service conversation: conversation-rpc-service - third: third-rpc-service \ No newline at end of file + third: third-rpc-service diff --git a/config/kafka.yml b/config/kafka.yml index fd06ae2bb..54d4f065a 100644 --- a/config/kafka.yml +++ b/config/kafka.yml @@ -1,13 +1,13 @@ # Username for authentication -username: '' +username: "openIM" # Password for authentication -password: '' +password: "openIM123" # Producer acknowledgment settings -producerAck: +producerAck: # Compression type to use (e.g., none, gzip, snappy) compressType: none # List of Kafka broker addresses -address: [ localhost:19094 ] +address: [localhost:19094] # Kafka topic for Redis integration toRedisTopic: toRedis # Kafka topic for MongoDB integration @@ -29,12 +29,12 @@ tls: # Enable or disable TLS enableTLS: false # CA certificate file path - caCrt: + caCrt: # Client certificate file path - clientCrt: + clientCrt: # Client key file path - clientKey: + clientKey: # Client key password - clientKeyPwd: + clientKeyPwd: # Whether to skip TLS verification (not recommended for production) insecureSkipVerify: false diff --git a/docker-compose.yml b/docker-compose.yml index 65b4e6625..c433c6cf4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -75,7 +75,6 @@ services: - "12380:2380" environment: - ETCD_NAME=s1 - - ETCD_DATA_DIR=/etcd-data - ETCD_LISTEN_CLIENT_URLS=http://0.0.0.0:2379 - ETCD_ADVERTISE_CLIENT_URLS=http://0.0.0.0:2379 - ETCD_LISTEN_PEER_URLS=http://0.0.0.0:2380 @@ -83,8 +82,27 @@ services: - ETCD_INITIAL_CLUSTER=s1=http://0.0.0.0:2380 - ETCD_INITIAL_CLUSTER_TOKEN=tkn - ETCD_INITIAL_CLUSTER_STATE=new + - ALLOW_NONE_AUTHENTICATION=no + - ETCD_ROOT_USER=root + - ETCD_ROOT_PASSWORD=openIM123 + - ETCD_USERNAME=openIM + - ETCD_PASSWORD=openIM123 volumes: - - "${DATA_DIR}/components/etcd:/etcd-data" + - "${DATA_DIR}/components/etcd:/bitnami/etcd" + command: | + /bin/bash -c ' + /opt/bitnami/scripts/etcd/entrypoint.sh /opt/bitnami/scripts/etcd/run.sh & + + sleep 10 + + etcdctl --endpoints=http://127.0.0.1:2379 --user=$${ETCD_ROOT_USER}:$${ETCD_ROOT_PASSWORD} user add $${ETCD_USERNAME} --new-user-password=$${ETCD_PASSWORD} || true + etcdctl --endpoints=http://127.0.0.1:2379 --user=$${ETCD_ROOT_USER}:$${ETCD_ROOT_PASSWORD} role add openim-role || true + etcdctl --endpoints=http://127.0.0.1:2379 --user=$${ETCD_ROOT_USER}:$${ETCD_ROOT_PASSWORD} role grant-permission openim-role --prefix=true readwrite / || true + etcdctl --endpoints=http://127.0.0.1:2379 --user=$${ETCD_ROOT_USER}:$${ETCD_ROOT_PASSWORD} role grant-permission openim-role --prefix=true readwrite "" || true + etcdctl --endpoints=http://127.0.0.1:2379 --user=$${ETCD_ROOT_USER}:$${ETCD_ROOT_PASSWORD} user grant-role $${ETCD_USERNAME} openim-role || true + + tail -f /dev/null + ' restart: always networks: - openim @@ -106,10 +124,16 @@ services: KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 0@kafka:9093 KAFKA_CFG_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094 KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,EXTERNAL://localhost:19094 - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT + KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,EXTERNAL:SASL_PLAINTEXT,PLAINTEXT:SASL_PLAINTEXT KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER KAFKA_NUM_PARTITIONS: 8 KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: "true" + + KAFKA_CFG_SASL_ENABLED_MECHANISMS: PLAIN + KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN + KAFKA_CLIENT_USERS: admin,openIM + KAFKA_CLIENT_PASSWORDS: admin-secret,openIM123 + networks: - openim From 0c81a763235b0c13f69683813f02b699087ceafb Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Thu, 29 May 2025 15:56:40 +0800 Subject: [PATCH 2/5] Update etcd command contents. --- docker-compose.yml | 93 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 71 insertions(+), 22 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index c433c6cf4..d5b587a58 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -75,6 +75,7 @@ services: - "12380:2380" environment: - ETCD_NAME=s1 + - ETCD_DATA_DIR=/etcd-data - ETCD_LISTEN_CLIENT_URLS=http://0.0.0.0:2379 - ETCD_ADVERTISE_CLIENT_URLS=http://0.0.0.0:2379 - ETCD_LISTEN_PEER_URLS=http://0.0.0.0:2380 @@ -88,20 +89,69 @@ services: - ETCD_USERNAME=openIM - ETCD_PASSWORD=openIM123 volumes: - - "${DATA_DIR}/components/etcd:/bitnami/etcd" - command: | - /bin/bash -c ' - /opt/bitnami/scripts/etcd/entrypoint.sh /opt/bitnami/scripts/etcd/run.sh & + - "${DATA_DIR}/components/etcd1:/etcd-data" + command: > + /bin/sh -c ' + etcd & + export ETCDCTL_API=3 - sleep 10 + echo "Waiting for etcd to become healthy..." + until etcdctl --endpoints=http://127.0.0.1:2379 endpoint health &>/dev/null; do + echo "Waiting for ETCD to start..." + sleep 1 + done - etcdctl --endpoints=http://127.0.0.1:2379 --user=$${ETCD_ROOT_USER}:$${ETCD_ROOT_PASSWORD} user add $${ETCD_USERNAME} --new-user-password=$${ETCD_PASSWORD} || true - etcdctl --endpoints=http://127.0.0.1:2379 --user=$${ETCD_ROOT_USER}:$${ETCD_ROOT_PASSWORD} role add openim-role || true - etcdctl --endpoints=http://127.0.0.1:2379 --user=$${ETCD_ROOT_USER}:$${ETCD_ROOT_PASSWORD} role grant-permission openim-role --prefix=true readwrite / || true - etcdctl --endpoints=http://127.0.0.1:2379 --user=$${ETCD_ROOT_USER}:$${ETCD_ROOT_PASSWORD} role grant-permission openim-role --prefix=true readwrite "" || true - etcdctl --endpoints=http://127.0.0.1:2379 --user=$${ETCD_ROOT_USER}:$${ETCD_ROOT_PASSWORD} user grant-role $${ETCD_USERNAME} openim-role || true + echo "etcd is healthy." - tail -f /dev/null + echo "Checking authentication status..." + if ! etcdctl --endpoints=http://127.0.0.1:2379 auth status | grep -q "Authentication Status: true"; then + echo "Authentication is disabled. Creating users and enabling..." + + # Create users and setup permissions + etcdctl --endpoints=http://127.0.0.1:2379 user add $${ETCD_ROOT_USER} --new-user-password=$${ETCD_ROOT_PASSWORD} || true + etcdctl --endpoints=http://127.0.0.1:2379 user add $${ETCD_USERNAME} --new-user-password=$${ETCD_PASSWORD} || true + + etcdctl --endpoints=http://127.0.0.1:2379 role add openim-role || true + etcdctl --endpoints=http://127.0.0.1:2379 role grant-permission openim-role --prefix=true readwrite / || true + etcdctl --endpoints=http://127.0.0.1:2379 role grant-permission openim-role --prefix=true readwrite "" || true + etcdctl --endpoints=http://127.0.0.1:2379 user grant-role $${ETCD_USERNAME} openim-role || true + + etcdctl --endpoints=http://127.0.0.1:2379 user grant-role $${ETCD_ROOT_USER} $${ETCD_USERNAME} root || true + + echo "Enabling authentication..." + etcdctl --endpoints=http://127.0.0.1:2379 auth enable + echo "Authentication enabled successfully" + else + echo "Authentication is already enabled. Checking OpenIM user..." + + # Check if openIM user exists and can perform operations + if ! etcdctl --endpoints=http://127.0.0.1:2379 --user=$${ETCD_USERNAME}:$${ETCD_PASSWORD} put /test/auth "auth-check" &>/dev/null; then + echo "OpenIM user test failed. Recreating user with root credentials..." + + # Try to create/update the openIM user using root credentials + etcdctl --endpoints=http://127.0.0.1:2379 --user=$${ETCD_ROOT_USER}:$${ETCD_ROOT_PASSWORD} user add $${ETCD_USERNAME} --new-user-password=$${ETCD_PASSWORD} --no-password-file || true + etcdctl --endpoints=http://127.0.0.1:2379 --user=$${ETCD_ROOT_USER}:$${ETCD_ROOT_PASSWORD} role add openim-role || true + etcdctl --endpoints=http://127.0.0.1:2379 --user=$${ETCD_ROOT_USER}:$${ETCD_ROOT_PASSWORD} role grant-permission openim-role --prefix=true readwrite / || true + etcdctl --endpoints=http://127.0.0.1:2379 --user=$${ETCD_ROOT_USER}:$${ETCD_ROOT_PASSWORD} role grant-permission openim-role --prefix=true readwrite "" || true + etcdctl --endpoints=http://127.0.0.1:2379 --user=$${ETCD_ROOT_USER}:$${ETCD_ROOT_PASSWORD} user grant-role $${ETCD_USERNAME} openim-role || true + etcdctl --endpoints=http://127.0.0.1:2379 user grant-role $${ETCD_ROOT_USER} $${ETCD_USERNAME} root || true + + echo "OpenIM user recreated with required permissions" + else + echo "OpenIM user exists and has correct permissions" + etcdctl --endpoints=http://127.0.0.1:2379 --user=$${ETCD_USERNAME}:$${ETCD_PASSWORD} del /test/auth &>/dev/null + fi + fi + + echo "Testing authentication with OpenIM user..." + if etcdctl --endpoints=http://127.0.0.1:2379 --user=$${ETCD_USERNAME}:$${ETCD_PASSWORD} put /test/auth "auth-works"; then + echo "Authentication working properly" + etcdctl --endpoints=http://127.0.0.1:2379 --user=$${ETCD_USERNAME}:$${ETCD_PASSWORD} del /test/auth + else + echo "WARNING: Authentication test failed" + fi + + tail -f /dev/null ' restart: always networks: @@ -172,7 +222,7 @@ services: - "11002:80" networks: - openim - + prometheus: image: ${PROMETHEUS_IMAGE} container_name: prometheus @@ -185,9 +235,9 @@ services: - ./config/instance-down-rules.yml:/etc/prometheus/instance-down-rules.yml - ${DATA_DIR}/components/prometheus/data:/prometheus command: - - '--config.file=/etc/prometheus/prometheus.yml' - - '--storage.tsdb.path=/prometheus' - - '--web.listen-address=:${PROMETHEUS_PORT}' + - "--config.file=/etc/prometheus/prometheus.yml" + - "--storage.tsdb.path=/prometheus" + - "--web.listen-address=:${PROMETHEUS_PORT}" network_mode: host alertmanager: @@ -200,8 +250,8 @@ services: - ./config/alertmanager.yml:/etc/alertmanager/alertmanager.yml - ./config/email.tmpl:/etc/alertmanager/email.tmpl command: - - '--config.file=/etc/alertmanager/alertmanager.yml' - - '--web.listen-address=:${ALERTMANAGER_PORT}' + - "--config.file=/etc/alertmanager/alertmanager.yml" + - "--web.listen-address=:${ALERTMANAGER_PORT}" network_mode: host grafana: @@ -233,9 +283,8 @@ services: - /sys:/host/sys:ro - /:/rootfs:ro command: - - '--path.procfs=/host/proc' - - '--path.sysfs=/host/sys' - - '--path.rootfs=/rootfs' - - '--web.listen-address=:19100' + - "--path.procfs=/host/proc" + - "--path.sysfs=/host/sys" + - "--path.rootfs=/rootfs" + - "--web.listen-address=:19100" network_mode: host - From 5a3dae738e3a09c0d2ab8e09fc59fa927c12b286 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Thu, 29 May 2025 17:00:53 +0800 Subject: [PATCH 3/5] update contents. --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index d5b587a58..0fcdda873 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -89,7 +89,7 @@ services: - ETCD_USERNAME=openIM - ETCD_PASSWORD=openIM123 volumes: - - "${DATA_DIR}/components/etcd1:/etcd-data" + - "${DATA_DIR}/components/etcd:/etcd-data" command: > /bin/sh -c ' etcd & From fa31264da81b3e007a54b9667401fbc30feee331 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Thu, 12 Jun 2025 16:02:48 +0800 Subject: [PATCH 4/5] feat: update auth logic to compatible old version. --- config/discovery.yml | 4 ++-- config/kafka.yml | 8 +++---- docker-compose.yml | 52 +++++++++++++++++++++++++++++++++----------- 3 files changed, 45 insertions(+), 19 deletions(-) diff --git a/config/discovery.yml b/config/discovery.yml index f44162022..340407023 100644 --- a/config/discovery.yml +++ b/config/discovery.yml @@ -2,8 +2,8 @@ enable: etcd etcd: rootDirectory: openim address: [localhost:12379] - username: "openIM" - password: "openIM123" + # username: "openIM" + # password: "openIM123" kubernetes: namespace: default diff --git a/config/kafka.yml b/config/kafka.yml index 54d4f065a..a3d4abc34 100644 --- a/config/kafka.yml +++ b/config/kafka.yml @@ -1,7 +1,7 @@ -# Username for authentication -username: "openIM" -# Password for authentication -password: "openIM123" +## Kafka authentication +# username: "openIM" +# password: "openIM123" + # Producer acknowledgment settings producerAck: # Compression type to use (e.g., none, gzip, snappy) diff --git a/docker-compose.yml b/docker-compose.yml index 0fcdda873..e7d42dbed 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -84,17 +84,17 @@ services: - ETCD_INITIAL_CLUSTER_TOKEN=tkn - ETCD_INITIAL_CLUSTER_STATE=new - ALLOW_NONE_AUTHENTICATION=no - - ETCD_ROOT_USER=root - - ETCD_ROOT_PASSWORD=openIM123 - - ETCD_USERNAME=openIM - - ETCD_PASSWORD=openIM123 + ## Optional: Enable etcd authentication by setting the following credentials + # - ETCD_ROOT_USER=root + # - ETCD_ROOT_PASSWORD=openIM123 + # - ETCD_USERNAME=openIM + # - ETCD_PASSWORD=openIM123 volumes: - "${DATA_DIR}/components/etcd:/etcd-data" command: > /bin/sh -c ' etcd & export ETCDCTL_API=3 - echo "Waiting for etcd to become healthy..." until etcdctl --endpoints=http://127.0.0.1:2379 endpoint health &>/dev/null; do echo "Waiting for ETCD to start..." @@ -103,6 +103,9 @@ services: echo "etcd is healthy." + if [ -n "$${ETCD_ROOT_USER}" ] && [ -n "$${ETCD_ROOT_PASSWORD}" ] && [ -n "$${ETCD_USERNAME}" ] && [ -n "$${ETCD_PASSWORD}" ]; then + echo "Authentication credentials provided. Setting up authentication..." + echo "Checking authentication status..." if ! etcdctl --endpoints=http://127.0.0.1:2379 auth status | grep -q "Authentication Status: true"; then echo "Authentication is disabled. Creating users and enabling..." @@ -142,13 +145,16 @@ services: etcdctl --endpoints=http://127.0.0.1:2379 --user=$${ETCD_USERNAME}:$${ETCD_PASSWORD} del /test/auth &>/dev/null fi fi - echo "Testing authentication with OpenIM user..." if etcdctl --endpoints=http://127.0.0.1:2379 --user=$${ETCD_USERNAME}:$${ETCD_PASSWORD} put /test/auth "auth-works"; then echo "Authentication working properly" etcdctl --endpoints=http://127.0.0.1:2379 --user=$${ETCD_USERNAME}:$${ETCD_PASSWORD} del /test/auth else echo "WARNING: Authentication test failed" + fi + else + echo "No authentication credentials provided. Running in no-auth mode." + echo "To enable authentication, set ETCD_ROOT_USER, ETCD_ROOT_PASSWORD, ETCD_USERNAME, and ETCD_PASSWORD environment variables." fi tail -f /dev/null @@ -172,18 +178,38 @@ services: KAFKA_CFG_NODE_ID: 0 KAFKA_CFG_PROCESS_ROLES: controller,broker KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 0@kafka:9093 - KAFKA_CFG_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094 - KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,EXTERNAL://localhost:19094 - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,EXTERNAL:SASL_PLAINTEXT,PLAINTEXT:SASL_PLAINTEXT KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER KAFKA_NUM_PARTITIONS: 8 KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: "true" - KAFKA_CFG_SASL_ENABLED_MECHANISMS: PLAIN - KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN - KAFKA_CLIENT_USERS: admin,openIM - KAFKA_CLIENT_PASSWORDS: admin-secret,openIM123 + KAFKA_CFG_LISTENERS: "PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094" + KAFKA_CFG_ADVERTISED_LISTENERS: "PLAINTEXT://kafka:9092,EXTERNAL://localhost:19094" + KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: "CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT" + KAFKA_CFG_INTER_BROKER_LISTENER_NAME: "PLAINTEXT" + # Authentication configuration variables - comment out to disable auth + # KAFKA_USERNAME: "openIM" + # KAFKA_PASSWORD: "openIM123" + command: > + /bin/sh -c ' + if [ -n "$${KAFKA_USERNAME}" ] && [ -n "$${KAFKA_PASSWORD}" ]; then + echo "=== Kafka SASL Authentication ENABLED ===" + echo "Username: $${KAFKA_USERNAME}" + + # Set environment variables for SASL authentication + export KAFKA_CFG_LISTENERS="SASL_PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094" + export KAFKA_CFG_ADVERTISED_LISTENERS="SASL_PLAINTEXT://kafka:9092,EXTERNAL://localhost:19094" + export KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP="CONTROLLER:PLAINTEXT,EXTERNAL:SASL_PLAINTEXT,SASL_PLAINTEXT:SASL_PLAINTEXT" + export KAFKA_CFG_SASL_ENABLED_MECHANISMS="PLAIN" + export KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL="PLAIN" + export KAFKA_CFG_INTER_BROKER_LISTENER_NAME="SASL_PLAINTEXT" + export KAFKA_CLIENT_USERS="$${KAFKA_USERNAME}" + export KAFKA_CLIENT_PASSWORDS="$${KAFKA_PASSWORD}" + fi + + # Start Kafka with the configured environment + exec /opt/bitnami/scripts/kafka/entrypoint.sh /opt/bitnami/scripts/kafka/run.sh + ' networks: - openim From 187a87f749d5eb5258221dc95e2d3b8347a653f7 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Thu, 12 Jun 2025 16:08:22 +0800 Subject: [PATCH 5/5] update comment. --- config/discovery.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/discovery.yml b/config/discovery.yml index 340407023..169bb4447 100644 --- a/config/discovery.yml +++ b/config/discovery.yml @@ -2,6 +2,8 @@ enable: etcd etcd: rootDirectory: openim address: [localhost:12379] + ## Attention: If you set auth in etcd + ## you must also update the username and password in Chat project. # username: "openIM" # password: "openIM123"