feature: add alertmanager function

config/alertmanager.yml

@@ -0,0 +1,25 @@
+global:
+  resolve_timeout: 5m
+  smtp_from: '18565885972@163.com'
+  smtp_smarthost: 'smtp.163.com:465'
+  smtp_auth_username: '18565885972@163.com'
+  smtp_auth_password: 'ZTASKEORBBNKWPRV'
+  smtp_require_tls: false
+  smtp_hello: 'xxx监控告警'
+
+templates:
+  - '/etc/alertmanager/email.tmpl'
+
+route:
+  group_by: ['alertname']
+  group_wait: 5s
+  group_interval: 5s
+  repeat_interval: 5m
+  receiver: 'email'
+receivers:
+  - name: 'email'
+    email_configs:
+      - to: '2393740379@qq.com'
+        html: '{{ template "email.to.html" . }}'
+        headers: { Subject: "[WARN]告警" }
+        send_resolved: true

config/email.tmpl

@@ -0,0 +1,13 @@
+{{ define "email.to.html" }}
+{{ range .Alerts }}
+
+告警程序: prometheus_alert<br>
+告警级别: {{ .Labels.severity }} 级<br>
+告警类型: {{ .Labels.alertname }}<br>
+故障主机: {{ .Labels.instance }}<br>
+故障服务: {{ .Labels.job }}<br>
+告警主题: {{ .Annotations.summary }}<br>
+触发时间: {{ .StartsAt.Format "2020-01-02 15:04:05"}} <br>
+
+{{ end }}
+{{ end }}

config/instanceDown_rules.yml

@@ -0,0 +1,11 @@
+groups:
+  - name: node_down
+    rules:
+      - alert: InstanceDown
+        expr: up == 0
+        for: 1m
+        labels:
+          user: test
+        annotations:
+          summary: "Instance {{ $labels.instance }} down"
+          description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 1 minutes."

deployments/templates/env_template.yaml

@@ -94,7 +94,7 @@ OPENIM_CHAT_NETWORK_ADDRESS=${OPENIM_CHAT_NETWORK_ADDRESS}
 # Address or hostname for the Prometheus network.
 # Default: PROMETHEUS_NETWORK_ADDRESS=172.28.0.11
 PROMETHEUS_NETWORK_ADDRESS=${PROMETHEUS_NETWORK_ADDRESS}
-
+  
 # Address or hostname for the Grafana network.
 # Default: GRAFANA_NETWORK_ADDRESS=172.28.0.12
 GRAFANA_NETWORK_ADDRESS=${GRAFANA_NETWORK_ADDRESS}
@@ -106,7 +106,10 @@ NODE_EXPORTER_NETWORK_ADDRESS=${NODE_EXPORTER_NETWORK_ADDRESS}
 # Address or hostname for the OpenIM admin network.
 # Default: OPENIM_ADMIN_NETWORK_ADDRESS=172.28.0.14
 OPENIM_ADMIN_FRONT_NETWORK_ADDRESS=${OPENIM_ADMIN_FRONT_NETWORK_ADDRESS}
-
+  
+# Address or hostname for the alertmanager network.
+# Default: ALERT_MANAGER_NETWORK_ADDRESS=172.28.0.15
+ALERT_MANAGER_NETWORK_ADDRESS=${ALERT_MANAGER_NETWORK_ADDRESS}
 # ===============================================
 # = Component Extension Configuration =
 # ===============================================
@@ -305,4 +308,8 @@ GRAFANA_PORT=${GRAFANA_PORT}
   
 # Port for the admin front.
 # Default: OPENIM_ADMIN_FRONT_PORT=11002
-OPENIM_ADMIN_FRONT_PORT=${OPENIM_ADMIN_FRONT_PORT}
+OPENIM_ADMIN_FRONT_PORT=${OPENIM_ADMIN_FRONT_PORT}
+
+# Port for the alertmanager.
+# Default: ALERT_MANAGER_PORT=19093
+ALERT_MANAGER_PORT=${ALERT_MANAGER_PORT}

deployments/templates/prometheus.yml

@@ -6,13 +6,13 @@ global:
 
 # Alertmanager configuration
 alerting:
-#alertmanagers:
-#  - static_configs:
-#      - targets: ['172.29.166.17:9093'] #alertmanager地址
+  alertmanagers:
+    - static_configs:
+        - targets: ['${ALERT_MANAGER_ADDRESS}:${ALERT_MANAGER_PORT}']
 
 # Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
 rule_files:
-#  - "node_down.yml"
+  - "instanceDown_rules.yml"
 # - "first_rules.yml"
 # - "second_rules.yml"
 

docker-compose.yml

@@ -162,12 +162,27 @@ services:
     restart: always
     volumes:
       - ./config/prometheus.yml:/etc/prometheus/prometheus.yml
+      - ./config/instanceDown_rules.yml:/etc/prometheus/instanceDown_rules.yml
     ports:
       - "${PROMETHEUS_PORT}:9090"
     networks:
       server:
         ipv4_address: ${PROMETHEUS_NETWORK_ADDRESS}
 
+  alertmanager:
+    image: prom/alertmanager
+    container_name: alertmanager
+    hostname: alertmanager
+    restart: always
+    volumes:
+      - ./config/alertmanager.yml:/etc/alertmanager/alertmanager.yml
+      - ./config/email.tmpl:/etc/alertmanager/email.tmpl
+    ports:
+      - "${ALERT_MANAGER_PORT}:9093"
+    networks:
+      server:
+        ipv4_address: ${ALERT_MANAGER_NETWORK_ADDRESS}
+
   grafana:
     image: grafana/grafana
     container_name: grafana

scripts/install/environment.sh

@@ -120,6 +120,8 @@ LAST_OCTET=$((LAST_OCTET + 1))
 NODE_EXPORTER_NETWORK_ADDRESS=$(generate_ip)
 LAST_OCTET=$((LAST_OCTET + 1))
 OPENIM_ADMIN_FRONT_NETWORK_ADDRESS=$(generate_ip)
+LAST_OCTET=$((LAST_OCTET + 1))
+ALERT_MANAGER_NETWORK_ADDRESS=$(generate_ip)
 ###################### openim 配置 ######################
 # read: https://github.com/openimsdk/open-im-server/blob/main/deployment/README.md
 def "OPENIM_DATA_DIR" "/data/openim"
@@ -259,6 +261,11 @@ def "PROMETHEUS_ADDRESS" "${DOCKER_BRIDGE_GATEWAY}" # Prometheus的地址
 ###################### node-exporter 配置 ######################
 def "NODE_EXPORTER_PORT" "19100"                       # node-exporter的端口
 def "NODE_EXPORTER_ADDRESS" "${DOCKER_BRIDGE_GATEWAY}" # node-exporter的地址
+
+###################### alertmanagerS 配置 ######################
+def "ALERT_MANAGER_PORT" "19093"                       # node-exporter的端口
+def "ALERT_MANAGER_ADDRESS" "${DOCKER_BRIDGE_GATEWAY}" # node-exporter的地址
+
 ###################### Grafana 配置信息 ######################
 def "GRAFANA_PORT" "3000"                        # Grafana的端口
 def "GRAFANA_ADDRESS" "${DOCKER_BRIDGE_GATEWAY}" # Grafana的地址