add monitoring and telegram alerts

Author: Mond1c

monitoring/alertmanager/alertmanager.yml

@@ -0,0 +1,98 @@
+global:
+  resolve_timeout: 5m
+
+templates:
+  - '/etc/alertmanager/*.tmpl'
+
+route:
+  receiver: 'default'
+  group_by: ['alertname', 'component']
+  group_wait: 10s
+  group_interval: 10s
+  repeat_interval: 12h
+  routes:
+    - match:
+        severity: critical
+      receiver: 'critical'
+      continue: true
+
+    - match:
+        severity: warning
+      receiver: 'warning'
+      continue: true
+
+    - match:
+        severity: info
+      receiver: 'info'
+
+receivers:
+  - name: 'default'
+    webhook_configs:
+      - url: 'http://localhost:5001/webhook'
+        send_resolved: true
+
+  - name: 'critical'
+    telegram_configs:
+      - bot_token: 'BOT_TOKEN'
+        chat_id: CHAT_ID
+        parse_mode: 'HTML'
+        message: |
+          🚨 <b>CRITICAL ALERT</b>
+
+          <b>Alert:</b> {{ .GroupLabels.alertname }}
+          <b>Severity:</b> {{ .CommonLabels.severity }}
+          <b>Component:</b> {{ .CommonLabels.component }}
+
+          {{ range .Alerts }}
+          <b>Description:</b> {{ .Annotations.description }}
+          <b>Status:</b> {{ .Status }}
+          {{ if .StartsAt }}<b>Started:</b> {{ .StartsAt.Format "2006-01-02 15:04:05" }}{{ end }}
+          {{ end }}
+        send_resolved: true
+
+  - name: 'warning'
+    telegram_configs:
+      - bot_token: 'BOT_TOKEN'
+        chat_id: CHAT_ID
+        parse_mode: 'HTML'
+        message: |
+          ⚠️ <b>WARNING ALERT</b>
+
+          <b>Alert:</b> {{ .GroupLabels.alertname }}
+          <b>Component:</b> {{ .CommonLabels.component }}
+
+          {{ range .Alerts }}
+          <b>Description:</b> {{ .Annotations.description }}
+          <b>Status:</b> {{ .Status }}
+          {{ if .StartsAt }}<b>Started:</b> {{ .StartsAt.Format "2006-01-02 15:04:05" }}{{ end }}
+          {{ end }}
+        send_resolved: true
+
+  - name: 'info'
+    telegram_configs:
+      - bot_token: 'BOT_TOKEN'
+        chat_id: CHAT_ID
+        parse_mode: 'HTML'
+        message: |
+          ℹ️ <b>INFO ALERT</b>
+
+          <b>Alert:</b> {{ .GroupLabels.alertname }}
+          <b>Component:</b> {{ .CommonLabels.component }}
+
+          {{ range .Alerts }}
+          <b>Description:</b> {{ .Annotations.description }}
+          {{ end }}
+        send_resolved: true
+
+inhibit_rules:
+  - source_match:
+      severity: 'critical'
+    target_match:
+      severity: 'warning'
+    equal: ['alertname', 'component']
+
+  - source_match:
+      severity: 'warning'
+    target_match:
+      severity: 'info'
+    equal: ['alertname', 'component']

monitoring/docker-compose.yml

@@ -0,0 +1,64 @@
+version: '3.8'
+
+services:
+  prometheus:
+    image: prom/prometheus:latest
+    container_name: webrtc-prometheus
+    volumes:
+      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
+      - ./prometheus/alerts.yml:/etc/prometheus/alerts.yml
+      - prometheus-data:/prometheus
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--storage.tsdb.path=/prometheus'
+      - '--web.console.libraries=/etc/prometheus/console_libraries'
+      - '--web.console.templates=/etc/prometheus/consoles'
+      - '--web.enable-lifecycle'
+    ports:
+      - "9090:9090"
+    restart: unless-stopped
+    networks:
+      - monitoring
+
+  grafana:
+    image: grafana/grafana:latest
+    container_name: webrtc-grafana
+    volumes:
+      - grafana-data:/var/lib/grafana
+      - ./grafana/provisioning:/etc/grafana/provisioning
+      - ./grafana/dashboards:/var/lib/grafana/dashboards
+    environment:
+      - GF_SECURITY_ADMIN_USER=admin
+      - GF_SECURITY_ADMIN_PASSWORD=admin
+      - GF_USERS_ALLOW_SIGN_UP=false
+    ports:
+      - "3000:3000"
+    restart: unless-stopped
+    networks:
+      - monitoring
+    depends_on:
+      - prometheus
+
+  alertmanager:
+    image: prom/alertmanager:latest
+    container_name: webrtc-alertmanager
+    volumes:
+      - ./alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml
+      - alertmanager-data:/alertmanager
+    command:
+      - '--config.file=/etc/alertmanager/alertmanager.yml'
+      - '--storage.path=/alertmanager'
+    ports:
+      - "9093:9093"
+    restart: unless-stopped
+    networks:
+      - monitoring
+
+networks:
+  monitoring:
+    driver: bridge
+
+volumes:
+  prometheus-data:
+  grafana-data:
+  alertmanager-data: