Merge branch 'add_rules' into 'master'

Add default prometheus rules



See merge request !2

Author: Paweł Krupa

README.md

@@ -37,6 +37,19 @@ All variables you can see [here](defaults/main.yml).
 
 This role doesn't have dependencies.
 
+## Defining alerting rules files
+
+Put the rules files to rules foleder
+
+Alerting rules are defined in the following syntax:
+```yaml
+ALERT <alert name>
+  IF <expression>
+  [ FOR <duration> ]
+  [ LABELS <label set> ]
+  [ ANNOTATIONS <label set> ]
+```
+
 ## Example Playbook
 ```yaml
 ---

defaults/main.yml

@@ -20,16 +20,16 @@ prometheus_bin_dir: "{{ prometheus_root_dir }}/current"
 prometheus_rules_dir: "{{ prometheus_config_dir }}/rules"
 prometheus_file_sd_config_dir: "{{ prometheus_config_dir }}/tgroups"
 
-prometheus_rules_src_dir: "{{ playbook_dir }}/rules"
-#prometheus_config_parts_src_dir: "{{ playbook_dir }}/../environments/main_server/files"
-#prometheus_tgroups_src_dir: "{{ playbook_dir }}/files/tgroups"
+# prometheus_rules_src_dir: "{{ playbook_dir }}/../files/rules"
+# prometheus_config_parts_src_dir: "{{ playbook_dir }}/../environments/main_server/files"
+# prometheus_tgroups_src_dir: "{{ playbook_dir }}/files/tgroups"
 
 prometheus_config_dir: /etc/prometheus
 prometheus_pid_path: /var/run/prometheus.pid
 prometheus_db_dir: /var/lib/prometheus
 # prometheus_db_dir: /home/soi/prometheus
 
-prometheus_rules_files: []
+prometheus_rules_files: [alert.rules]
 
 prometheus_web_listen_address: "0.0.0.0:9090"
 prometheus_web_external_url: 'http://localhost:9090/'

files/rules/alert.rules

@@ -0,0 +1,9 @@
+# Alert for any instance that is unreachable for >5 minutes.
+ALERT InstanceDown
+  IF up == 0
+  FOR 5m
+  LABELS { severity = "slack" }
+  ANNOTATIONS {
+    summary = "Instance {{ $labels.instance }} down",
+    description = "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes.",
+  }

tasks/configuration.yml

@@ -1,4 +1,41 @@
 ---
+# - name: register rules files exists on server
+#   shell: "ls -1 {{ prometheus_rules_dir }}"
+#   register: rules_content
+#   changed_when: False
+#   
+# - name: delete unmanaged rules files
+#   file:
+#     path: "{{ prometheus_rules_dir }}/{{ item }}"
+#     state: absent
+#   with_items: "{{ rules_content.stdout_lines | difference(prometheus_rules_files) }}"
+#   notify:
+#     - reload prometheus
+# 
+# - name: copy rule files from playbook directory
+#   copy:
+#     src: "{{ prometheus_rules_src_dir }}/{{ item }}"
+#     dest: "{{ prometheus_rules_dir }}/{{ item }}"
+#     owner: "{{ prometheus_user }}"
+#     group: "{{ prometheus_group }}"
+#     mode: 0644
+#     validate: "{{ prometheus_bin_dir }}/promtool check-rules %s"
+#   with_items:
+#     - "{{ prometheus_rules_files }}"
+#   notify:
+#     - reload prometheus
+- name: copy rule files to prometheus
+  copy:
+    src: "rules/"
+    dest: "{{ prometheus_rules_dir }}"
+    owner: "{{ prometheus_user }}"
+    group: "{{ prometheus_group }}"
+    mode: 0644
+    validate: "{{ prometheus_bin_dir }}/promtool check-rules %s"
+  notify:
+    - reload prometheus
+
+
 - name: configure prometheus
   tags:
     - config
@@ -13,3 +50,4 @@
   notify:
     - reload prometheus
 
+

templates/etc/prometheus/prometheus.yml.j2

@@ -14,12 +14,12 @@ global:
   {% endfor %}
 {% endif %}
 
-  # Load and evaluate rules in this file every 'evaluation_interval' seconds. This field may be repeated.
-{% if prometheus_rule_files is defined and prometheus_rule_files|count > 0 %}
+{% if prometheus_rules_files is defined %}
+# Rule files specifies a list of files from which rules are read.
 rule_files:
-  {% for file in prometheus_rule_files %}
-  - '{{ file }}'
-  {% endfor%}
+  {% for dest in prometheus_rules_files %}
+  - {{ prometheus_rules_dir }}/{{ dest }}
+  {% endfor %}
 {% endif %}
 
 scrape_configs: