From cd59ad3951fa76ecd2a36deb55c3774dd9b39854 Mon Sep 17 00:00:00 2001 From: arcsector <26469747+arcsector@users.noreply.github.com> Date: Thu, 1 Jun 2023 11:36:05 -0700 Subject: [PATCH 1/4] Change log level in cfg and CLI --- roles/splunk/tasks/adhoc_change_log_level.yml | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 roles/splunk/tasks/adhoc_change_log_level.yml diff --git a/roles/splunk/tasks/adhoc_change_log_level.yml b/roles/splunk/tasks/adhoc_change_log_level.yml new file mode 100644 index 00000000..1d35ad27 --- /dev/null +++ b/roles/splunk/tasks/adhoc_change_log_level.yml @@ -0,0 +1,44 @@ +--- +- name: Change log level of {{ splunk_component }} to {{ splunk_component_log_level }} via CLI + ansible.builtin.command: | + {{ splunk_home }}/bin/splunk set log-level {{ splunk_component }} -level {{ splunk_component_log_level }} -auth '{{ splunk_auth }}' + register: log_level_chg_out + changed_when: log_level_chg_out.rc == 0 + failed_when: log_level_chg_out.rc != 0 + become: true + become_user: "{{ splunk_nix_user }}" + when: + - splunk_component is defined + - splunk_component_log_level is defined + +- name: Set log and log-local facts so we don't misspell them + ansible.builtin.set_fact: + splunk_log_cfg: "{{ splunk_home }}/etc/log.cfg" + splunk_log_local_cfg: "{{ splunk_home }}/etc/log-local.cfg" + +- name: Make sure log-local.cfg exists + ansible.builtin.stat: + path: "{{ splunk_log_local_cfg }}" + register: log_local_stat + become: true + +- name: Copy log to log-local if it doesn't exist + ansible.builtin.copy: + remote_src: true + src: "{{ splunk_log_cfg }}" + dest: "{{ splunk_log_local_cfg }}" + owner: "{{ splunk_nix_user }}" + group: "{{ splunk_nix_group }}" + mode: preserve + become: true + when: not log_local_stat.stat.exists + +- name: If it exists, add after splunkd stanza or change the log level and persist it in log-local.cfg + ansible.builtin.lineinfile: + path: "{{ splunk_log_local_cfg }}" + regexp: 'category.{{ splunk_component }}=[^,\s]+' + line: 'category.{{ splunk_component }}={{ splunk_component_log_level }}' + insertafter: '\[splunkd\]' + become: true + become_user: "{{ splunk_nix_user }}" + notify: restart splunk From 21e1582ef4192d89c347b2d781bf93c39329fc61 Mon Sep 17 00:00:00 2001 From: arcsector <26469747+arcsector@users.noreply.github.com> Date: Thu, 1 Jun 2023 11:38:43 -0700 Subject: [PATCH 2/4] Adding log level change to readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 1106ef42..82813a33 100644 --- a/README.md +++ b/README.md @@ -125,6 +125,7 @@ This section contains additional reference documentation. Note: Any task with an **adhoc** prefix means that it can be used independently as a `deployment_task` in a playbook. You can use the tasks to resolve various Splunk problems or perform one-time activities, such as decommissioning an indexer from an indexer cluster. +- **adhoc_change_log_level.yml** - Changes the log-level at the CLI level, then makes sure `log-local.cfg` exists, and adds or modifies the the log level in there. Requires the variables `splunk_component` and `splunk_component_log_level` provided to the task. - **adhoc_clean_dispatch.yml** - This task is intended to be used for restoring service to search heads should the dispatch directory become full. You should not need to use this task in a healthy environment, but it is at your disposal should the need arise. The task will stop splunk, remove all files in the dispatch directory, and then start splunk. - **adhoc_configure_hostname** - Configure a Splunk server's hostname using the value from inventory_hostname. It configures the system hostname, serverName in server.conf and host in inputs.conf. All Splunk configuration changes are made using the ini_file module, which will preserve any other existing configurations that may exist in server.conf and/or inputs.conf. - **adhoc_decom_indexer.yml** - Executes a splunk offline --enforce-counts command. This is useful when decommissioning one or more indexers from an indexer cluster. From d3a19caf13bbfd49d9dd20e6dd7b90abf1044e19 Mon Sep 17 00:00:00 2001 From: arcsector <26469747+arcsector@users.noreply.github.com> Date: Tue, 6 Jun 2023 10:10:10 -0700 Subject: [PATCH 3/4] Adding firstmatch and line start to insertafter --- roles/splunk/tasks/adhoc_change_log_level.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/splunk/tasks/adhoc_change_log_level.yml b/roles/splunk/tasks/adhoc_change_log_level.yml index 1d35ad27..18a4e707 100644 --- a/roles/splunk/tasks/adhoc_change_log_level.yml +++ b/roles/splunk/tasks/adhoc_change_log_level.yml @@ -38,7 +38,8 @@ path: "{{ splunk_log_local_cfg }}" regexp: 'category.{{ splunk_component }}=[^,\s]+' line: 'category.{{ splunk_component }}={{ splunk_component_log_level }}' - insertafter: '\[splunkd\]' + insertafter: '^\[splunkd\]' + firstmatch: true become: true become_user: "{{ splunk_nix_user }}" notify: restart splunk From f3073407b6b4cc122cc3c61d91e2364a8a6e6011 Mon Sep 17 00:00:00 2001 From: arcsector <26469747+arcsector@users.noreply.github.com> Date: Wed, 7 Jun 2023 20:12:41 -0700 Subject: [PATCH 4/4] splunk_components var as a list of dicts for changing log levels --- README.md | 2 +- roles/splunk/tasks/adhoc_change_log_level.yml | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 82813a33..e8c6dedd 100644 --- a/README.md +++ b/README.md @@ -125,7 +125,7 @@ This section contains additional reference documentation. Note: Any task with an **adhoc** prefix means that it can be used independently as a `deployment_task` in a playbook. You can use the tasks to resolve various Splunk problems or perform one-time activities, such as decommissioning an indexer from an indexer cluster. -- **adhoc_change_log_level.yml** - Changes the log-level at the CLI level, then makes sure `log-local.cfg` exists, and adds or modifies the the log level in there. Requires the variables `splunk_component` and `splunk_component_log_level` provided to the task. +- **adhoc_change_log_level.yml** - Changes the log-level at the CLI level, then makes sure `log-local.cfg` exists, and adds or modifies the the log level in there. Requires the variable `splunk_components` as a list of dictionaries with keys `splunk_component` and `splunk_component_log_level` provided to the task, like: `splunk_components: [{splunk_component: "TcpInputProc", splunk_component_log_level: "DEBUG"}]`. - **adhoc_clean_dispatch.yml** - This task is intended to be used for restoring service to search heads should the dispatch directory become full. You should not need to use this task in a healthy environment, but it is at your disposal should the need arise. The task will stop splunk, remove all files in the dispatch directory, and then start splunk. - **adhoc_configure_hostname** - Configure a Splunk server's hostname using the value from inventory_hostname. It configures the system hostname, serverName in server.conf and host in inputs.conf. All Splunk configuration changes are made using the ini_file module, which will preserve any other existing configurations that may exist in server.conf and/or inputs.conf. - **adhoc_decom_indexer.yml** - Executes a splunk offline --enforce-counts command. This is useful when decommissioning one or more indexers from an indexer cluster. diff --git a/roles/splunk/tasks/adhoc_change_log_level.yml b/roles/splunk/tasks/adhoc_change_log_level.yml index 18a4e707..db4da058 100644 --- a/roles/splunk/tasks/adhoc_change_log_level.yml +++ b/roles/splunk/tasks/adhoc_change_log_level.yml @@ -1,7 +1,7 @@ --- - name: Change log level of {{ splunk_component }} to {{ splunk_component_log_level }} via CLI ansible.builtin.command: | - {{ splunk_home }}/bin/splunk set log-level {{ splunk_component }} -level {{ splunk_component_log_level }} -auth '{{ splunk_auth }}' + {{ splunk_home }}/bin/splunk set log-level {{ item.splunk_component }} -level {{ item.splunk_component_log_level }} -auth '{{ splunk_auth }}' register: log_level_chg_out changed_when: log_level_chg_out.rc == 0 failed_when: log_level_chg_out.rc != 0 @@ -10,6 +10,7 @@ when: - splunk_component is defined - splunk_component_log_level is defined + loop: "{{ splunk_components }}" - name: Set log and log-local facts so we don't misspell them ansible.builtin.set_fact: @@ -36,10 +37,10 @@ - name: If it exists, add after splunkd stanza or change the log level and persist it in log-local.cfg ansible.builtin.lineinfile: path: "{{ splunk_log_local_cfg }}" - regexp: 'category.{{ splunk_component }}=[^,\s]+' - line: 'category.{{ splunk_component }}={{ splunk_component_log_level }}' + regexp: 'category.{{ item.splunk_component }}=[^,\s]+' + line: 'category.{{ item.splunk_component }}={{ item.splunk_component_log_level }}' insertafter: '^\[splunkd\]' firstmatch: true become: true become_user: "{{ splunk_nix_user }}" - notify: restart splunk + loop: "{{ splunk_components }}"