diff --git a/README.md b/README.md index 1106ef42..9c5b6d3f 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,7 @@ The following example playbooks have been included in this project for your refe - **splunk_app_install.yml** - Install or upgrade apps on Splunk hosts using the configure_apps.yml task in the splunk role. Note that the apps you want to deploy should be defined in either host_vars or group_vars, along with a splunk_app_deploy_path. Refer to the documentation for app deployment for details. - **splunk_install_or_upgrade.yml** - Install or upgrade Splunk (or Splunk UFs) on hosts using the check_splunk.yml task in the splunk role. - **splunk_shc_deploy.yml** - Installs Splunk and initializes search head clustering on a shdeployer and group of hosts that will serve as a new search head cluster. +- **splunk_idx_deploy.yml** - Installs Splunk and initializes indexer clustering on cluster manager and indexers. It's possible to activate multisite cluster with variables. - **splunk_upgrade_full_stack.yml** - Example playbook that demonstrates how to upgrade an entire Splunk deployment with a single-site indexer cluster and a search head cluster using the splunk role. Note: This playbook does not upgrade forwarders, although you could easily add an extra play to do that. ## Extended Documentation @@ -139,8 +140,8 @@ Note: Any task with an **adhoc** prefix means that it can be used independently - **configure_deploymentclient.yml** - Generates a new deploymentclient.conf file from the deploymentclient.conf.j2 template and installs it to $SPLUNK_HOME/etc/system/local/deploymentclient.conf. This task is included automatically during new installations when values have been configured for the `clientName` and `splunk_uri_ds` variables. - **configure_dmc.yml** - Configures the DMC as an Indexer Peer in SH mode, adds hosts to the host as search peers, and configures the host MC in auto mode - **configure_facl.yml** - Configure file system access control lists (FACLs) to allow the splunk user to read /var/log files and add the splunk user's group to /etc/audit/auditd.conf to read /var/log/audit/ directory. This allows the splunk user to read privileged files from a non-privileged system account. Note: This task is performed automatically during new installations when splunk is installed as a non-root user. -- **configure_idxc_manager.yml** - Configures a Splunk host to act as a manager node using `splunk_idxc_rf`, `splunk_idxc_sf`, `splunk_idxc_key`, and `splunk_idxc_label`. -- **configure_idxc_member.yml** - Configures a Splunk host as an indexer cluster member using `splunk_uri_cm`, `splunk_idxc_rep_port`, and `splunk_idxc_key`. +- **configure_idxc_manager.yml** - Configures a Splunk host to act as a manager node using `splunk_idxc_rf`, `splunk_idxc_sf`, `splunk_idxc_key`, `splunk_idxc_label` and `splunk_idxc_multisite`. For multisite clusters are additional variables necessary like `splunk_idxc_available_sites`, `splunk_idxc_site_mappings`, `splunk_idxc_site_rf`, `splunk_idxc_site_sf` and `splunk_idxc_site_affinity`. +- **configure_idxc_member.yml** - Configures a Splunk host as an indexer cluster member using `splunk_uri_cm`, `splunk_idxc_rep_port`, `splunk_idxc_key` and `splunk_idxc_site_affinity`. - **configure_idxc_sh.yml** - Configures a search head to join an existing indexer cluster using `splunk_uri_cm` and `splunk_idxc_key`. - **configure_license.yml** - Configure the license group to the `splunk_license_group` variable defined. Default is `Trial`. Available values are "Trial, Free, Enterprise, Forwarder, Manager or Peer. If set to `Peer`, the `splunk_uri_lm` must be defined. Note: This could also be accomplished using configure_apps.yml with a git repository. - **configure_os.yml** - Increases ulimits for the splunk user and disables Transparent Huge Pages (THP) per Splunk implementation best practices. @@ -193,4 +194,4 @@ If you have questions or need support, you can: ## License Copyright 2018-2021 Splunk. -Distributed under the terms of the Apache 2.0 license, ansible-role-for-splunk is free and open-source software. +Distributed under the terms of the Apache 2.0 license, ansible-role-for-splunk is free and open-source software. \ No newline at end of file diff --git a/playbooks/splunk_idxc_deploy.yml b/playbooks/splunk_idxc_deploy.yml index 8d48a90a..e9cf53e8 100644 --- a/playbooks/splunk_idxc_deploy.yml +++ b/playbooks/splunk_idxc_deploy.yml @@ -21,3 +21,10 @@ - ../roles/splunk vars: - deployment_task: configure_idxc_member.yml + +- hosts: + - search + roles: + - ../roles/splunk + vars: + - deployment_task: configure_idxc_sh.yml diff --git a/roles/splunk/defaults/main.yml b/roles/splunk/defaults/main.yml index 90c1408d..c81e8ab3 100644 --- a/roles/splunk/defaults/main.yml +++ b/roles/splunk/defaults/main.yml @@ -13,11 +13,19 @@ splunk_package_url_uf: "https://download.splunk.com/products/universalforwarder/ splunk_download_local: true # This defines how the download process works. If `true` it will download to localhost and copy around to hosts from there. If `false` each host will download the package individually. splunk_install_type: undefined # There are two ways to configure this. The easiest way is to nest hosts under either a "full" group or a "uf" group in your inventory and main.yml will handle it for you. Or, you can also set the value via a group_vars or host_vars file. splunk_install_path: /opt # Base directory on the operating system to which splunk should be installed +splunk_file_type: tgz # Default is a tgz installer +splunk_package_gpg: SplunkPGPKey.pub least_privileged: false # Do not change. This get automatically set in `tasks/main.yml` based on the version and install type. -splunk_nix_user: splunk -splunk_nix_group: splunk local_os_user: false # Whenther or not to force creation of a user using the `luseradd` or not. local_os_group: false # Whether or not to force creation of a group using the `lgroupadd` or not. +splunk_nix_user: splunk +splunk_nix_group: splunk +splunk_server_sslVersions: tls1.2 +splunk_web_sslVersions: tls1.2 +splunk_webserver_port: 8000 +splunk_licenses: [] +splunk_ansible_configs_dir: _ansible_configs +splunk_tmp_file_dir: /tmp splunk_uri_lm: undefined splunk_license_file: [] # This can be a list of license files to copy to the host. splunk_license_group: Trial # The default matches with the group splunk ships with. You can also set the value via a group_vars or host_vars file. @@ -27,6 +35,7 @@ clientName: undefined phoneHomeIntervalInSecs: undefined splunk_general_key: undefined # Configures a pass4SymmKey in server.conf under the general stanza splunk_ds_key: undefined # Configures a pass4SymmKey in server.conf for authenticating against a deployment server +splunk_ds_requireAuth: false splunk_admin_username: admin splunk_admin_password: undefined # Use ansible-vault encrypt_string, e.g. ansible-vault encrypt_string --ask-vault-pass 'var_value_to_encrypt' --name 'var_name' splunk_configure_secret: false # If set to true, you need to update files/splunk.secret @@ -42,21 +51,28 @@ splunk_force_kill: False systemd_unit_full: Splunkd # You can change this in `host_vars` or `group_vars` to customize the service name. systemd_unit_uf: SplunkForwarder # You can change this in `host_vars` or `group_vars` to customize the service name. splunk_disable_mgmt_port: false # If set to true, will disable splunkd management port during installation -splunk_mgmt_uri: "{{ ansible_fqdn }}" # If the `ansible_fqdn` is not resolvable by other hosts, you can set it to something like `ansible_facts.default_ipv4.address` in `host_vars` or `group_vars` to use the IP address instead. +splunk_disable_kvstore: false # If set to true, will disable the kvstore splunkd_port: 8089 # If changed, will overwrite the default port number used by splunkd git_local_clone_path: ~/ # Base directory under which repositories for app deplyoment should be cloned to git_server: undefined # e.g. ssh://git@mygithost:1234 - Note that this may be set in an all.yml group_var or inside the git_apps dictionary within host_vars -git_key: undefined # Path to SSH key for cloning repositories - Note that this may be set in an all.yml group_var or inside the git_apps dictionary within host_vars +git_key: ~ # Path to SSH key for cloning repositories - Note that this may be set in an all.yml group_var or inside the git_apps dictionary within host_vars git_project: undefined git_version: master # Configure default version to clone, overridable inside the git_apps dictionary within host_vars +splunk_app_deploy_path: etc/apps # Path under $SPLUNK_HOME/ to deploy apps to - Note that this may be set in group_vars, host_vars, playbook vars, or inside the git_apps dictionary within host_vars app_relative_path: # set a sub-path you want to sync within a repo. If the repo contains multiple apps in the root directory, just set this to a trailing slash. -splunk_app_deploy_path: undefined # Path under $SPLUNK_HOME/ to deploy apps to - Note that this may be set in group_vars, host_vars, playbook vars, or inside the git_apps dictionary within host_vars +splunk_app_deploy_path: # Path under $SPLUNK_HOME/ to deploy apps to - Note that this may be set in group_vars, host_vars, playbook vars, or inside the git_apps dictionary within host_vars # IDXC Vars -splunk_idxc_key: mypass4symmkey +splunk_idxc_key: undefined # Use ansible-vault encrypt_string, e.g. ansible-vault encrypt_string --ask-vault-pass 'var_value_to_encrypt' --name 'var_name' +splunk_idxc_multisite: false splunk_idxc_rf: 2 splunk_idxc_sf: 2 +splunk_idxc_available_sites: "site1" +splunk_idxc_site_mappings: "" # Default are no mappings +splunk_idxc_site_rf: "origin:1,total:2" +splunk_idxc_site_sf: "origin:1,total:2" splunk_idxc_rep_port: 9887 splunk_idxc_label: myidxc +splunk_idxc_site_affinity: "site0" # Default to site0 for Searchheads and Forwarders, and need to be set to != site0 on manager splunk_apply_cluster_bundle_retries: 3 # How many times to retry indexer cluster bundle apply if it fails splunk_apply_cluster_bundle_delay: 60 # Delay in seconds between retries to apply indexer cluster bundle splunk_apply_shcluster_bundle_retries: 3 # How many times to retry SHC bundle apply if it fails @@ -67,8 +83,8 @@ splunk_shc_label: myshc splunk_shc_rf: 3 splunk_shc_rep_port: 8100 splunk_shc_target_group: shc -splunk_shc_deployer: "{{ groups['shdeployer'] | first }}" # If you manage multiple SHCs, configure the var value in group_vars -splunk_shc_uri_list: "{% for h in groups[splunk_shc_target_group] %}https://{{ hostvars[h].splunk_mgmt_uri }}:{{ splunkd_port }}{% if not loop.last %},{% endif %}{% endfor %}" # If you manage multiple SHCs, configure the var value in group_vars +splunk_shc_deployer: "{{ groups['splunk_shdeployer'] | first }}" # If you manage multiple SHCs, configure the var value in group_vars +splunk_shc_uri_list: "{% for h in groups[splunk_shc_target_group] %}https://{{ hostvars[h].ansible_fqdn }}:{{ splunkd_port }}{% if not loop.last %},{% endif %}{% endfor %}" # If you manage multiple SHCs, configure the var value in group_vars start_splunk_handler_fired: false # Do not change; used to prevent unnecessary splunk restarts # Linux and scripting related vars add_crashlog_script: false # Set to true to install a script and cron job to automatically cleanup splunk crash logs older than 7 days diff --git a/roles/splunk/tasks/configure_idxc_manager.yml b/roles/splunk/tasks/configure_idxc_manager.yml index 6511c423..99458c94 100644 --- a/roles/splunk/tasks/configure_idxc_manager.yml +++ b/roles/splunk/tasks/configure_idxc_manager.yml @@ -8,7 +8,18 @@ # https://docs.splunk.com/Documentation/Splunk/latest/Indexer/Configuremanagerwithserverconf - name: Setting clustering mode based on Splunk version number set_fact: - mode_value: "{% if splunk_version_release | float < 8.1 %}master{% else %}manager{% endif %}" + mode_value: "{{ splunk_version_release is version('8.1', '<') | ternary('master', 'manager') }}" + +- name: Setting site affinity of cluster manager if not set + set_fact: + site_affinity: "{% if splunk_idxc_multisite and splunk_idxc_site_affinity == 'site0' %}site1{% else %}{{ splunk_idxc_site_affinity }}{% endif %}" + +- name: Extract encrypted value + include_tasks: check_decrypted_secret.yml + vars: + req_secret_conf: server + req_secret_section: clustering + req_secret_option: pass4SymmKey - name: Configure clustering stanza for cluster manager node ini_file: @@ -23,10 +34,53 @@ notify: - restart splunk - wait for splunkd - no_log: true + no_log: false loop: - - { option: "mode", value: "{{ mode_value}}" } + - { option: "mode", value: "{{ mode_value }}" } - { option: "replication_factor", value: "{{ splunk_idxc_rf }}" } - { option: "search_factor", value: "{{ splunk_idxc_sf }}" } - - { option: "pass4SymmKey", value: "{{ splunk_idxc_key }}" } - { option: "cluster_label", value: "{{ splunk_idxc_label }}" } + - { option: "multisite", value: "{{ splunk_idxc_multisite | ternary('true', 'false') }}" } + - { option: "available_sites", value: "{{ splunk_idxc_available_sites }}" } + - { option: "site_replication_factor", value: "{{ splunk_idxc_site_rf }}" } + - { option: "site_search_factor", value: "{{ splunk_idxc_site_sf }}" } + - { option: "site_mappings", value: "{{ splunk_idxc_site_mappings }}" } + +- name: Configure clustering stanza psk for cluster manager node + ini_file: + path: "{{ splunk_home }}/etc/system/local/server.conf" + section: clustering + option: "{{ item.option }}" + value: "{{ item.value }}" + mode: 0644 + owner: "{{ splunk_nix_user }}" + group: "{{ splunk_nix_group }}" + become: true + when: + - splunk_idxc_key != "undefined" + - encrypted_secret_value.stdout == "" or (splunk_idxc_key != decrypted_secret_value.stdout | default('')) + notify: + - restart splunk + - wait for splunkd + no_log: true + loop: + - { option: "pass4SymmKey", value: "{{ splunk_idxc_key }}" } + +- name: Configure general stanza on manager node + ini_file: + path: "{{ splunk_home }}/etc/system/local/server.conf" + section: general + option: "{{ item.option }}" + value: "{{ item.value }}" + mode: 0644 + owner: "{{ splunk_nix_user }}" + group: "{{ splunk_nix_group }}" + become: true + notify: + - restart splunk + - wait for splunkd + no_log: true + when: + - splunk_idxc_multisite + loop: + - { option: "site", value: "{{ site_affinity }}" } diff --git a/roles/splunk/tasks/configure_idxc_member.yml b/roles/splunk/tasks/configure_idxc_member.yml index 3eea6f93..717e155c 100644 --- a/roles/splunk/tasks/configure_idxc_member.yml +++ b/roles/splunk/tasks/configure_idxc_member.yml @@ -8,17 +8,21 @@ # https://docs.splunk.com/Documentation/Splunk/8.0.9/Indexer/ConfigurepeerswithCLI - name: Setting clustering mode based on Splunk version number set_fact: - mode_value: "{% if splunk_version_release | float < 8.1 %}slave{% else %}peer{% endif %}" + mode_value: "{{ splunk_version_release is version('8.1', '<') | ternary('slave', 'peer') }}" + manager_uri_call: "{{ splunk_version_release is version('8.1', '<') | ternary('master_uri', 'manager_uri') }}" +# TODO: Pre check if cluster settings are different, because this always needs a restart - name: Configure idxc member - command: "{{ splunk_home }}/bin/splunk edit cluster-config -mode {{ mode_value }} -auth {{ splunk_auth }} -master_uri {{ splunk_uri_cm }} -replication_port {{ splunk_idxc_rep_port }} -secret {{ splunk_idxc_key }}" + command: "{{ splunk_home }}/bin/splunk edit cluster-config -mode {{ mode_value }} {% if splunk_idxc_multisite %}-site {{ splunk_idxc_site_affinity }}{% endif %} -auth {{ splunk_auth }} -{{ manager_uri_call }} {{ splunk_uri_cm }} -replication_port {{ splunk_idxc_rep_port }} -secret {{ splunk_idxc_key }}" become: true become_user: "{{ splunk_nix_user }}" register: idxc_peer_init_result changed_when: idxc_peer_init_result.rc == 0 failed_when: idxc_peer_init_result.rc != 0 - notify: restart splunk - no_log: true + notify: + - restart splunk + - wait for splunkd + no_log: false until: idxc_peer_init_result.rc == 0 retries: 6 delay: 5 diff --git a/roles/splunk/tasks/configure_idxc_sh.yml b/roles/splunk/tasks/configure_idxc_sh.yml index 38468bcf..2c70202e 100644 --- a/roles/splunk/tasks/configure_idxc_sh.yml +++ b/roles/splunk/tasks/configure_idxc_sh.yml @@ -1,13 +1,38 @@ --- +- name: Run splunk version command to check currently installed version + include_tasks: check_splunk_version.yml + +- name: Setting clustering mode based on Splunk version number + set_fact: + manager_uri_call: "{{ splunk_version_release is version('8.1', '<') | ternary('master_uri', 'manager_uri') }}" + manager_command_call: "{{ splunk_version_release is version('8.1', '<') | ternary('cluster-master', 'cluster-manager') }}" + +# TODO: Maybe switch to config file, this is the cleaner solution and would work with multiple indexer clusters - name: Configure search head to join indexer cluster - command: "{{ splunk_home }}/bin/splunk edit cluster-config -mode searchhead -master_uri {{ splunk_uri_cm }} -secret {{ splunk_idxc_key }} -auth {{ splunk_auth }}" + command: "{{ splunk_home }}/bin/splunk edit cluster-config -mode searchhead -{{ manager_uri_call }} {{ splunk_uri_cm }} -secret {{ splunk_idxc_key }} -auth {{ splunk_auth }}" become: true become_user: "{{ splunk_nix_user }}" register: idxc_sh_join_result changed_when: idxc_sh_join_result.rc == 0 failed_when: idxc_sh_join_result.rc != 0 - notify: restart splunk + notify: + - restart splunk + - wait for splunkd no_log: true until: idxc_sh_join_result.rc == 0 retries: 6 delay: 5 + +# Error Code 22, means no change in current configuration +- name: Configure search head site affinity + command: "{{ splunk_home }}/bin/splunk edit {{ manager_command_call }} {{ splunk_uri_cm }} -site {{ splunk_idxc_site_affinity }} -auth {{ splunk_auth }}" + when: splunk_idxc_multisite + become: true + become_user: "{{ splunk_nix_user }}" + register: idxc_sh_join_result + changed_when: idxc_sh_join_result.rc == 0 + failed_when: idxc_sh_join_result.rc != 0 and idxc_sh_join_result.rc != 22 + no_log: true + until: idxc_sh_join_result.rc == 0 or idxc_sh_join_result.rc == 22 + retries: 6 + delay: 5