diff --git a/inventory/splunk_defaults_linux.yml b/inventory/splunk_defaults_linux.yml index 73bfa2e0..fce2b49f 100644 --- a/inventory/splunk_defaults_linux.yml +++ b/inventory/splunk_defaults_linux.yml @@ -7,7 +7,7 @@ restart_retry_delay: 30 retry_num: 60 hide_password: false wait_for_splunk_retry_num: 60 -shc_sync_retry_num: 60 +shc_sync_retry_num: 100 config: max_retries: 3 diff --git a/roles/splunk_search_head/tasks/search_head_clustering.yml b/roles/splunk_search_head/tasks/search_head_clustering.yml index 8d181e0b..8bccb9f4 100644 --- a/roles/splunk_search_head/tasks/search_head_clustering.yml +++ b/roles/splunk_search_head/tasks/search_head_clustering.yml @@ -47,6 +47,26 @@ vars: splunk_instance_address: "{{ splunk.deployer_url }}" +# Wait for all other SHC members to be reachable before attempting captain bootstrap. +# Without this wait, the bootstrap command fails when other pods haven't finished starting +# Splunk yet (first boot takes ~8-10 minutes). The default retry_num (60 × 6s = 360s) +# is shorter than the first-boot Splunk start time, causing the ansible run to exit with +# code 2 and triggering an unnecessary pod restart that adds ~16 minutes to SHC readiness. +# Using shc_sync_retry_num (60 × 6s default) here — can be raised independently if needed. +- name: Wait for all SHC members to be reachable before captain bootstrap + uri: + url: "{{ cert_prefix }}://{{ item }}:{{ splunk.svc_port }}" + method: GET + validate_certs: false + use_proxy: no + register: task_response + until: task_response.status == 200 + retries: "{{ shc_sync_retry_num }}" + delay: "{{ retry_delay }}" + loop: "{{ groups['splunk_search_head'] | difference([inventory_hostname]) }}" + when: splunk_search_head_captain | bool + no_log: "{{ hide_password }}" + #INFRA-38882: Update exec command? - name: Boostrap SHC captain command: "{{ splunk.exec }} bootstrap shcluster-captain -servers_list '{% for host in groups['splunk_search_head'] %}{{ cert_prefix }}://{{ host }}:{{ splunk.svc_port }}{% if not loop.last %},{% endif %}{% endfor %}' -auth '{{ splunk.admin_user }}:{{ splunk.password }}'"