diff --git a/docs/ADVANCED.md b/docs/ADVANCED.md index 4341071d..4c18a9bd 100644 --- a/docs/ADVANCED.md +++ b/docs/ADVANCED.md @@ -137,10 +137,19 @@ Splunk-Ansible ships with an inventory script in `inventory/environ.py`. The scr | SPLUNK_ES_SSL_ENABLEMENT | Set the ssl-enablement flag in ES. Valid values are 'auto', 'strict', and 'ignore'. Defaults to auto when present. | no | no | no | | SPLUNK_SERVICE_NAME | Used alongside `POD_NAMESPACE` and `CLUSTER_DOMAIN` to construct a k8s-supported `issuer_uri` value for oauth2 configurations | no | no | no | | SPLUNK_HEADLESS_SERVICE_NAME | Used alongside `POD_NAME`, `POD_NAMESPACE`, and `CLUSTER_DOMAIN` to construct a k8s-supported `serverName` value. This is also used for the `search_head_uri` and `register_replication_address` settings when clustering is enabled. | no | no | no | +| SPLUNK_NOAH_ENABLED | Explicitly enables Noah-specific provisioning for supported indexer, search-head, and deployer roles. Defaults to `false`; a `noahService` stanza alone does not enable Noah mode. | no | no | no | | POD_NAME | Defines the current pod name in a k8s environment | no | no | no | | POD_NAMESPACE | Defines the namespace of the current pod in a k8s environment | no | no | no | | CLUSTER_DOMAIN | Defines the domain name for DNS resolution in a k8s cluster (default: cluster.local) | no | no | no | +For Linux search-head cluster members, Ansible now writes the SHC member, +replication-listener, stable server-name, and (for classic deployments) +Cluster Manager peering configuration while splunkd is stopped. This applies +to both classic and Noah deployments. The later SHC commands still form and +verify the cluster, but they do not request duplicate restarts when the +effective pre-start configuration already matches. If splunkd is already +running, Ansible keeps the established live-configuration and restart path. + \* Password must be set either in `default.yml` or as the environment variable `SPLUNK_PASSWORD` #### Additional Splunk Universal Forwarder variables diff --git a/inventory/environ.py b/inventory/environ.py index cc36176b..d1ab0384 100755 --- a/inventory/environ.py +++ b/inventory/environ.py @@ -130,6 +130,7 @@ def getDefaultVars(): defaultVars = loadDefaults() defaultVars["splunk"]["role"] = os.environ.get('SPLUNK_ROLE', defaultVars["splunk"].get("role") or "splunk_standalone") overrideEnvironmentVars(defaultVars) + getNoah(defaultVars) getAnsibleContext(defaultVars) getASan(defaultVars) getDisablePopups(defaultVars) @@ -192,7 +193,22 @@ def getServiceName(vars_scope): if serviceName != "" and namespace != "": vars_scope["splunk"]["issuer_uri"] = "{}.{}.svc.{}".format(serviceName, namespace, clusterDomain) if headlessServiceName != "" and namespace != "": - vars_scope["splunk"]["server_name"] = "{}.{}.{}.svc.{}".format(podName, headlessServiceName, namespace, clusterDomain) + server_name = "{}.{}.{}.svc.{}".format(podName, headlessServiceName, namespace, clusterDomain) + vars_scope["splunk"]["server_name"] = server_name + if vars_scope.get("splunk_noah_enabled", False): + vars_scope["splunk"]["noah_advertised_addr"] = "https://{}:{}".format(server_name, vars_scope["splunk"]["svc_port"]) + +def getNoah(vars_scope): + """Enable Noah provisioning only when explicitly requested.""" + value = os.environ.get("SPLUNK_NOAH_ENABLED", vars_scope.get("splunk_noah_enabled", False)) + if isinstance(value, bool): + vars_scope["splunk_noah_enabled"] = value + return + + normalized = str(value).strip().lower() + if normalized not in ("true", "false"): + raise ValueError("SPLUNK_NOAH_ENABLED must be either 'true' or 'false'") + vars_scope["splunk_noah_enabled"] = normalized == "true" def getSplunkPaths(vars_scope): """ diff --git a/inventory/splunk_defaults_linux.yml b/inventory/splunk_defaults_linux.yml index 73bfa2e0..7d6f3def 100644 --- a/inventory/splunk_defaults_linux.yml +++ b/inventory/splunk_defaults_linux.yml @@ -8,6 +8,7 @@ retry_num: 60 hide_password: false wait_for_splunk_retry_num: 60 shc_sync_retry_num: 60 +splunk_noah_enabled: false config: max_retries: 3 diff --git a/roles/splunk_common/handlers/restart_splunk.yml b/roles/splunk_common/handlers/restart_splunk.yml index 118ede21..8c0fc794 100644 --- a/roles/splunk_common/handlers/restart_splunk.yml +++ b/roles/splunk_common/handlers/restart_splunk.yml @@ -1,5 +1,24 @@ --- -- name: "Restart the splunkd service - Via CLI" +- name: "Restart Noah-managed splunkd - Via bounded CLI stop and start" + shell: | + set -e + if {{ splunk.exec }} status >/dev/null 2>&1; then + {{ splunk.exec }} stop --answer-yes + fi + {{ splunk.exec }} start --answer-yes --accept-license + args: + executable: /bin/sh + become: yes + become_user: "{{ splunk.user }}" + register: task_result + until: task_result.rc == 0 + retries: "{{ retry_num }}" + delay: "{{ restart_retry_delay }}" + when: + - not splunk.enable_service + - splunk_noah_enabled | default(false) | bool + +- name: "Restart classic splunkd service - Via CLI" command: "{{ splunk.exec }} restart --answer-yes --accept-license" become: yes become_user: "{{ splunk.user }}" @@ -7,7 +26,9 @@ until: task_result.rc == 0 retries: "{{ retry_num }}" delay: "{{ restart_retry_delay }}" - when: not splunk.enable_service + when: + - not splunk.enable_service + - not (splunk_noah_enabled | default(false) | bool) - name: "Restart the splunkd service - Via Linux systemd or init" service: diff --git a/roles/splunk_common/tasks/configure_deployer_prestart.yml b/roles/splunk_common/tasks/configure_deployer_prestart.yml new file mode 100644 index 00000000..7ad86554 --- /dev/null +++ b/roles/splunk_common/tasks/configure_deployer_prestart.yml @@ -0,0 +1,69 @@ +--- +- name: Validate pre-start SHC deployer configuration inputs + assert: + that: + - splunk.shc.pass4SymmKey | default("") | length > 0 + - splunk.shc.label | default("") | length > 0 + fail_msg: >- + Pre-start SHC deployer configuration requires the SHC label and shared + symmetric key. + no_log: true + +- name: Create the local Splunk configuration directory for the SHC deployer + file: + path: "{{ splunk.home }}/etc/system/local" + state: directory + mode: 0770 + owner: "{{ splunk.user }}" + group: "{{ splunk.group }}" + become: yes + become_user: "{{ splunk.user }}" + +- name: Configure the SHC deployer before the first splunkd start + ini_file: + path: "{{ splunk.home }}/etc/system/local/server.conf" + section: "shclustering" + option: "{{ item.option }}" + value: "{{ item.value }}" + allow_no_value: false + state: present + mode: 0660 + owner: "{{ splunk.user }}" + group: "{{ splunk.group }}" + loop: + - option: "pass4SymmKey" + value: "{{ splunk.shc.pass4SymmKey }}" + - option: "shcluster_label" + value: "{{ splunk.shc.label }}" + loop_control: + label: "{{ item.option }}" + become: yes + become_user: "{{ splunk.user }}" + no_log: true + +- name: Read effective pre-start SHC deployer configuration + command: + argv: + - "{{ splunk.exec }}" + - "btool" + - "server" + - "list" + - "shclustering" + - "--no-log" + register: deployer_prestart_btool + changed_when: false + become: yes + become_user: "{{ splunk.user }}" + no_log: true + +- name: Validate effective pre-start SHC deployer configuration + assert: + that: + - (deployer_prestart_btool.stdout | regex_search("(?m)^pass4SymmKey\\s*=\\s*\\S+\\s*$")) is not none + - ("shcluster_label = " ~ splunk.shc.label) in deployer_prestart_btool.stdout + fail_msg: "Effective pre-start SHC deployer configuration does not match the requested SHC." + no_log: true + +- name: Record declarative SHC deployer pre-start configuration + set_fact: + deployer_prestart_configured: true diff --git a/roles/splunk_common/tasks/configure_shc_prestart.yml b/roles/splunk_common/tasks/configure_shc_prestart.yml new file mode 100644 index 00000000..dc5a448a --- /dev/null +++ b/roles/splunk_common/tasks/configure_shc_prestart.yml @@ -0,0 +1,186 @@ +--- +- name: Set expected pre-start SHC configuration + set_fact: + shc_prestart_server_name: "{{ splunk.server_name | default(splunk.hostname, true) }}" + shc_prestart_mgmt_uri: "{{ cert_prefix }}://{{ splunk.hostname }}:{{ splunk.svc_port }}" + shc_prestart_deployer_uri: "{{ cert_prefix }}://{{ splunk.deployer_url }}:{{ splunk.svc_port }}" + shc_prestart_server_conf: "{{ splunk.home }}/etc/system/local/server.conf" + shc_prestart_indexer_peer_enabled: >- + {{ + not (splunk_noah_enabled | default(false) | bool) + and (splunk_indexer_cluster | bool) + and (splunk.multisite_master is not defined) + and (splunk.set_search_peers | default(false) | bool) + }} + +- name: Validate pre-start SHC configuration inputs + assert: + that: + - cert_prefix in ["http", "https"] + - shc_prestart_server_name | length > 0 + - splunk.hostname | default("") | length > 0 + - splunk.deployer_url | default("") | length > 0 + - splunk.shc.pass4SymmKey | default("") | length > 0 + - splunk.shc.replication_factor | int > 0 + - splunk.shc.replication_port | int > 0 + - splunk.shc.replication_port | int <= 65535 + - splunk.shc.replication_port | int != splunk.svc_port | int + - splunk.shc.replication_port | int != splunk.http_port | int + - splunk.shc.replication_port | int != splunk.s2s.port | int + - splunk.shc.replication_port | int != splunk.kvstore.port | int + fail_msg: >- + Pre-start SHC configuration requires stable identity, valid non-conflicting + ports, a deployer endpoint, replication factor, and shared secret. + no_log: true + +- name: Validate classic pre-start indexer peering inputs + assert: + that: + - splunk.cluster_master_url | default("") | length > 0 + - splunk.idxc.pass4SymmKey | default("") | length > 0 + fail_msg: >- + Classic SHC pre-start indexer peering requires a Cluster Manager endpoint + and indexer-cluster shared secret. + when: shc_prestart_indexer_peer_enabled | bool + no_log: true + +- name: Create the local Splunk configuration directory + file: + path: "{{ splunk.home }}/etc/system/local" + state: directory + mode: 0770 + owner: "{{ splunk.user }}" + group: "{{ splunk.group }}" + become: yes + become_user: "{{ splunk.user }}" + +- name: Configure stable SHC identity before splunkd starts + ini_file: + path: "{{ shc_prestart_server_conf }}" + section: "general" + option: "serverName" + value: "{{ shc_prestart_server_name }}" + allow_no_value: false + state: present + mode: 0660 + owner: "{{ splunk.user }}" + group: "{{ splunk.group }}" + become: yes + become_user: "{{ splunk.user }}" + +- name: Configure SHC formation before splunkd starts + ini_file: + path: "{{ shc_prestart_server_conf }}" + section: "shclustering" + option: "{{ item.option }}" + value: "{{ item.value }}" + allow_no_value: false + state: present + mode: 0660 + owner: "{{ splunk.user }}" + group: "{{ splunk.group }}" + loop: + - option: "disabled" + value: "false" + - option: "mgmt_uri" + value: "{{ shc_prestart_mgmt_uri }}" + - option: "replication_factor" + value: "{{ splunk.shc.replication_factor }}" + - option: "conf_deploy_fetch_url" + value: "{{ shc_prestart_deployer_uri }}" + - option: "shcluster_label" + value: "{{ splunk.shc.label }}" + - option: "register_replication_address" + value: "{{ shc_prestart_server_name }}" + - option: "search_head_uri" + value: "{{ shc_prestart_server_name }}" + become: yes + become_user: "{{ splunk.user }}" + +- name: Write the SHC symmetric key before the first splunkd start + ini_file: + path: "{{ shc_prestart_server_conf }}" + section: "shclustering" + option: "pass4SymmKey" + value: "{{ splunk.shc.pass4SymmKey }}" + allow_no_value: false + state: present + mode: 0660 + owner: "{{ splunk.user }}" + group: "{{ splunk.group }}" + when: first_run | bool + become: yes + become_user: "{{ splunk.user }}" + no_log: true + +- name: Configure the SHC replication listener before splunkd starts + ini_file: + path: "{{ shc_prestart_server_conf }}" + section: "replication_port://{{ splunk.shc.replication_port }}" + option: "disabled" + value: "false" + allow_no_value: false + state: present + mode: 0660 + owner: "{{ splunk.user }}" + group: "{{ splunk.group }}" + become: yes + become_user: "{{ splunk.user }}" + +- name: Configure classic indexer-cluster peering before splunkd starts + ini_file: + path: "{{ shc_prestart_server_conf }}" + section: "clustering" + option: "{{ item.option }}" + value: "{{ item.value }}" + allow_no_value: false + state: present + mode: 0660 + owner: "{{ splunk.user }}" + group: "{{ splunk.group }}" + loop: + - option: "mode" + value: "searchhead" + - option: "master_uri" + value: "{{ cert_prefix }}://{{ splunk.cluster_master_url }}:{{ splunk.svc_port }}" + - option: "pass4SymmKey" + value: "{{ splunk.idxc.pass4SymmKey }}" + when: shc_prestart_indexer_peer_enabled | bool + become: yes + become_user: "{{ splunk.user }}" + no_log: true + +- name: Read effective pre-start SHC configuration + command: + argv: + - "{{ splunk.exec }}" + - "btool" + - "server" + - "list" + - "--no-log" + register: shc_prestart_btool + changed_when: false + become: yes + become_user: "{{ splunk.user }}" + no_log: true + +- name: Validate effective pre-start SHC configuration + assert: + that: + - (shc_prestart_btool.stdout | regex_search("(?m)^serverName\\s*=\\s*" ~ (shc_prestart_server_name | regex_escape) ~ "\\s*$")) is not none + - (shc_prestart_btool.stdout | regex_search("(?m)^disabled\\s*=\\s*(0|false)\\s*$")) is not none + - ("mgmt_uri = " ~ shc_prestart_mgmt_uri) in shc_prestart_btool.stdout + - ("replication_factor = " ~ (splunk.shc.replication_factor | string)) in shc_prestart_btool.stdout + - ("conf_deploy_fetch_url = " ~ shc_prestart_deployer_uri) in shc_prestart_btool.stdout + - (shc_prestart_btool.stdout | regex_search("(?m)^pass4SymmKey\\s*=\\s*\\S+\\s*$")) is not none + - ("register_replication_address = " ~ shc_prestart_server_name) in shc_prestart_btool.stdout + - ("search_head_uri = " ~ shc_prestart_server_name) in shc_prestart_btool.stdout + - ("[replication_port://" ~ (splunk.shc.replication_port | string) ~ "]") in shc_prestart_btool.stdout + fail_msg: "Effective pre-start SHC configuration does not match the requested member configuration." + no_log: true + +- name: Record declarative SHC pre-start configuration + set_fact: + shc_prestart_configured: true + shc_prestart_defer_initial_restart: true + shc_prestart_indexer_peer_configured: "{{ shc_prestart_indexer_peer_enabled | bool }}" diff --git a/roles/splunk_common/tasks/main.yml b/roles/splunk_common/tasks/main.yml index fdc0fc91..362dba70 100644 --- a/roles/splunk_common/tasks/main.yml +++ b/roles/splunk_common/tasks/main.yml @@ -58,6 +58,12 @@ - include_tasks: set_general_symmkey_password.yml when: "'pass4SymmKey' in splunk and splunk.pass4SymmKey" +- name: Configure Noah safely before temporary authentication startup + include_role: + name: splunk_noah + tasks_from: pre_auth + when: splunk_noah_enabled | default(false) | bool + - include_tasks: enable_admin_auth.yml - include_tasks: configure_mgmt_port.yml @@ -138,6 +144,36 @@ when: "'conf' in splunk and splunk.conf" no_log: "{{ hide_password }}" +- name: Configure the Noah client role before full splunkd startup + include_role: + name: splunk_noah + tasks_from: post_config + when: splunk_noah_enabled | default(false) | bool + +- name: Check SHC participant process state before declarative configuration + include_tasks: get_splunk_status.yml + when: + - ansible_system is match("Linux") + - splunk.role in ["splunk_search_head", "splunk_search_head_captain", "splunk_deployer"] + - splunk_search_head_cluster | bool + +- name: Configure SHC before splunkd starts + include_tasks: configure_shc_prestart.yml + when: + - ansible_system is match("Linux") + - splunk.role == "splunk_search_head" or splunk.role == "splunk_search_head_captain" + - splunk_search_head_cluster | bool + - splunk_status.rc != 0 + +- name: Configure SHC deployer before splunkd starts + include_tasks: configure_deployer_prestart.yml + when: + - ansible_system is match("Linux") + - splunk.role == "splunk_deployer" + - splunk_search_head_cluster | bool + - first_run | bool + - splunk_status.rc != 0 + # Generate outputs.conf before splunk starts to prevent data being indexed locally - include_tasks: enable_forwarding.yml when: diff --git a/roles/splunk_common/tasks/peer_cluster_master.yml b/roles/splunk_common/tasks/peer_cluster_master.yml index 0d19f1c8..aefaef97 100644 --- a/roles/splunk_common/tasks/peer_cluster_master.yml +++ b/roles/splunk_common/tasks/peer_cluster_master.yml @@ -2,6 +2,7 @@ - include_tasks: ../../../roles/splunk_common/tasks/wait_for_splunk_instance.yml vars: splunk_instance_address: "{{ splunk.cluster_master_url }}" + when: not (shc_prestart_indexer_peer_configured | default(false) | bool) # wait_for_splunk_instance.yml above only confirms the cluster manager's # management port answers HTTP; it can return 200 while the manager is @@ -10,14 +11,18 @@ # cluster-specific endpoint to be ready too, so the join below isn't # attempted against a manager that can't yet service it. - include_tasks: ../../../roles/splunk_common/tasks/wait_for_cluster_manager_ready.yml - when: not uds_enabled | bool + when: + - not uds_enabled | bool + - not (shc_prestart_indexer_peer_configured | default(false) | bool) # http://docs.splunk.com/Documentation/Splunk/latest/DistSearch/SHCandindexercluster#Integrate_with_a_single-site_indexer_cluster - name: Peer cluster master TCP command: "{{ splunk.exec }} edit cluster-config -mode searchhead -master_uri {{ cert_prefix }}://{{ splunk.cluster_master_url }}:{{ splunk.svc_port }} -replication_port {{ splunk.idxc.replication_port }} -secret '{{ splunk.idxc.pass4SymmKey }}' -auth '{{ splunk.admin_user }}:{{ splunk.password }}'" become: yes become_user: "{{ splunk.user }}" - when: not uds_enabled | bool + when: + - not uds_enabled | bool + - not (shc_prestart_indexer_peer_configured | default(false) | bool) register: peer_cluster_master until: peer_cluster_master.rc == 0 or "Cannot edit this searchhead" in peer_cluster_master.stderr changed_when: peer_cluster_master.rc == 0 @@ -33,7 +38,9 @@ command: "{{ splunk.exec }} edit cluster-config -mode searchhead -master_uri http+unix://{{ splunk.uds_socket_path_url }} -replication_port {{ splunk.idxc.replication_port }} -secret '{{ splunk.idxc.pass4SymmKey }}' -auth '{{ splunk.admin_user }}:{{ splunk.password }}'" become: yes become_user: "{{ splunk.user }}" - when: uds_enabled | bool + when: + - uds_enabled | bool + - not (shc_prestart_indexer_peer_configured | default(false) | bool) register: peer_cluster_master until: peer_cluster_master.rc == 0 or "Cannot edit this searchhead" in peer_cluster_master.stderr changed_when: peer_cluster_master.rc == 0 diff --git a/roles/splunk_common/tasks/set_server_name.yml b/roles/splunk_common/tasks/set_server_name.yml index 72a31c1b..19bd1a7a 100644 --- a/roles/splunk_common/tasks/set_server_name.yml +++ b/roles/splunk_common/tasks/set_server_name.yml @@ -22,7 +22,7 @@ - ini_file: dest: "{{ splunk.home }}/etc/system/local/server.conf" - section: shcclustering + section: shclustering option: "{{ item }}" value: "{{ splunk.server_name }}" owner: "{{ splunk.user }}" @@ -36,4 +36,4 @@ register: set_shcclustering_server_name - include_tasks: ../handlers/restart_splunk.yml - when: set_server_name is changed or set_clustering_server_name is changed or set_shcclustering_server_name is changed \ No newline at end of file + when: set_server_name is changed or set_clustering_server_name is changed or set_shcclustering_server_name is changed diff --git a/roles/splunk_deployer/tasks/main.yml b/roles/splunk_deployer/tasks/main.yml index 6255ebae..68654e02 100644 --- a/roles/splunk_deployer/tasks/main.yml +++ b/roles/splunk_deployer/tasks/main.yml @@ -17,6 +17,7 @@ no_log: "{{ hide_password }}" notify: - Restart the splunkd service + when: not (deployer_prestart_configured | default(false) | bool) # https://docs.splunk.com/Documentation/Splunk/latest/DistSearch/PropagateSHCconfigurationchanges#Set_the_deployer_push_mode - name: Set deployer push mode diff --git a/roles/splunk_indexer/tasks/indexer_clustering.yml b/roles/splunk_indexer/tasks/indexer_clustering.yml index e7b2991e..4ee23e2f 100644 --- a/roles/splunk_indexer/tasks/indexer_clustering.yml +++ b/roles/splunk_indexer/tasks/indexer_clustering.yml @@ -5,9 +5,28 @@ # CSPL-4006: Check if SSL replication port is configured in server.conf # If SSL port exists, we should NOT use -replication_port flag as it will overwrite the SSL config +- name: Extract server.conf content from splunk.conf (dict format) + set_fact: + idxc_server_conf_content: "{{ splunk.conf.server.content }}" + when: + - "'conf' in splunk and splunk.conf" + - splunk.conf is mapping + - "'server' in splunk.conf" + - "'content' in splunk.conf.server" + +- name: Extract server.conf content from splunk.conf (list format) + set_fact: + idxc_server_conf_content: >- + {{ (splunk.conf | selectattr('key', 'equalto', 'server') | first | default({})).get('value', {}).get('content', {}) }} + when: + - "'conf' in splunk and splunk.conf" + - splunk.conf is not mapping + - name: Check if SSL replication port is configured in server.conf set_fact: - has_ssl_replication_port_in_conf: "{{ 'conf' in splunk and 'server' in splunk.conf and 'content' in splunk.conf.server and (splunk.conf.server.content.keys() | select('match', '^replication_port-ssl://') | list | length > 0) }}" + has_ssl_replication_port_in_conf: >- + {{ (idxc_server_conf_content | default({})).keys() + | select('match', '^replication_port-ssl://') | list | length > 0 }} # Determine if SSL replication should be used (either via flag or detected in config) - name: Determine SSL replication mode @@ -51,11 +70,9 @@ vars: conf_file: "server.conf" conf_directory: "{{ splunk.home }}/etc/system/local" - conf_stanzas: "{{ splunk.conf.server.content }}" + conf_stanzas: "{{ idxc_server_conf_content }}" when: - has_ssl_replication_port_in_conf | default(false) | bool - - "'conf' in splunk" - - "'server' in splunk.conf" - - "'content' in splunk.conf.server" + - idxc_server_conf_content | default({}) | length > 0 #INFRA-38882: Update \ No newline at end of file diff --git a/roles/splunk_indexer/tasks/setup_multisite.yml b/roles/splunk_indexer/tasks/setup_multisite.yml index 7646e739..3392c75a 100644 --- a/roles/splunk_indexer/tasks/setup_multisite.yml +++ b/roles/splunk_indexer/tasks/setup_multisite.yml @@ -9,9 +9,28 @@ # CSPL-4006: Check if SSL replication port is configured in server.conf # If SSL port exists, we should NOT use -replication_port flag as it will overwrite the SSL config +- name: Extract server.conf content from splunk.conf (dict format, multisite) + set_fact: + multisite_server_conf_content: "{{ splunk.conf.server.content }}" + when: + - "'conf' in splunk and splunk.conf" + - splunk.conf is mapping + - "'server' in splunk.conf" + - "'content' in splunk.conf.server" + +- name: Extract server.conf content from splunk.conf (list format, multisite) + set_fact: + multisite_server_conf_content: >- + {{ (splunk.conf | selectattr('key', 'equalto', 'server') | first | default({})).get('value', {}).get('content', {}) }} + when: + - "'conf' in splunk and splunk.conf" + - splunk.conf is not mapping + - name: Check if SSL replication port is configured in server.conf (multisite) set_fact: - has_ssl_replication_port_in_conf_multisite: "{{ 'conf' in splunk and 'server' in splunk.conf and 'content' in splunk.conf.server and (splunk.conf.server.content.keys() | select('match', '^replication_port-ssl://') | list | length > 0) }}" + has_ssl_replication_port_in_conf_multisite: >- + {{ (multisite_server_conf_content | default({})).keys() + | select('match', '^replication_port-ssl://') | list | length > 0 }} # Determine if SSL replication should be used (either via flag or detected in config) - name: Determine SSL replication mode (multisite) @@ -53,11 +72,9 @@ vars: conf_file: "server.conf" conf_directory: "{{ splunk.home }}/etc/system/local" - conf_stanzas: "{{ splunk.conf.server.content }}" + conf_stanzas: "{{ multisite_server_conf_content }}" when: - has_ssl_replication_port_in_conf_multisite | default(false) | bool - - "'conf' in splunk" - - "'server' in splunk.conf" - - "'content' in splunk.conf.server" + - multisite_server_conf_content | default({}) | length > 0 #INFRA-38882: Update \ No newline at end of file diff --git a/roles/splunk_noah/README.md b/roles/splunk_noah/README.md new file mode 100644 index 00000000..f4888e26 --- /dev/null +++ b/roles/splunk_noah/README.md @@ -0,0 +1,36 @@ +# Noah provisioning role + +Noah mode is an explicit layer on top of the existing `splunk.role`. It is +enabled only by `SPLUNK_NOAH_ENABLED=true`; adding a `[noahService]` stanza +does not activate this role. + +| `splunk.role` | Noah client behavior | Heartbeats | Peer/bucket-map discovery | +| --- | --- | --- | --- | +| `splunk_indexer` | Peer | Enabled | Disabled | +| `splunk_search_head` | Search client | Disabled | Enabled | +| `splunk_deployer` | None | Disabled | Disabled | + +Provisioning has two deliberate entry points: + +1. `pre_auth.yml` writes a complete but disabled `[noahService]` stanza + before docker-splunk starts its temporary authentication process. +2. `post_config.yml` selects exactly one role profile after declarative + configuration is rendered and before the first full splunkd start. + +Unsupported Splunk roles fail validation when Noah mode is enabled. With Noah +mode disabled, Noah client configuration is not written. + +The SOK controller owns the shared `[noahService]` values such as `uri`, +`tenant`, and the desired enabled state. Kubernetes identity variables provide +the indexer's advertised address. Authentication material is staged in +`server.conf` from a Kubernetes Secret; this role does not require the Noah +key in an environment variable. If the standard docker-splunk +`splunk.pass4SymmKey` value is also present, the pre-auth path writes it using +`no_log` so existing deployments retain their current behavior. + +Search-head cluster formation is deliberately shared rather than implemented +inside this Noah role. The common Linux SHC pre-start task writes the stable +member and replication configuration while splunkd is stopped for both classic +and Noah deployments. Classic mode also writes its Cluster Manager peering; +Noah mode writes only the Noah search-client settings here. The later SHC +commands form and verify the cluster without scheduling duplicate restarts. diff --git a/roles/splunk_noah/defaults/main.yml b/roles/splunk_noah/defaults/main.yml new file mode 100644 index 00000000..21eb7e1d --- /dev/null +++ b/roles/splunk_noah/defaults/main.yml @@ -0,0 +1,15 @@ +--- +# Noah augments the existing Splunk role; it does not replace splunk.role. +splunk_noah_role_profiles: + splunk_indexer: + client_role: peer + task_file: indexer.yml + heartbeat_period: "30" + splunk_search_head: + client_role: search_client + task_file: search_head.yml + heartbeat_period: "0" + splunk_deployer: + client_role: none + task_file: deployer.yml + heartbeat_period: "0" diff --git a/roles/splunk_noah/tasks/deployer.yml b/roles/splunk_noah/tasks/deployer.yml new file mode 100644 index 00000000..862dd85b --- /dev/null +++ b/roles/splunk_noah/tasks/deployer.yml @@ -0,0 +1,14 @@ +--- +# The deployer manages SHC configuration. It is not a Noah membership or search client. +- name: Keep the deployer out of Noah membership + ini_file: + dest: "{{ splunk.home }}/etc/system/local/server.conf" + section: noahService + option: "{{ item.key }}" + value: "{{ item.value }}" + owner: "{{ splunk.user }}" + group: "{{ splunk.group }}" + loop: + - { key: disabled, value: "true" } + - { key: heartbeatPeriod, value: "{{ splunk_noah_client_profile.heartbeat_period }}" } + - { key: usePeers, value: "false" } diff --git a/roles/splunk_noah/tasks/indexer.yml b/roles/splunk_noah/tasks/indexer.yml new file mode 100644 index 00000000..b71abcee --- /dev/null +++ b/roles/splunk_noah/tasks/indexer.yml @@ -0,0 +1,31 @@ +--- +- name: Require a stable advertised address for a Noah indexer + assert: + that: + - splunk.noah_advertised_addr is defined + - splunk.noah_advertised_addr | length > 0 + fail_msg: >- + Noah indexers require POD_NAME, POD_NAMESPACE, and + SPLUNK_HEADLESS_SERVICE_NAME to construct their stable advertised address. + +- name: Configure Noah indexer membership + ini_file: + dest: "{{ splunk.home }}/etc/system/local/server.conf" + section: noahService + option: "{{ item.key }}" + value: "{{ item.value }}" + owner: "{{ splunk.user }}" + group: "{{ splunk.group }}" + loop: + - { key: disabled, value: "false" } + - { key: heartbeatPeriod, value: "{{ splunk_noah_client_profile.heartbeat_period }}" } + - { key: usePeers, value: "false" } + +- name: Configure the stable address advertised by the Noah indexer + ini_file: + dest: "{{ splunk.home }}/etc/system/local/server.conf" + section: noahService + option: advertisedAddr + value: "{{ splunk.noah_advertised_addr }}" + owner: "{{ splunk.user }}" + group: "{{ splunk.group }}" diff --git a/roles/splunk_noah/tasks/post_config.yml b/roles/splunk_noah/tasks/post_config.yml new file mode 100644 index 00000000..7626a042 --- /dev/null +++ b/roles/splunk_noah/tasks/post_config.yml @@ -0,0 +1,19 @@ +--- +# set_config_file.yml has restored the desired noahService stanza at this point. +# Apply exactly one role profile before the first full splunkd start. +- include_tasks: validate.yml + +- name: Persist the stable Kubernetes server name before splunkd starts + ini_file: + dest: "{{ splunk.home }}/etc/system/local/server.conf" + section: general + option: serverName + value: "{{ splunk.server_name }}" + owner: "{{ splunk.user }}" + group: "{{ splunk.group }}" + when: + - splunk.server_name is defined + - splunk.server_name is not none + +- name: Configure the {{ splunk_noah_client_profile.client_role }} Noah client profile + include_tasks: "{{ splunk_noah_client_profile.task_file }}" diff --git a/roles/splunk_noah/tasks/pre_auth.yml b/roles/splunk_noah/tasks/pre_auth.yml new file mode 100644 index 00000000..b91c54b8 --- /dev/null +++ b/roles/splunk_noah/tasks/pre_auth.yml @@ -0,0 +1,69 @@ +--- +# enable_admin_auth.yml starts a temporary no-auth splunkd. Keep Noah disabled +# for that process, but write every required field so its config parser is safe. +- include_tasks: validate.yml + +- name: Extract the noahService stanza from splunk.conf (dict format) + set_fact: + noah_service_stanza: >- + {{ splunk.conf.server.content.noahService }} + when: splunk.conf is mapping + +- name: Extract the noahService stanza from splunk.conf (list format) + set_fact: + noah_service_stanza: >- + {{ (splunk.conf | selectattr('key', 'equalto', 'server') | first | default({})).get('value', {}).get('content', {}).get('noahService', {}) }} + when: splunk.conf is not mapping + +- name: Validate the operator-provided Noah service configuration + assert: + that: + - noah_service_stanza is mapping + - noah_service_stanza | length > 0 + - noah_service_stanza.get('uri', '') | length > 0 + - noah_service_stanza.get('tenant', '') | length > 0 + fail_msg: >- + SPLUNK_NOAH_ENABLED=true requires splunk.conf to contain a server entry + with content.noahService including at least 'uri' and 'tenant'. + +- name: Write Noah service configuration for temporary authentication startup + ini_file: + dest: "{{ splunk.home }}/etc/system/local/server.conf" + section: noahService + option: "{{ item.key }}" + value: "{{ 'true' if item.key == 'disabled' else item.value }}" + owner: "{{ splunk.user }}" + group: "{{ splunk.group }}" + with_dict: "{{ noah_service_stanza }}" + no_log: "{{ hide_password }}" + +- name: Keep Noah disabled during temporary authentication startup + ini_file: + dest: "{{ splunk.home }}/etc/system/local/server.conf" + section: noahService + option: disabled + value: "true" + owner: "{{ splunk.user }}" + group: "{{ splunk.group }}" + +- name: Write a parseable role-specific Noah heartbeat before authentication startup + ini_file: + dest: "{{ splunk.home }}/etc/system/local/server.conf" + section: noahService + option: heartbeatPeriod + value: "{{ splunk_noah_client_profile.heartbeat_period }}" + owner: "{{ splunk.user }}" + group: "{{ splunk.group }}" + +- name: Write the Noah symmetric key when supplied through docker-splunk defaults + ini_file: + dest: "{{ splunk.home }}/etc/system/local/server.conf" + section: noahService + option: pass4SymmKey + value: "{{ splunk.pass4SymmKey }}" + owner: "{{ splunk.user }}" + group: "{{ splunk.group }}" + when: + - splunk.pass4SymmKey is defined + - splunk.pass4SymmKey + no_log: true diff --git a/roles/splunk_noah/tasks/search_head.yml b/roles/splunk_noah/tasks/search_head.yml new file mode 100644 index 00000000..2f889fc5 --- /dev/null +++ b/roles/splunk_noah/tasks/search_head.yml @@ -0,0 +1,32 @@ +--- +- name: Validate Noah search-head bootstrap inputs + assert: + that: + - splunk.role == "splunk_search_head" + fail_msg: >- + The Noah search-client configuration is only valid for a search-head role. + +- name: Configure the Noah search client + ini_file: + dest: "{{ splunk.home }}/etc/system/local/server.conf" + section: noahService + option: "{{ item.key }}" + value: "{{ item.value }}" + owner: "{{ splunk.user }}" + group: "{{ splunk.group }}" + loop: + - { key: disabled, value: "false" } + - { key: heartbeatPeriod, value: "{{ splunk_noah_client_profile.heartbeat_period }}" } + - { key: usePeers, value: "true" } + +- name: Configure decoupled search and indexing + ini_file: + dest: "{{ splunk.home }}/etc/system/local/server.conf" + section: decouple_search_indexing + option: "{{ item.key }}" + value: "{{ item.value }}" + owner: "{{ splunk.user }}" + group: "{{ splunk.group }}" + loop: + - { key: decoupleSearchIndexing, value: "true" } + - { key: role, value: search_head } diff --git a/roles/splunk_noah/tasks/validate.yml b/roles/splunk_noah/tasks/validate.yml new file mode 100644 index 00000000..8f325af7 --- /dev/null +++ b/roles/splunk_noah/tasks/validate.yml @@ -0,0 +1,12 @@ +--- +- name: Validate the Splunk role supported by Noah mode + assert: + that: + - splunk.role in splunk_noah_role_profiles + fail_msg: >- + Noah mode does not support SPLUNK_ROLE={{ splunk.role }}. + Supported roles are splunk_indexer, splunk_search_head, and splunk_deployer. + +- name: Select the Noah client profile for this Splunk role + set_fact: + splunk_noah_client_profile: "{{ splunk_noah_role_profiles[splunk.role] }}" diff --git a/roles/splunk_search_head/tasks/main.yml b/roles/splunk_search_head/tasks/main.yml index f74c88dc..58b25cae 100644 --- a/roles/splunk_search_head/tasks/main.yml +++ b/roles/splunk_search_head/tasks/main.yml @@ -74,3 +74,5 @@ - splunk.dfs.enable | bool - include_tasks: ../../../roles/splunk_common/tasks/check_for_required_restarts.yml + when: + - not (shc_prestart_defer_initial_restart | default(false) | bool) diff --git a/roles/splunk_search_head/tasks/search_head_clustering.yml b/roles/splunk_search_head/tasks/search_head_clustering.yml index 8d181e0b..fe1a269a 100644 --- a/roles/splunk_search_head/tasks/search_head_clustering.yml +++ b/roles/splunk_search_head/tasks/search_head_clustering.yml @@ -10,11 +10,12 @@ register: task_result changed_when: task_result.rc == 0 until: task_result.rc == 0 - retries: "{{ retry_num }}" + retries: "{{ shc_sync_retry_num if (splunk_noah_enabled | default(false) | bool) else retry_num }}" delay: "{{ retry_delay }}" notify: - Restart the splunkd service no_log: "{{ hide_password }}" + when: not (shc_prestart_configured | default(false) | bool) - name: Set desired preferred captaincy splunk_api: @@ -40,6 +41,7 @@ - name: Flush restart handlers meta: flush_handlers + when: not (shc_prestart_configured | default(false) | bool) - include_tasks: ../../../roles/splunk_common/tasks/wait_for_splunk_process.yml @@ -47,6 +49,22 @@ vars: splunk_instance_address: "{{ splunk.deployer_url }}" +- name: Wait for all Noah SHC members before captain bootstrap + uri: + url: "{{ cert_prefix }}://{{ item }}:{{ splunk.svc_port }}" + method: GET + validate_certs: false + use_proxy: no + register: noah_shc_member_ready + until: noah_shc_member_ready.status == 200 + retries: "{{ shc_sync_retry_num }}" + delay: "{{ retry_delay }}" + loop: "{{ groups['splunk_search_head'] | difference([inventory_hostname]) }}" + when: + - splunk_noah_enabled | default(false) | bool + - splunk_search_head_captain | bool + no_log: "{{ hide_password }}" + #INFRA-38882: Update exec command? - name: Boostrap SHC captain command: "{{ splunk.exec }} bootstrap shcluster-captain -servers_list '{% for host in groups['splunk_search_head'] %}{{ cert_prefix }}://{{ host }}:{{ splunk.svc_port }}{% if not loop.last %},{% endif %}{% endfor %}' -auth '{{ splunk.admin_user }}:{{ splunk.password }}'" @@ -55,9 +73,11 @@ when: splunk_search_head_captain | bool register: task_result until: task_result.rc == 0 or "node seems to have already joined another cluster" in task_result.stderr - changed_when: task_result.rc == 0 + changed_when: + - task_result.rc == 0 + - not (shc_prestart_configured | default(false) | bool) failed_when: task_result.rc !=0 and "node seems to have already joined another cluster" not in task_result.stderr - retries: "{{ retry_num }}" + retries: "{{ shc_sync_retry_num if (splunk_noah_enabled | default(false) | bool) else retry_num }}" delay: "{{ retry_delay }}" notify: - Restart the splunkd service diff --git a/site.yml b/site.yml index 0f22ae12..5ccf1a10 100644 --- a/site.yml +++ b/site.yml @@ -39,7 +39,9 @@ - name: Check all instances for required restarts include_tasks: ./roles/splunk_common/tasks/check_for_required_restarts.yml - when: splunk_restart_triggered is not defined or not splunk_restart_triggered + when: + - splunk_restart_triggered is not defined or not splunk_restart_triggered + - not (shc_prestart_defer_initial_restart | default(false) | bool) - name: Flush restart handlers meta: flush_handlers diff --git a/tests/small/test_environ.py b/tests/small/test_environ.py index 1c1b08ba..eb4dbcaa 100644 --- a/tests/small/test_environ.py +++ b/tests/small/test_environ.py @@ -19,6 +19,47 @@ import environ + +@pytest.mark.parametrize(("value", "expected"), [ + ("true", True), + ("TRUE", True), + ("false", False), + (True, True), + (False, False), +]) +def test_getNoah(value, expected): + vars_scope = {"splunk_noah_enabled": False} + with patch("os.environ", new={"SPLUNK_NOAH_ENABLED": value}): + environ.getNoah(vars_scope) + assert vars_scope["splunk_noah_enabled"] is expected + + +def test_getNoah_rejects_invalid_value(): + with patch("os.environ", new={"SPLUNK_NOAH_ENABLED": "sometimes"}): + with pytest.raises(ValueError, match="must be either 'true' or 'false'"): + environ.getNoah({"splunk_noah_enabled": False}) + + +def test_getServiceName_builds_noah_advertised_address_only_in_noah_mode(): + environment = { + "POD_NAME": "idxc-site1-0", + "POD_NAMESPACE": "splunk", + "SPLUNK_HEADLESS_SERVICE_NAME": "idxc-headless", + "CLUSTER_DOMAIN": "cluster.local", + } + vars_scope = {"splunk_noah_enabled": True, "splunk": {"svc_port": 8089}} + with patch("os.environ", new=environment): + environ.getServiceName(vars_scope) + + expected_name = "idxc-site1-0.idxc-headless.splunk.svc.cluster.local" + assert vars_scope["splunk"]["server_name"] == expected_name + assert vars_scope["splunk"]["noah_advertised_addr"] == "https://{}:8089".format(expected_name) + + classic_vars = {"splunk_noah_enabled": False, "splunk": {"svc_port": 8089}} + with patch("os.environ", new=environment): + environ.getServiceName(classic_vars) + assert "noah_advertised_addr" not in classic_vars["splunk"] + @pytest.mark.parametrize(("regex", "result"), [ (r"(FOOBAR)", {"foobar": "123"}), diff --git a/tests/small/test_noah_role_configuration.py b/tests/small/test_noah_role_configuration.py new file mode 100644 index 00000000..27c06c34 --- /dev/null +++ b/tests/small/test_noah_role_configuration.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python +"""Contract tests for Noah provisioning and shared SHC startup behavior.""" + +from __future__ import absolute_import + +import os + +import yaml + + +FILE_DIR = os.path.dirname(os.path.realpath(__file__)) +REPO_DIR = os.path.join(FILE_DIR, "..", "..") + + +def load_yaml(relative_path): + with open(os.path.join(REPO_DIR, relative_path)) as stream: + return yaml.safe_load(stream) + + +def read_file(relative_path): + with open(os.path.join(REPO_DIR, relative_path)) as stream: + return stream.read() + + +def named_task(tasks, name): + return next(task for task in tasks if task.get("name") == name) + + +def test_noah_role_matrix_is_explicit_and_complete(): + defaults = load_yaml("roles/splunk_noah/defaults/main.yml") + assert defaults["splunk_noah_role_profiles"] == { + "splunk_indexer": { + "client_role": "peer", + "task_file": "indexer.yml", + "heartbeat_period": "30", + }, + "splunk_search_head": { + "client_role": "search_client", + "task_file": "search_head.yml", + "heartbeat_period": "0", + }, + "splunk_deployer": { + "client_role": "none", + "task_file": "deployer.yml", + "heartbeat_period": "0", + }, + } + + validation = read_file("roles/splunk_noah/tasks/validate.yml") + assert "splunk.role in splunk_noah_role_profiles" in validation + assert "does not support SPLUNK_ROLE" in validation + + +def test_common_role_calls_noah_at_the_two_lifecycle_boundaries(): + tasks = load_yaml("roles/splunk_common/tasks/main.yml") + pre_auth = named_task(tasks, "Configure Noah safely before temporary authentication startup") + post_config = named_task(tasks, "Configure the Noah client role before full splunkd startup") + + assert pre_auth["include_role"]["tasks_from"] == "pre_auth" + assert post_config["include_role"]["tasks_from"] == "post_config" + expected_gate = "splunk_noah_enabled | default(false) | bool" + assert pre_auth["when"] == expected_gate + assert post_config["when"] == expected_gate + pre_index = tasks.index(pre_auth) + auth_index = next(i for i, task in enumerate(tasks) if task.get("include_tasks") == "enable_admin_auth.yml") + config_index = next(i for i, task in enumerate(tasks) if task.get("include_tasks") == "set_config_file.yml") + post_index = tasks.index(post_config) + start_index = next(i for i, task in enumerate(tasks) if task.get("include_tasks") == "start_splunk.yml") + assert pre_index < auth_index + assert config_index < post_index < start_index + + +def test_common_role_configures_every_stopped_shc_before_splunkd_start(): + tasks = load_yaml("roles/splunk_common/tasks/main.yml") + prestart = named_task(tasks, "Configure SHC before splunkd starts") + + assert prestart["include_tasks"] == "configure_shc_prestart.yml" + assert "splunk_search_head_cluster | bool" in prestart["when"] + assert "splunk_noah_enabled" not in str(prestart["when"]) + assert tasks.index(prestart) < next( + i for i, task in enumerate(tasks) if task.get("include_tasks") == "start_splunk.yml" + ) + + +def test_common_role_configures_a_fresh_deployer_before_splunkd_start(): + tasks = load_yaml("roles/splunk_common/tasks/main.yml") + status = named_task(tasks, "Check SHC participant process state before declarative configuration") + prestart = named_task(tasks, "Configure SHC deployer before splunkd starts") + deployer_tasks = load_yaml("roles/splunk_deployer/tasks/main.yml") + late_reconcile = named_task(deployer_tasks, "Set deployer SHC key and label") + + assert "splunk_deployer" in status["when"][1] + assert prestart["include_tasks"] == "configure_deployer_prestart.yml" + assert "first_run | bool" in prestart["when"] + assert "splunk_status.rc != 0" in prestart["when"] + assert tasks.index(prestart) < next( + i for i, task in enumerate(tasks) if task.get("include_tasks") == "start_splunk.yml" + ) + assert "deployer_prestart_configured" in late_reconcile["when"] + + +def test_deployer_prestart_writes_and_validates_the_shc_contract(): + text = read_file("roles/splunk_common/tasks/configure_deployer_prestart.yml") + + assert 'section: "shclustering"' in text + assert 'option: "pass4SymmKey"' in text + assert 'option: "shcluster_label"' in text + assert 'deployer_prestart_configured: true' in text + assert "Restart the splunkd service" not in text + + +def test_pre_auth_keeps_noah_disabled_and_writes_a_safe_heartbeat(): + text = read_file("roles/splunk_noah/tasks/pre_auth.yml") + assert "'true' if item.key == 'disabled'" in text + assert "Keep Noah disabled during temporary authentication startup" in text + assert "splunk_noah_client_profile.heartbeat_period" in text + assert "noah_service_stanza" in text + assert "splunk.conf is mapping" in text + assert "splunk.conf is not mapping" in text + assert "selectattr('key', 'equalto', 'server')" in text + + +def test_each_supported_role_has_only_its_intended_noah_behavior(): + indexer = read_file("roles/splunk_noah/tasks/indexer.yml") + search_head = read_file("roles/splunk_noah/tasks/search_head.yml") + deployer = read_file("roles/splunk_noah/tasks/deployer.yml") + + assert "advertisedAddr" in indexer + assert 'key: usePeers, value: "false"' in indexer + assert "decouple_search_indexing" not in indexer + + assert 'key: usePeers, value: "true"' in search_head + assert "decouple_search_indexing" in search_head + assert "advertisedAddr" not in search_head + assert "shclustering" not in search_head + assert "replication_port://" not in search_head + + assert 'key: disabled, value: "true"' in deployer + assert 'key: usePeers, value: "false"' in deployer + assert "decouple_search_indexing" not in deployer + + +def test_search_head_prestart_configuration_is_shared_by_classic_and_noah(): + text = read_file("roles/splunk_common/tasks/configure_shc_prestart.yml") + + assert 'section: "shclustering"' in text + assert 'section: "replication_port://{{ splunk.shc.replication_port }}"' in text + assert 'option: "register_replication_address"' in text + assert 'option: "search_head_uri"' in text + assert "shc_prestart_configured: true" in text + assert "shc_prestart_defer_initial_restart: true" in text + assert "not (splunk_noah_enabled | default(false) | bool)" in text + assert "shcclustering" not in text + + +def test_prestart_secret_is_only_written_for_a_fresh_etc_volume(): + tasks = load_yaml("roles/splunk_common/tasks/configure_shc_prestart.yml") + secret = named_task(tasks, "Write the SHC symmetric key before the first splunkd start") + + assert secret["when"] == "first_run | bool" + assert secret["no_log"] is True + + +def test_classic_indexer_peering_is_declarative_before_initial_start(): + prestart = read_file("roles/splunk_common/tasks/configure_shc_prestart.yml") + peer_tasks = load_yaml("roles/splunk_common/tasks/peer_cluster_master.yml") + peer_tcp = named_task(peer_tasks, "Peer cluster master TCP") + + assert 'section: "clustering"' in prestart + assert 'value: "searchhead"' in prestart + assert any("shc_prestart_indexer_peer_configured" in condition for condition in peer_tcp["when"]) + + +def test_shc_retries_are_mode_specific_but_restart_elimination_is_shared(): + tasks = load_yaml("roles/splunk_search_head/tasks/search_head_clustering.yml") + initialize = named_task(tasks, "Initialize SHC cluster config") + wait_members = named_task(tasks, "Wait for all Noah SHC members before captain bootstrap") + bootstrap = named_task(tasks, "Boostrap SHC captain") + + expected_retries = "{{ shc_sync_retry_num if (splunk_noah_enabled | default(false) | bool) else retry_num }}" + assert initialize["retries"] == expected_retries + assert bootstrap["retries"] == expected_retries + assert "splunk_noah_enabled | default(false) | bool" in wait_members["when"] + assert "not (shc_prestart_configured | default(false) | bool)" in initialize["when"] + assert "not (shc_prestart_configured | default(false) | bool)" in bootstrap["changed_when"] + + +def test_initial_shc_restart_check_is_deferred_in_role_and_top_level_play(): + role_tasks = load_yaml("roles/splunk_search_head/tasks/main.yml") + restart_check = next( + task for task in role_tasks + if task.get("include_tasks") == "../../../roles/splunk_common/tasks/check_for_required_restarts.yml" + ) + site = read_file("site.yml") + + assert "not (shc_prestart_defer_initial_restart | default(false) | bool)" in restart_check["when"] + assert "not (shc_prestart_defer_initial_restart | default(false) | bool)" in site + + +def test_late_server_name_reconciliation_uses_the_real_shc_stanza(): + text = read_file("roles/splunk_common/tasks/set_server_name.yml") + + assert "section: shclustering" in text + assert "section: shcclustering" not in text + + +def test_restart_handler_keeps_the_existing_notification_contract(): + handlers = load_yaml("roles/splunk_common/handlers/main.yml") + assert handlers[0]["name"] == "Restart the splunkd service" + assert handlers[0]["include_tasks"] == "../handlers/restart_splunk.yml" + + restart_tasks = load_yaml("roles/splunk_common/handlers/restart_splunk.yml") + noah = named_task(restart_tasks, "Restart Noah-managed splunkd - Via bounded CLI stop and start") + classic = named_task(restart_tasks, "Restart classic splunkd service - Via CLI") + assert "splunk_noah_enabled | default(false) | bool" in noah["when"] + assert noah["shell"].startswith("set -e\n") + assert " restart " not in noah["shell"] + assert "not (splunk_noah_enabled | default(false) | bool)" in classic["when"] + assert " restart --answer-yes --accept-license" in classic["command"]