From bb5321b2dc1ce6fbfb29937013afbf4feb5f10b5 Mon Sep 17 00:00:00 2001 From: Vivek Reddy Date: Wed, 26 Aug 2026 19:15:07 +0000 Subject: [PATCH 1/4] feat: add explicit Noah role provisioning --- docs/ADVANCED.md | 1 + inventory/environ.py | 18 ++- inventory/splunk_defaults_linux.yml | 1 + .../splunk_common/handlers/restart_splunk.yml | 25 +++- roles/splunk_common/tasks/main.yml | 12 ++ roles/splunk_noah/README.md | 34 +++++ roles/splunk_noah/defaults/main.yml | 15 ++ roles/splunk_noah/tasks/deployer.yml | 14 ++ roles/splunk_noah/tasks/indexer.yml | 31 ++++ roles/splunk_noah/tasks/post_config.yml | 19 +++ roles/splunk_noah/tasks/pre_auth.yml | 62 ++++++++ roles/splunk_noah/tasks/search_head.yml | 83 +++++++++++ roles/splunk_noah/tasks/validate.yml | 12 ++ .../tasks/search_head_clustering.yml | 28 +++- tests/small/test_environ.py | 41 ++++++ tests/small/test_noah_role_configuration.py | 136 ++++++++++++++++++ 16 files changed, 525 insertions(+), 7 deletions(-) create mode 100644 roles/splunk_noah/README.md create mode 100644 roles/splunk_noah/defaults/main.yml create mode 100644 roles/splunk_noah/tasks/deployer.yml create mode 100644 roles/splunk_noah/tasks/indexer.yml create mode 100644 roles/splunk_noah/tasks/post_config.yml create mode 100644 roles/splunk_noah/tasks/pre_auth.yml create mode 100644 roles/splunk_noah/tasks/search_head.yml create mode 100644 roles/splunk_noah/tasks/validate.yml create mode 100644 tests/small/test_noah_role_configuration.py diff --git a/docs/ADVANCED.md b/docs/ADVANCED.md index 4341071d..ae40d76d 100644 --- a/docs/ADVANCED.md +++ b/docs/ADVANCED.md @@ -137,6 +137,7 @@ Splunk-Ansible ships with an inventory script in `inventory/environ.py`. The scr | SPLUNK_ES_SSL_ENABLEMENT | Set the ssl-enablement flag in ES. Valid values are 'auto', 'strict', and 'ignore'. Defaults to auto when present. | no | no | no | | SPLUNK_SERVICE_NAME | Used alongside `POD_NAMESPACE` and `CLUSTER_DOMAIN` to construct a k8s-supported `issuer_uri` value for oauth2 configurations | no | no | no | | SPLUNK_HEADLESS_SERVICE_NAME | Used alongside `POD_NAME`, `POD_NAMESPACE`, and `CLUSTER_DOMAIN` to construct a k8s-supported `serverName` value. This is also used for the `search_head_uri` and `register_replication_address` settings when clustering is enabled. | no | no | no | +| SPLUNK_NOAH_ENABLED | Explicitly enables Noah-specific provisioning for supported indexer, search-head, and deployer roles. Defaults to `false`; a `noahService` stanza alone does not enable Noah mode. | no | no | no | | POD_NAME | Defines the current pod name in a k8s environment | no | no | no | | POD_NAMESPACE | Defines the namespace of the current pod in a k8s environment | no | no | no | | CLUSTER_DOMAIN | Defines the domain name for DNS resolution in a k8s cluster (default: cluster.local) | no | no | no | diff --git a/inventory/environ.py b/inventory/environ.py index cc36176b..d1ab0384 100755 --- a/inventory/environ.py +++ b/inventory/environ.py @@ -130,6 +130,7 @@ def getDefaultVars(): defaultVars = loadDefaults() defaultVars["splunk"]["role"] = os.environ.get('SPLUNK_ROLE', defaultVars["splunk"].get("role") or "splunk_standalone") overrideEnvironmentVars(defaultVars) + getNoah(defaultVars) getAnsibleContext(defaultVars) getASan(defaultVars) getDisablePopups(defaultVars) @@ -192,7 +193,22 @@ def getServiceName(vars_scope): if serviceName != "" and namespace != "": vars_scope["splunk"]["issuer_uri"] = "{}.{}.svc.{}".format(serviceName, namespace, clusterDomain) if headlessServiceName != "" and namespace != "": - vars_scope["splunk"]["server_name"] = "{}.{}.{}.svc.{}".format(podName, headlessServiceName, namespace, clusterDomain) + server_name = "{}.{}.{}.svc.{}".format(podName, headlessServiceName, namespace, clusterDomain) + vars_scope["splunk"]["server_name"] = server_name + if vars_scope.get("splunk_noah_enabled", False): + vars_scope["splunk"]["noah_advertised_addr"] = "https://{}:{}".format(server_name, vars_scope["splunk"]["svc_port"]) + +def getNoah(vars_scope): + """Enable Noah provisioning only when explicitly requested.""" + value = os.environ.get("SPLUNK_NOAH_ENABLED", vars_scope.get("splunk_noah_enabled", False)) + if isinstance(value, bool): + vars_scope["splunk_noah_enabled"] = value + return + + normalized = str(value).strip().lower() + if normalized not in ("true", "false"): + raise ValueError("SPLUNK_NOAH_ENABLED must be either 'true' or 'false'") + vars_scope["splunk_noah_enabled"] = normalized == "true" def getSplunkPaths(vars_scope): """ diff --git a/inventory/splunk_defaults_linux.yml b/inventory/splunk_defaults_linux.yml index 73bfa2e0..7d6f3def 100644 --- a/inventory/splunk_defaults_linux.yml +++ b/inventory/splunk_defaults_linux.yml @@ -8,6 +8,7 @@ retry_num: 60 hide_password: false wait_for_splunk_retry_num: 60 shc_sync_retry_num: 60 +splunk_noah_enabled: false config: max_retries: 3 diff --git a/roles/splunk_common/handlers/restart_splunk.yml b/roles/splunk_common/handlers/restart_splunk.yml index 118ede21..8c0fc794 100644 --- a/roles/splunk_common/handlers/restart_splunk.yml +++ b/roles/splunk_common/handlers/restart_splunk.yml @@ -1,5 +1,24 @@ --- -- name: "Restart the splunkd service - Via CLI" +- name: "Restart Noah-managed splunkd - Via bounded CLI stop and start" + shell: | + set -e + if {{ splunk.exec }} status >/dev/null 2>&1; then + {{ splunk.exec }} stop --answer-yes + fi + {{ splunk.exec }} start --answer-yes --accept-license + args: + executable: /bin/sh + become: yes + become_user: "{{ splunk.user }}" + register: task_result + until: task_result.rc == 0 + retries: "{{ retry_num }}" + delay: "{{ restart_retry_delay }}" + when: + - not splunk.enable_service + - splunk_noah_enabled | default(false) | bool + +- name: "Restart classic splunkd service - Via CLI" command: "{{ splunk.exec }} restart --answer-yes --accept-license" become: yes become_user: "{{ splunk.user }}" @@ -7,7 +26,9 @@ until: task_result.rc == 0 retries: "{{ retry_num }}" delay: "{{ restart_retry_delay }}" - when: not splunk.enable_service + when: + - not splunk.enable_service + - not (splunk_noah_enabled | default(false) | bool) - name: "Restart the splunkd service - Via Linux systemd or init" service: diff --git a/roles/splunk_common/tasks/main.yml b/roles/splunk_common/tasks/main.yml index fdc0fc91..a62ea05d 100644 --- a/roles/splunk_common/tasks/main.yml +++ b/roles/splunk_common/tasks/main.yml @@ -58,6 +58,12 @@ - include_tasks: set_general_symmkey_password.yml when: "'pass4SymmKey' in splunk and splunk.pass4SymmKey" +- name: Configure Noah safely before temporary authentication startup + include_role: + name: splunk_noah + tasks_from: pre_auth + when: splunk_noah_enabled | default(false) | bool + - include_tasks: enable_admin_auth.yml - include_tasks: configure_mgmt_port.yml @@ -138,6 +144,12 @@ when: "'conf' in splunk and splunk.conf" no_log: "{{ hide_password }}" +- name: Configure the Noah client role before full splunkd startup + include_role: + name: splunk_noah + tasks_from: post_config + when: splunk_noah_enabled | default(false) | bool + # Generate outputs.conf before splunk starts to prevent data being indexed locally - include_tasks: enable_forwarding.yml when: diff --git a/roles/splunk_noah/README.md b/roles/splunk_noah/README.md new file mode 100644 index 00000000..2b659b80 --- /dev/null +++ b/roles/splunk_noah/README.md @@ -0,0 +1,34 @@ +# Noah provisioning role + +Noah mode is an explicit layer on top of the existing `splunk.role`. It is +enabled only by `SPLUNK_NOAH_ENABLED=true`; adding a `[noahService]` stanza +does not activate this role. + +| `splunk.role` | Noah client behavior | Heartbeats | Peer/bucket-map discovery | +| --- | --- | --- | --- | +| `splunk_indexer` | Peer | Enabled | Disabled | +| `splunk_search_head` | Search client | Disabled | Enabled | +| `splunk_deployer` | None | Disabled | Disabled | + +Provisioning has two deliberate entry points: + +1. `pre_auth.yml` writes a complete but disabled `[noahService]` stanza + before docker-splunk starts its temporary authentication process. +2. `post_config.yml` selects exactly one role profile after declarative + configuration is rendered and before the first full splunkd start. + +Unsupported Splunk roles fail validation when Noah mode is enabled. With Noah +mode disabled, existing classic provisioning follows its original path. + +The SOK controller owns the shared `[noahService]` values such as `uri`, +`tenant`, and the desired enabled state. Kubernetes identity variables provide +the indexer's advertised address. Authentication material is staged in +`server.conf` from a Kubernetes Secret; this role does not require the Noah +key in an environment variable. If the standard docker-splunk +`splunk.pass4SymmKey` value is also present, the pre-auth path writes it using +`no_log` so existing deployments retain their current behavior. + +Search-head cluster settings remain in the search-head lifecycle role. This +role only writes their first-start configuration before splunkd starts, which +lets the existing SHC commands verify or form the cluster without an +unnecessary configuration restart. diff --git a/roles/splunk_noah/defaults/main.yml b/roles/splunk_noah/defaults/main.yml new file mode 100644 index 00000000..21eb7e1d --- /dev/null +++ b/roles/splunk_noah/defaults/main.yml @@ -0,0 +1,15 @@ +--- +# Noah augments the existing Splunk role; it does not replace splunk.role. +splunk_noah_role_profiles: + splunk_indexer: + client_role: peer + task_file: indexer.yml + heartbeat_period: "30" + splunk_search_head: + client_role: search_client + task_file: search_head.yml + heartbeat_period: "0" + splunk_deployer: + client_role: none + task_file: deployer.yml + heartbeat_period: "0" diff --git a/roles/splunk_noah/tasks/deployer.yml b/roles/splunk_noah/tasks/deployer.yml new file mode 100644 index 00000000..862dd85b --- /dev/null +++ b/roles/splunk_noah/tasks/deployer.yml @@ -0,0 +1,14 @@ +--- +# The deployer manages SHC configuration. It is not a Noah membership or search client. +- name: Keep the deployer out of Noah membership + ini_file: + dest: "{{ splunk.home }}/etc/system/local/server.conf" + section: noahService + option: "{{ item.key }}" + value: "{{ item.value }}" + owner: "{{ splunk.user }}" + group: "{{ splunk.group }}" + loop: + - { key: disabled, value: "true" } + - { key: heartbeatPeriod, value: "{{ splunk_noah_client_profile.heartbeat_period }}" } + - { key: usePeers, value: "false" } diff --git a/roles/splunk_noah/tasks/indexer.yml b/roles/splunk_noah/tasks/indexer.yml new file mode 100644 index 00000000..b71abcee --- /dev/null +++ b/roles/splunk_noah/tasks/indexer.yml @@ -0,0 +1,31 @@ +--- +- name: Require a stable advertised address for a Noah indexer + assert: + that: + - splunk.noah_advertised_addr is defined + - splunk.noah_advertised_addr | length > 0 + fail_msg: >- + Noah indexers require POD_NAME, POD_NAMESPACE, and + SPLUNK_HEADLESS_SERVICE_NAME to construct their stable advertised address. + +- name: Configure Noah indexer membership + ini_file: + dest: "{{ splunk.home }}/etc/system/local/server.conf" + section: noahService + option: "{{ item.key }}" + value: "{{ item.value }}" + owner: "{{ splunk.user }}" + group: "{{ splunk.group }}" + loop: + - { key: disabled, value: "false" } + - { key: heartbeatPeriod, value: "{{ splunk_noah_client_profile.heartbeat_period }}" } + - { key: usePeers, value: "false" } + +- name: Configure the stable address advertised by the Noah indexer + ini_file: + dest: "{{ splunk.home }}/etc/system/local/server.conf" + section: noahService + option: advertisedAddr + value: "{{ splunk.noah_advertised_addr }}" + owner: "{{ splunk.user }}" + group: "{{ splunk.group }}" diff --git a/roles/splunk_noah/tasks/post_config.yml b/roles/splunk_noah/tasks/post_config.yml new file mode 100644 index 00000000..7626a042 --- /dev/null +++ b/roles/splunk_noah/tasks/post_config.yml @@ -0,0 +1,19 @@ +--- +# set_config_file.yml has restored the desired noahService stanza at this point. +# Apply exactly one role profile before the first full splunkd start. +- include_tasks: validate.yml + +- name: Persist the stable Kubernetes server name before splunkd starts + ini_file: + dest: "{{ splunk.home }}/etc/system/local/server.conf" + section: general + option: serverName + value: "{{ splunk.server_name }}" + owner: "{{ splunk.user }}" + group: "{{ splunk.group }}" + when: + - splunk.server_name is defined + - splunk.server_name is not none + +- name: Configure the {{ splunk_noah_client_profile.client_role }} Noah client profile + include_tasks: "{{ splunk_noah_client_profile.task_file }}" diff --git a/roles/splunk_noah/tasks/pre_auth.yml b/roles/splunk_noah/tasks/pre_auth.yml new file mode 100644 index 00000000..b3207bc5 --- /dev/null +++ b/roles/splunk_noah/tasks/pre_auth.yml @@ -0,0 +1,62 @@ +--- +# enable_admin_auth.yml starts a temporary no-auth splunkd. Keep Noah disabled +# for that process, but write every required field so its config parser is safe. +- include_tasks: validate.yml + +- name: Validate the operator-provided Noah service configuration + assert: + that: + - >- + (((splunk.conf | default({})).get('server', {})).get('content', {})).get('noahService') + is mapping + - >- + (((splunk.conf | default({})).get('server', {})).get('content', {})).get('noahService', {}).get('uri', '') + | length > 0 + - >- + (((splunk.conf | default({})).get('server', {})).get('content', {})).get('noahService', {}).get('tenant', '') + | length > 0 + fail_msg: >- + SPLUNK_NOAH_ENABLED=true requires splunk.conf.server.content.noahService + in the docker-splunk defaults. + +- name: Write Noah service configuration for temporary authentication startup + ini_file: + dest: "{{ splunk.home }}/etc/system/local/server.conf" + section: noahService + option: "{{ item.key }}" + value: "{{ 'true' if item.key == 'disabled' else item.value }}" + owner: "{{ splunk.user }}" + group: "{{ splunk.group }}" + with_dict: "{{ splunk.conf.server.content.noahService }}" + no_log: "{{ hide_password }}" + +- name: Keep Noah disabled during temporary authentication startup + ini_file: + dest: "{{ splunk.home }}/etc/system/local/server.conf" + section: noahService + option: disabled + value: "true" + owner: "{{ splunk.user }}" + group: "{{ splunk.group }}" + +- name: Write a parseable role-specific Noah heartbeat before authentication startup + ini_file: + dest: "{{ splunk.home }}/etc/system/local/server.conf" + section: noahService + option: heartbeatPeriod + value: "{{ splunk_noah_client_profile.heartbeat_period }}" + owner: "{{ splunk.user }}" + group: "{{ splunk.group }}" + +- name: Write the Noah symmetric key when supplied through docker-splunk defaults + ini_file: + dest: "{{ splunk.home }}/etc/system/local/server.conf" + section: noahService + option: pass4SymmKey + value: "{{ splunk.pass4SymmKey }}" + owner: "{{ splunk.user }}" + group: "{{ splunk.group }}" + when: + - splunk.pass4SymmKey is defined + - splunk.pass4SymmKey + no_log: true diff --git a/roles/splunk_noah/tasks/search_head.yml b/roles/splunk_noah/tasks/search_head.yml new file mode 100644 index 00000000..c34f4dcd --- /dev/null +++ b/roles/splunk_noah/tasks/search_head.yml @@ -0,0 +1,83 @@ +--- +- name: Validate Noah search-head bootstrap inputs + assert: + that: + - splunk.server_name is defined + - splunk.server_name | length > 0 + - splunk.hostname is defined + - splunk.deployer_url is defined + - splunk.deployer_url | length > 0 + - splunk.shc is defined + - splunk.shc.pass4SymmKey is defined + - splunk.shc.pass4SymmKey + - splunk.shc.label is defined + - splunk.shc.replication_port is defined + - splunk.shc.replication_factor is defined + fail_msg: >- + Noah search heads require stable Kubernetes identity plus the deployer, + SHC symmetric key, label, replication port, and replication factor. + +- name: Configure the Noah search client + ini_file: + dest: "{{ splunk.home }}/etc/system/local/server.conf" + section: noahService + option: "{{ item.key }}" + value: "{{ item.value }}" + owner: "{{ splunk.user }}" + group: "{{ splunk.group }}" + loop: + - { key: disabled, value: "false" } + - { key: heartbeatPeriod, value: "{{ splunk_noah_client_profile.heartbeat_period }}" } + - { key: usePeers, value: "true" } + +- name: Configure decoupled search and indexing + ini_file: + dest: "{{ splunk.home }}/etc/system/local/server.conf" + section: decouple_search_indexing + option: "{{ item.key }}" + value: "{{ item.value }}" + owner: "{{ splunk.user }}" + group: "{{ splunk.group }}" + loop: + - { key: decoupleSearchIndexing, value: "true" } + - { key: role, value: search_head } + +- name: Write the SHC symmetric key before splunkd starts + ini_file: + dest: "{{ splunk.home }}/etc/system/local/server.conf" + section: shclustering + option: pass4SymmKey + value: "{{ splunk.shc.pass4SymmKey }}" + owner: "{{ splunk.user }}" + group: "{{ splunk.group }}" + no_log: true + +- name: Configure stable SHC identity and formation before splunkd starts + ini_file: + dest: "{{ splunk.home }}/etc/system/local/server.conf" + section: shclustering + option: "{{ item.key }}" + value: "{{ item.value }}" + owner: "{{ splunk.user }}" + group: "{{ splunk.group }}" + loop: + - { key: register_replication_address, value: "{{ splunk.server_name }}" } + - { key: search_head_uri, value: "{{ splunk.server_name }}" } + - { key: shcluster_label, value: "{{ splunk.shc.label }}" } + - { key: replication_factor, value: "{{ splunk.shc.replication_factor }}" } + - { key: conf_deploy_fetch_url, value: "{{ cert_prefix }}://{{ splunk.deployer_url }}:{{ splunk.svc_port }}" } + - { key: mgmt_uri, value: "{{ cert_prefix }}://{{ splunk.hostname }}:{{ splunk.svc_port }}" } + - { key: disabled, value: "false" } + +- name: Configure the SHC replication listener before splunkd starts + ini_file: + dest: "{{ splunk.home }}/etc/system/local/server.conf" + section: "replication_port://{{ splunk.shc.replication_port }}" + option: disabled + value: "false" + owner: "{{ splunk.user }}" + group: "{{ splunk.group }}" + +- name: Record that Noah SHC configuration was completed before startup + set_fact: + noah_shc_prestart_configured: true diff --git a/roles/splunk_noah/tasks/validate.yml b/roles/splunk_noah/tasks/validate.yml new file mode 100644 index 00000000..8f325af7 --- /dev/null +++ b/roles/splunk_noah/tasks/validate.yml @@ -0,0 +1,12 @@ +--- +- name: Validate the Splunk role supported by Noah mode + assert: + that: + - splunk.role in splunk_noah_role_profiles + fail_msg: >- + Noah mode does not support SPLUNK_ROLE={{ splunk.role }}. + Supported roles are splunk_indexer, splunk_search_head, and splunk_deployer. + +- name: Select the Noah client profile for this Splunk role + set_fact: + splunk_noah_client_profile: "{{ splunk_noah_role_profiles[splunk.role] }}" diff --git a/roles/splunk_search_head/tasks/search_head_clustering.yml b/roles/splunk_search_head/tasks/search_head_clustering.yml index 8d181e0b..3a40a4d9 100644 --- a/roles/splunk_search_head/tasks/search_head_clustering.yml +++ b/roles/splunk_search_head/tasks/search_head_clustering.yml @@ -8,9 +8,11 @@ become: yes become_user: "{{ splunk.user }}" register: task_result - changed_when: task_result.rc == 0 + changed_when: + - task_result.rc == 0 + - not (noah_shc_prestart_configured | default(false) | bool) until: task_result.rc == 0 - retries: "{{ retry_num }}" + retries: "{{ shc_sync_retry_num if (splunk_noah_enabled | default(false) | bool) else retry_num }}" delay: "{{ retry_delay }}" notify: - Restart the splunkd service @@ -47,6 +49,22 @@ vars: splunk_instance_address: "{{ splunk.deployer_url }}" +- name: Wait for all Noah SHC members before captain bootstrap + uri: + url: "{{ cert_prefix }}://{{ item }}:{{ splunk.svc_port }}" + method: GET + validate_certs: false + use_proxy: no + register: noah_shc_member_ready + until: noah_shc_member_ready.status == 200 + retries: "{{ shc_sync_retry_num }}" + delay: "{{ retry_delay }}" + loop: "{{ groups['splunk_search_head'] | difference([inventory_hostname]) }}" + when: + - splunk_noah_enabled | default(false) | bool + - splunk_search_head_captain | bool + no_log: "{{ hide_password }}" + #INFRA-38882: Update exec command? - name: Boostrap SHC captain command: "{{ splunk.exec }} bootstrap shcluster-captain -servers_list '{% for host in groups['splunk_search_head'] %}{{ cert_prefix }}://{{ host }}:{{ splunk.svc_port }}{% if not loop.last %},{% endif %}{% endfor %}' -auth '{{ splunk.admin_user }}:{{ splunk.password }}'" @@ -55,9 +73,11 @@ when: splunk_search_head_captain | bool register: task_result until: task_result.rc == 0 or "node seems to have already joined another cluster" in task_result.stderr - changed_when: task_result.rc == 0 + changed_when: + - task_result.rc == 0 + - not (noah_shc_prestart_configured | default(false) | bool) failed_when: task_result.rc !=0 and "node seems to have already joined another cluster" not in task_result.stderr - retries: "{{ retry_num }}" + retries: "{{ shc_sync_retry_num if (splunk_noah_enabled | default(false) | bool) else retry_num }}" delay: "{{ retry_delay }}" notify: - Restart the splunkd service diff --git a/tests/small/test_environ.py b/tests/small/test_environ.py index 1c1b08ba..eb4dbcaa 100644 --- a/tests/small/test_environ.py +++ b/tests/small/test_environ.py @@ -19,6 +19,47 @@ import environ + +@pytest.mark.parametrize(("value", "expected"), [ + ("true", True), + ("TRUE", True), + ("false", False), + (True, True), + (False, False), +]) +def test_getNoah(value, expected): + vars_scope = {"splunk_noah_enabled": False} + with patch("os.environ", new={"SPLUNK_NOAH_ENABLED": value}): + environ.getNoah(vars_scope) + assert vars_scope["splunk_noah_enabled"] is expected + + +def test_getNoah_rejects_invalid_value(): + with patch("os.environ", new={"SPLUNK_NOAH_ENABLED": "sometimes"}): + with pytest.raises(ValueError, match="must be either 'true' or 'false'"): + environ.getNoah({"splunk_noah_enabled": False}) + + +def test_getServiceName_builds_noah_advertised_address_only_in_noah_mode(): + environment = { + "POD_NAME": "idxc-site1-0", + "POD_NAMESPACE": "splunk", + "SPLUNK_HEADLESS_SERVICE_NAME": "idxc-headless", + "CLUSTER_DOMAIN": "cluster.local", + } + vars_scope = {"splunk_noah_enabled": True, "splunk": {"svc_port": 8089}} + with patch("os.environ", new=environment): + environ.getServiceName(vars_scope) + + expected_name = "idxc-site1-0.idxc-headless.splunk.svc.cluster.local" + assert vars_scope["splunk"]["server_name"] == expected_name + assert vars_scope["splunk"]["noah_advertised_addr"] == "https://{}:8089".format(expected_name) + + classic_vars = {"splunk_noah_enabled": False, "splunk": {"svc_port": 8089}} + with patch("os.environ", new=environment): + environ.getServiceName(classic_vars) + assert "noah_advertised_addr" not in classic_vars["splunk"] + @pytest.mark.parametrize(("regex", "result"), [ (r"(FOOBAR)", {"foobar": "123"}), diff --git a/tests/small/test_noah_role_configuration.py b/tests/small/test_noah_role_configuration.py new file mode 100644 index 00000000..af13ea84 --- /dev/null +++ b/tests/small/test_noah_role_configuration.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python +"""Contract tests for explicit Noah provisioning and classic-mode isolation.""" + +from __future__ import absolute_import + +import os + +import yaml + + +FILE_DIR = os.path.dirname(os.path.realpath(__file__)) +REPO_DIR = os.path.join(FILE_DIR, "..", "..") + + +def load_yaml(relative_path): + with open(os.path.join(REPO_DIR, relative_path)) as stream: + return yaml.safe_load(stream) + + +def read_file(relative_path): + with open(os.path.join(REPO_DIR, relative_path)) as stream: + return stream.read() + + +def named_task(tasks, name): + return next(task for task in tasks if task.get("name") == name) + + +def test_noah_role_matrix_is_explicit_and_complete(): + defaults = load_yaml("roles/splunk_noah/defaults/main.yml") + assert defaults["splunk_noah_role_profiles"] == { + "splunk_indexer": { + "client_role": "peer", + "task_file": "indexer.yml", + "heartbeat_period": "30", + }, + "splunk_search_head": { + "client_role": "search_client", + "task_file": "search_head.yml", + "heartbeat_period": "0", + }, + "splunk_deployer": { + "client_role": "none", + "task_file": "deployer.yml", + "heartbeat_period": "0", + }, + } + + validation = read_file("roles/splunk_noah/tasks/validate.yml") + assert "splunk.role in splunk_noah_role_profiles" in validation + assert "does not support SPLUNK_ROLE" in validation + + +def test_common_role_calls_noah_at_the_two_lifecycle_boundaries(): + tasks = load_yaml("roles/splunk_common/tasks/main.yml") + pre_auth = named_task(tasks, "Configure Noah safely before temporary authentication startup") + post_config = named_task(tasks, "Configure the Noah client role before full splunkd startup") + + assert pre_auth["include_role"]["tasks_from"] == "pre_auth" + assert post_config["include_role"]["tasks_from"] == "post_config" + expected_gate = "splunk_noah_enabled | default(false) | bool" + assert pre_auth["when"] == expected_gate + assert post_config["when"] == expected_gate + pre_index = tasks.index(pre_auth) + auth_index = next(i for i, task in enumerate(tasks) if task.get("include_tasks") == "enable_admin_auth.yml") + config_index = next(i for i, task in enumerate(tasks) if task.get("include_tasks") == "set_config_file.yml") + post_index = tasks.index(post_config) + start_index = next(i for i, task in enumerate(tasks) if task.get("include_tasks") == "start_splunk.yml") + assert pre_index < auth_index + assert config_index < post_index < start_index + + +def test_pre_auth_keeps_noah_disabled_and_writes_a_safe_heartbeat(): + text = read_file("roles/splunk_noah/tasks/pre_auth.yml") + assert "'true' if item.key == 'disabled'" in text + assert "Keep Noah disabled during temporary authentication startup" in text + assert "splunk_noah_client_profile.heartbeat_period" in text + assert "splunk.conf.server.content.noahService" in text + + +def test_each_supported_role_has_only_its_intended_noah_behavior(): + indexer = read_file("roles/splunk_noah/tasks/indexer.yml") + search_head = read_file("roles/splunk_noah/tasks/search_head.yml") + deployer = read_file("roles/splunk_noah/tasks/deployer.yml") + + assert "advertisedAddr" in indexer + assert 'key: usePeers, value: "false"' in indexer + assert "decouple_search_indexing" not in indexer + + assert 'key: usePeers, value: "true"' in search_head + assert "decouple_search_indexing" in search_head + assert "noah_shc_prestart_configured: true" in search_head + assert "advertisedAddr" not in search_head + + assert 'key: disabled, value: "true"' in deployer + assert 'key: usePeers, value: "false"' in deployer + assert "decouple_search_indexing" not in deployer + + +def test_search_head_uses_valid_splunk_stanzas_for_prestart_configuration(): + tasks = load_yaml("roles/splunk_noah/tasks/search_head.yml") + shc = named_task(tasks, "Configure stable SHC identity and formation before splunkd starts") + listener = named_task(tasks, "Configure the SHC replication listener before splunkd starts") + + assert shc["ini_file"]["section"] == "shclustering" + assert listener["ini_file"]["section"] == "replication_port://{{ splunk.shc.replication_port }}" + assert "shcclustering" not in read_file("roles/splunk_noah/tasks/search_head.yml") + + +def test_shc_noah_retries_are_gated_and_classic_retry_value_is_preserved(): + tasks = load_yaml("roles/splunk_search_head/tasks/search_head_clustering.yml") + initialize = named_task(tasks, "Initialize SHC cluster config") + wait_members = named_task(tasks, "Wait for all Noah SHC members before captain bootstrap") + bootstrap = named_task(tasks, "Boostrap SHC captain") + + expected_retries = "{{ shc_sync_retry_num if (splunk_noah_enabled | default(false) | bool) else retry_num }}" + assert initialize["retries"] == expected_retries + assert bootstrap["retries"] == expected_retries + assert "splunk_noah_enabled | default(false) | bool" in wait_members["when"] + assert "not (noah_shc_prestart_configured | default(false) | bool)" in initialize["changed_when"] + assert "not (noah_shc_prestart_configured | default(false) | bool)" in bootstrap["changed_when"] + + +def test_restart_handler_keeps_the_existing_notification_contract(): + handlers = load_yaml("roles/splunk_common/handlers/main.yml") + assert handlers[0]["name"] == "Restart the splunkd service" + assert handlers[0]["include_tasks"] == "../handlers/restart_splunk.yml" + + restart_tasks = load_yaml("roles/splunk_common/handlers/restart_splunk.yml") + noah = named_task(restart_tasks, "Restart Noah-managed splunkd - Via bounded CLI stop and start") + classic = named_task(restart_tasks, "Restart classic splunkd service - Via CLI") + assert "splunk_noah_enabled | default(false) | bool" in noah["when"] + assert noah["shell"].startswith("set -e\n") + assert " restart " not in noah["shell"] + assert "not (splunk_noah_enabled | default(false) | bool)" in classic["when"] + assert " restart --answer-yes --accept-license" in classic["command"] From cbfb8e8dd5a5192b4a36483039172feb97b0aea4 Mon Sep 17 00:00:00 2001 From: Vivek Reddy Date: Wed, 26 Aug 2026 12:48:00 -0700 Subject: [PATCH 2/4] fix: avoid duplicate SHC restarts in classic and Noah modes --- docs/ADVANCED.md | 8 + .../tasks/configure_shc_prestart.yml | 186 ++++++++++++++++++ roles/splunk_common/tasks/main.yml | 15 ++ .../tasks/peer_cluster_master.yml | 13 +- roles/splunk_common/tasks/set_server_name.yml | 4 +- roles/splunk_noah/README.md | 12 +- roles/splunk_noah/tasks/search_head.yml | 55 +----- roles/splunk_search_head/tasks/main.yml | 2 + .../tasks/search_head_clustering.yml | 8 +- site.yml | 4 +- tests/small/test_noah_role_configuration.py | 77 ++++++-- 11 files changed, 304 insertions(+), 80 deletions(-) create mode 100644 roles/splunk_common/tasks/configure_shc_prestart.yml diff --git a/docs/ADVANCED.md b/docs/ADVANCED.md index ae40d76d..4c18a9bd 100644 --- a/docs/ADVANCED.md +++ b/docs/ADVANCED.md @@ -142,6 +142,14 @@ Splunk-Ansible ships with an inventory script in `inventory/environ.py`. The scr | POD_NAMESPACE | Defines the namespace of the current pod in a k8s environment | no | no | no | | CLUSTER_DOMAIN | Defines the domain name for DNS resolution in a k8s cluster (default: cluster.local) | no | no | no | +For Linux search-head cluster members, Ansible now writes the SHC member, +replication-listener, stable server-name, and (for classic deployments) +Cluster Manager peering configuration while splunkd is stopped. This applies +to both classic and Noah deployments. The later SHC commands still form and +verify the cluster, but they do not request duplicate restarts when the +effective pre-start configuration already matches. If splunkd is already +running, Ansible keeps the established live-configuration and restart path. + \* Password must be set either in `default.yml` or as the environment variable `SPLUNK_PASSWORD` #### Additional Splunk Universal Forwarder variables diff --git a/roles/splunk_common/tasks/configure_shc_prestart.yml b/roles/splunk_common/tasks/configure_shc_prestart.yml new file mode 100644 index 00000000..dc5a448a --- /dev/null +++ b/roles/splunk_common/tasks/configure_shc_prestart.yml @@ -0,0 +1,186 @@ +--- +- name: Set expected pre-start SHC configuration + set_fact: + shc_prestart_server_name: "{{ splunk.server_name | default(splunk.hostname, true) }}" + shc_prestart_mgmt_uri: "{{ cert_prefix }}://{{ splunk.hostname }}:{{ splunk.svc_port }}" + shc_prestart_deployer_uri: "{{ cert_prefix }}://{{ splunk.deployer_url }}:{{ splunk.svc_port }}" + shc_prestart_server_conf: "{{ splunk.home }}/etc/system/local/server.conf" + shc_prestart_indexer_peer_enabled: >- + {{ + not (splunk_noah_enabled | default(false) | bool) + and (splunk_indexer_cluster | bool) + and (splunk.multisite_master is not defined) + and (splunk.set_search_peers | default(false) | bool) + }} + +- name: Validate pre-start SHC configuration inputs + assert: + that: + - cert_prefix in ["http", "https"] + - shc_prestart_server_name | length > 0 + - splunk.hostname | default("") | length > 0 + - splunk.deployer_url | default("") | length > 0 + - splunk.shc.pass4SymmKey | default("") | length > 0 + - splunk.shc.replication_factor | int > 0 + - splunk.shc.replication_port | int > 0 + - splunk.shc.replication_port | int <= 65535 + - splunk.shc.replication_port | int != splunk.svc_port | int + - splunk.shc.replication_port | int != splunk.http_port | int + - splunk.shc.replication_port | int != splunk.s2s.port | int + - splunk.shc.replication_port | int != splunk.kvstore.port | int + fail_msg: >- + Pre-start SHC configuration requires stable identity, valid non-conflicting + ports, a deployer endpoint, replication factor, and shared secret. + no_log: true + +- name: Validate classic pre-start indexer peering inputs + assert: + that: + - splunk.cluster_master_url | default("") | length > 0 + - splunk.idxc.pass4SymmKey | default("") | length > 0 + fail_msg: >- + Classic SHC pre-start indexer peering requires a Cluster Manager endpoint + and indexer-cluster shared secret. + when: shc_prestart_indexer_peer_enabled | bool + no_log: true + +- name: Create the local Splunk configuration directory + file: + path: "{{ splunk.home }}/etc/system/local" + state: directory + mode: 0770 + owner: "{{ splunk.user }}" + group: "{{ splunk.group }}" + become: yes + become_user: "{{ splunk.user }}" + +- name: Configure stable SHC identity before splunkd starts + ini_file: + path: "{{ shc_prestart_server_conf }}" + section: "general" + option: "serverName" + value: "{{ shc_prestart_server_name }}" + allow_no_value: false + state: present + mode: 0660 + owner: "{{ splunk.user }}" + group: "{{ splunk.group }}" + become: yes + become_user: "{{ splunk.user }}" + +- name: Configure SHC formation before splunkd starts + ini_file: + path: "{{ shc_prestart_server_conf }}" + section: "shclustering" + option: "{{ item.option }}" + value: "{{ item.value }}" + allow_no_value: false + state: present + mode: 0660 + owner: "{{ splunk.user }}" + group: "{{ splunk.group }}" + loop: + - option: "disabled" + value: "false" + - option: "mgmt_uri" + value: "{{ shc_prestart_mgmt_uri }}" + - option: "replication_factor" + value: "{{ splunk.shc.replication_factor }}" + - option: "conf_deploy_fetch_url" + value: "{{ shc_prestart_deployer_uri }}" + - option: "shcluster_label" + value: "{{ splunk.shc.label }}" + - option: "register_replication_address" + value: "{{ shc_prestart_server_name }}" + - option: "search_head_uri" + value: "{{ shc_prestart_server_name }}" + become: yes + become_user: "{{ splunk.user }}" + +- name: Write the SHC symmetric key before the first splunkd start + ini_file: + path: "{{ shc_prestart_server_conf }}" + section: "shclustering" + option: "pass4SymmKey" + value: "{{ splunk.shc.pass4SymmKey }}" + allow_no_value: false + state: present + mode: 0660 + owner: "{{ splunk.user }}" + group: "{{ splunk.group }}" + when: first_run | bool + become: yes + become_user: "{{ splunk.user }}" + no_log: true + +- name: Configure the SHC replication listener before splunkd starts + ini_file: + path: "{{ shc_prestart_server_conf }}" + section: "replication_port://{{ splunk.shc.replication_port }}" + option: "disabled" + value: "false" + allow_no_value: false + state: present + mode: 0660 + owner: "{{ splunk.user }}" + group: "{{ splunk.group }}" + become: yes + become_user: "{{ splunk.user }}" + +- name: Configure classic indexer-cluster peering before splunkd starts + ini_file: + path: "{{ shc_prestart_server_conf }}" + section: "clustering" + option: "{{ item.option }}" + value: "{{ item.value }}" + allow_no_value: false + state: present + mode: 0660 + owner: "{{ splunk.user }}" + group: "{{ splunk.group }}" + loop: + - option: "mode" + value: "searchhead" + - option: "master_uri" + value: "{{ cert_prefix }}://{{ splunk.cluster_master_url }}:{{ splunk.svc_port }}" + - option: "pass4SymmKey" + value: "{{ splunk.idxc.pass4SymmKey }}" + when: shc_prestart_indexer_peer_enabled | bool + become: yes + become_user: "{{ splunk.user }}" + no_log: true + +- name: Read effective pre-start SHC configuration + command: + argv: + - "{{ splunk.exec }}" + - "btool" + - "server" + - "list" + - "--no-log" + register: shc_prestart_btool + changed_when: false + become: yes + become_user: "{{ splunk.user }}" + no_log: true + +- name: Validate effective pre-start SHC configuration + assert: + that: + - (shc_prestart_btool.stdout | regex_search("(?m)^serverName\\s*=\\s*" ~ (shc_prestart_server_name | regex_escape) ~ "\\s*$")) is not none + - (shc_prestart_btool.stdout | regex_search("(?m)^disabled\\s*=\\s*(0|false)\\s*$")) is not none + - ("mgmt_uri = " ~ shc_prestart_mgmt_uri) in shc_prestart_btool.stdout + - ("replication_factor = " ~ (splunk.shc.replication_factor | string)) in shc_prestart_btool.stdout + - ("conf_deploy_fetch_url = " ~ shc_prestart_deployer_uri) in shc_prestart_btool.stdout + - (shc_prestart_btool.stdout | regex_search("(?m)^pass4SymmKey\\s*=\\s*\\S+\\s*$")) is not none + - ("register_replication_address = " ~ shc_prestart_server_name) in shc_prestart_btool.stdout + - ("search_head_uri = " ~ shc_prestart_server_name) in shc_prestart_btool.stdout + - ("[replication_port://" ~ (splunk.shc.replication_port | string) ~ "]") in shc_prestart_btool.stdout + fail_msg: "Effective pre-start SHC configuration does not match the requested member configuration." + no_log: true + +- name: Record declarative SHC pre-start configuration + set_fact: + shc_prestart_configured: true + shc_prestart_defer_initial_restart: true + shc_prestart_indexer_peer_configured: "{{ shc_prestart_indexer_peer_enabled | bool }}" diff --git a/roles/splunk_common/tasks/main.yml b/roles/splunk_common/tasks/main.yml index a62ea05d..7e8bde65 100644 --- a/roles/splunk_common/tasks/main.yml +++ b/roles/splunk_common/tasks/main.yml @@ -150,6 +150,21 @@ tasks_from: post_config when: splunk_noah_enabled | default(false) | bool +- name: Check SHC process state before declarative configuration + include_tasks: get_splunk_status.yml + when: + - ansible_system is match("Linux") + - splunk.role == "splunk_search_head" or splunk.role == "splunk_search_head_captain" + - splunk_search_head_cluster | bool + +- name: Configure SHC before splunkd starts + include_tasks: configure_shc_prestart.yml + when: + - ansible_system is match("Linux") + - splunk.role == "splunk_search_head" or splunk.role == "splunk_search_head_captain" + - splunk_search_head_cluster | bool + - splunk_status.rc != 0 + # Generate outputs.conf before splunk starts to prevent data being indexed locally - include_tasks: enable_forwarding.yml when: diff --git a/roles/splunk_common/tasks/peer_cluster_master.yml b/roles/splunk_common/tasks/peer_cluster_master.yml index 0d19f1c8..aefaef97 100644 --- a/roles/splunk_common/tasks/peer_cluster_master.yml +++ b/roles/splunk_common/tasks/peer_cluster_master.yml @@ -2,6 +2,7 @@ - include_tasks: ../../../roles/splunk_common/tasks/wait_for_splunk_instance.yml vars: splunk_instance_address: "{{ splunk.cluster_master_url }}" + when: not (shc_prestart_indexer_peer_configured | default(false) | bool) # wait_for_splunk_instance.yml above only confirms the cluster manager's # management port answers HTTP; it can return 200 while the manager is @@ -10,14 +11,18 @@ # cluster-specific endpoint to be ready too, so the join below isn't # attempted against a manager that can't yet service it. - include_tasks: ../../../roles/splunk_common/tasks/wait_for_cluster_manager_ready.yml - when: not uds_enabled | bool + when: + - not uds_enabled | bool + - not (shc_prestart_indexer_peer_configured | default(false) | bool) # http://docs.splunk.com/Documentation/Splunk/latest/DistSearch/SHCandindexercluster#Integrate_with_a_single-site_indexer_cluster - name: Peer cluster master TCP command: "{{ splunk.exec }} edit cluster-config -mode searchhead -master_uri {{ cert_prefix }}://{{ splunk.cluster_master_url }}:{{ splunk.svc_port }} -replication_port {{ splunk.idxc.replication_port }} -secret '{{ splunk.idxc.pass4SymmKey }}' -auth '{{ splunk.admin_user }}:{{ splunk.password }}'" become: yes become_user: "{{ splunk.user }}" - when: not uds_enabled | bool + when: + - not uds_enabled | bool + - not (shc_prestart_indexer_peer_configured | default(false) | bool) register: peer_cluster_master until: peer_cluster_master.rc == 0 or "Cannot edit this searchhead" in peer_cluster_master.stderr changed_when: peer_cluster_master.rc == 0 @@ -33,7 +38,9 @@ command: "{{ splunk.exec }} edit cluster-config -mode searchhead -master_uri http+unix://{{ splunk.uds_socket_path_url }} -replication_port {{ splunk.idxc.replication_port }} -secret '{{ splunk.idxc.pass4SymmKey }}' -auth '{{ splunk.admin_user }}:{{ splunk.password }}'" become: yes become_user: "{{ splunk.user }}" - when: uds_enabled | bool + when: + - uds_enabled | bool + - not (shc_prestart_indexer_peer_configured | default(false) | bool) register: peer_cluster_master until: peer_cluster_master.rc == 0 or "Cannot edit this searchhead" in peer_cluster_master.stderr changed_when: peer_cluster_master.rc == 0 diff --git a/roles/splunk_common/tasks/set_server_name.yml b/roles/splunk_common/tasks/set_server_name.yml index 72a31c1b..19bd1a7a 100644 --- a/roles/splunk_common/tasks/set_server_name.yml +++ b/roles/splunk_common/tasks/set_server_name.yml @@ -22,7 +22,7 @@ - ini_file: dest: "{{ splunk.home }}/etc/system/local/server.conf" - section: shcclustering + section: shclustering option: "{{ item }}" value: "{{ splunk.server_name }}" owner: "{{ splunk.user }}" @@ -36,4 +36,4 @@ register: set_shcclustering_server_name - include_tasks: ../handlers/restart_splunk.yml - when: set_server_name is changed or set_clustering_server_name is changed or set_shcclustering_server_name is changed \ No newline at end of file + when: set_server_name is changed or set_clustering_server_name is changed or set_shcclustering_server_name is changed diff --git a/roles/splunk_noah/README.md b/roles/splunk_noah/README.md index 2b659b80..f4888e26 100644 --- a/roles/splunk_noah/README.md +++ b/roles/splunk_noah/README.md @@ -18,7 +18,7 @@ Provisioning has two deliberate entry points: configuration is rendered and before the first full splunkd start. Unsupported Splunk roles fail validation when Noah mode is enabled. With Noah -mode disabled, existing classic provisioning follows its original path. +mode disabled, Noah client configuration is not written. The SOK controller owns the shared `[noahService]` values such as `uri`, `tenant`, and the desired enabled state. Kubernetes identity variables provide @@ -28,7 +28,9 @@ key in an environment variable. If the standard docker-splunk `splunk.pass4SymmKey` value is also present, the pre-auth path writes it using `no_log` so existing deployments retain their current behavior. -Search-head cluster settings remain in the search-head lifecycle role. This -role only writes their first-start configuration before splunkd starts, which -lets the existing SHC commands verify or form the cluster without an -unnecessary configuration restart. +Search-head cluster formation is deliberately shared rather than implemented +inside this Noah role. The common Linux SHC pre-start task writes the stable +member and replication configuration while splunkd is stopped for both classic +and Noah deployments. Classic mode also writes its Cluster Manager peering; +Noah mode writes only the Noah search-client settings here. The later SHC +commands form and verify the cluster without scheduling duplicate restarts. diff --git a/roles/splunk_noah/tasks/search_head.yml b/roles/splunk_noah/tasks/search_head.yml index c34f4dcd..2f889fc5 100644 --- a/roles/splunk_noah/tasks/search_head.yml +++ b/roles/splunk_noah/tasks/search_head.yml @@ -2,20 +2,9 @@ - name: Validate Noah search-head bootstrap inputs assert: that: - - splunk.server_name is defined - - splunk.server_name | length > 0 - - splunk.hostname is defined - - splunk.deployer_url is defined - - splunk.deployer_url | length > 0 - - splunk.shc is defined - - splunk.shc.pass4SymmKey is defined - - splunk.shc.pass4SymmKey - - splunk.shc.label is defined - - splunk.shc.replication_port is defined - - splunk.shc.replication_factor is defined + - splunk.role == "splunk_search_head" fail_msg: >- - Noah search heads require stable Kubernetes identity plus the deployer, - SHC symmetric key, label, replication port, and replication factor. + The Noah search-client configuration is only valid for a search-head role. - name: Configure the Noah search client ini_file: @@ -41,43 +30,3 @@ loop: - { key: decoupleSearchIndexing, value: "true" } - { key: role, value: search_head } - -- name: Write the SHC symmetric key before splunkd starts - ini_file: - dest: "{{ splunk.home }}/etc/system/local/server.conf" - section: shclustering - option: pass4SymmKey - value: "{{ splunk.shc.pass4SymmKey }}" - owner: "{{ splunk.user }}" - group: "{{ splunk.group }}" - no_log: true - -- name: Configure stable SHC identity and formation before splunkd starts - ini_file: - dest: "{{ splunk.home }}/etc/system/local/server.conf" - section: shclustering - option: "{{ item.key }}" - value: "{{ item.value }}" - owner: "{{ splunk.user }}" - group: "{{ splunk.group }}" - loop: - - { key: register_replication_address, value: "{{ splunk.server_name }}" } - - { key: search_head_uri, value: "{{ splunk.server_name }}" } - - { key: shcluster_label, value: "{{ splunk.shc.label }}" } - - { key: replication_factor, value: "{{ splunk.shc.replication_factor }}" } - - { key: conf_deploy_fetch_url, value: "{{ cert_prefix }}://{{ splunk.deployer_url }}:{{ splunk.svc_port }}" } - - { key: mgmt_uri, value: "{{ cert_prefix }}://{{ splunk.hostname }}:{{ splunk.svc_port }}" } - - { key: disabled, value: "false" } - -- name: Configure the SHC replication listener before splunkd starts - ini_file: - dest: "{{ splunk.home }}/etc/system/local/server.conf" - section: "replication_port://{{ splunk.shc.replication_port }}" - option: disabled - value: "false" - owner: "{{ splunk.user }}" - group: "{{ splunk.group }}" - -- name: Record that Noah SHC configuration was completed before startup - set_fact: - noah_shc_prestart_configured: true diff --git a/roles/splunk_search_head/tasks/main.yml b/roles/splunk_search_head/tasks/main.yml index f74c88dc..58b25cae 100644 --- a/roles/splunk_search_head/tasks/main.yml +++ b/roles/splunk_search_head/tasks/main.yml @@ -74,3 +74,5 @@ - splunk.dfs.enable | bool - include_tasks: ../../../roles/splunk_common/tasks/check_for_required_restarts.yml + when: + - not (shc_prestart_defer_initial_restart | default(false) | bool) diff --git a/roles/splunk_search_head/tasks/search_head_clustering.yml b/roles/splunk_search_head/tasks/search_head_clustering.yml index 3a40a4d9..fe1a269a 100644 --- a/roles/splunk_search_head/tasks/search_head_clustering.yml +++ b/roles/splunk_search_head/tasks/search_head_clustering.yml @@ -8,15 +8,14 @@ become: yes become_user: "{{ splunk.user }}" register: task_result - changed_when: - - task_result.rc == 0 - - not (noah_shc_prestart_configured | default(false) | bool) + changed_when: task_result.rc == 0 until: task_result.rc == 0 retries: "{{ shc_sync_retry_num if (splunk_noah_enabled | default(false) | bool) else retry_num }}" delay: "{{ retry_delay }}" notify: - Restart the splunkd service no_log: "{{ hide_password }}" + when: not (shc_prestart_configured | default(false) | bool) - name: Set desired preferred captaincy splunk_api: @@ -42,6 +41,7 @@ - name: Flush restart handlers meta: flush_handlers + when: not (shc_prestart_configured | default(false) | bool) - include_tasks: ../../../roles/splunk_common/tasks/wait_for_splunk_process.yml @@ -75,7 +75,7 @@ until: task_result.rc == 0 or "node seems to have already joined another cluster" in task_result.stderr changed_when: - task_result.rc == 0 - - not (noah_shc_prestart_configured | default(false) | bool) + - not (shc_prestart_configured | default(false) | bool) failed_when: task_result.rc !=0 and "node seems to have already joined another cluster" not in task_result.stderr retries: "{{ shc_sync_retry_num if (splunk_noah_enabled | default(false) | bool) else retry_num }}" delay: "{{ retry_delay }}" diff --git a/site.yml b/site.yml index 0f22ae12..5ccf1a10 100644 --- a/site.yml +++ b/site.yml @@ -39,7 +39,9 @@ - name: Check all instances for required restarts include_tasks: ./roles/splunk_common/tasks/check_for_required_restarts.yml - when: splunk_restart_triggered is not defined or not splunk_restart_triggered + when: + - splunk_restart_triggered is not defined or not splunk_restart_triggered + - not (shc_prestart_defer_initial_restart | default(false) | bool) - name: Flush restart handlers meta: flush_handlers diff --git a/tests/small/test_noah_role_configuration.py b/tests/small/test_noah_role_configuration.py index af13ea84..1d28aa2e 100644 --- a/tests/small/test_noah_role_configuration.py +++ b/tests/small/test_noah_role_configuration.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -"""Contract tests for explicit Noah provisioning and classic-mode isolation.""" +"""Contract tests for Noah provisioning and shared SHC startup behavior.""" from __future__ import absolute_import @@ -70,6 +70,18 @@ def test_common_role_calls_noah_at_the_two_lifecycle_boundaries(): assert config_index < post_index < start_index +def test_common_role_configures_every_stopped_shc_before_splunkd_start(): + tasks = load_yaml("roles/splunk_common/tasks/main.yml") + prestart = named_task(tasks, "Configure SHC before splunkd starts") + + assert prestart["include_tasks"] == "configure_shc_prestart.yml" + assert "splunk_search_head_cluster | bool" in prestart["when"] + assert "splunk_noah_enabled" not in str(prestart["when"]) + assert tasks.index(prestart) < next( + i for i, task in enumerate(tasks) if task.get("include_tasks") == "start_splunk.yml" + ) + + def test_pre_auth_keeps_noah_disabled_and_writes_a_safe_heartbeat(): text = read_file("roles/splunk_noah/tasks/pre_auth.yml") assert "'true' if item.key == 'disabled'" in text @@ -89,25 +101,47 @@ def test_each_supported_role_has_only_its_intended_noah_behavior(): assert 'key: usePeers, value: "true"' in search_head assert "decouple_search_indexing" in search_head - assert "noah_shc_prestart_configured: true" in search_head assert "advertisedAddr" not in search_head + assert "shclustering" not in search_head + assert "replication_port://" not in search_head assert 'key: disabled, value: "true"' in deployer assert 'key: usePeers, value: "false"' in deployer assert "decouple_search_indexing" not in deployer -def test_search_head_uses_valid_splunk_stanzas_for_prestart_configuration(): - tasks = load_yaml("roles/splunk_noah/tasks/search_head.yml") - shc = named_task(tasks, "Configure stable SHC identity and formation before splunkd starts") - listener = named_task(tasks, "Configure the SHC replication listener before splunkd starts") +def test_search_head_prestart_configuration_is_shared_by_classic_and_noah(): + text = read_file("roles/splunk_common/tasks/configure_shc_prestart.yml") + + assert 'section: "shclustering"' in text + assert 'section: "replication_port://{{ splunk.shc.replication_port }}"' in text + assert 'option: "register_replication_address"' in text + assert 'option: "search_head_uri"' in text + assert "shc_prestart_configured: true" in text + assert "shc_prestart_defer_initial_restart: true" in text + assert "not (splunk_noah_enabled | default(false) | bool)" in text + assert "shcclustering" not in text + + +def test_prestart_secret_is_only_written_for_a_fresh_etc_volume(): + tasks = load_yaml("roles/splunk_common/tasks/configure_shc_prestart.yml") + secret = named_task(tasks, "Write the SHC symmetric key before the first splunkd start") + + assert secret["when"] == "first_run | bool" + assert secret["no_log"] is True - assert shc["ini_file"]["section"] == "shclustering" - assert listener["ini_file"]["section"] == "replication_port://{{ splunk.shc.replication_port }}" - assert "shcclustering" not in read_file("roles/splunk_noah/tasks/search_head.yml") +def test_classic_indexer_peering_is_declarative_before_initial_start(): + prestart = read_file("roles/splunk_common/tasks/configure_shc_prestart.yml") + peer_tasks = load_yaml("roles/splunk_common/tasks/peer_cluster_master.yml") + peer_tcp = named_task(peer_tasks, "Peer cluster master TCP") -def test_shc_noah_retries_are_gated_and_classic_retry_value_is_preserved(): + assert 'section: "clustering"' in prestart + assert 'value: "searchhead"' in prestart + assert any("shc_prestart_indexer_peer_configured" in condition for condition in peer_tcp["when"]) + + +def test_shc_retries_are_mode_specific_but_restart_elimination_is_shared(): tasks = load_yaml("roles/splunk_search_head/tasks/search_head_clustering.yml") initialize = named_task(tasks, "Initialize SHC cluster config") wait_members = named_task(tasks, "Wait for all Noah SHC members before captain bootstrap") @@ -117,8 +151,27 @@ def test_shc_noah_retries_are_gated_and_classic_retry_value_is_preserved(): assert initialize["retries"] == expected_retries assert bootstrap["retries"] == expected_retries assert "splunk_noah_enabled | default(false) | bool" in wait_members["when"] - assert "not (noah_shc_prestart_configured | default(false) | bool)" in initialize["changed_when"] - assert "not (noah_shc_prestart_configured | default(false) | bool)" in bootstrap["changed_when"] + assert "not (shc_prestart_configured | default(false) | bool)" in initialize["when"] + assert "not (shc_prestart_configured | default(false) | bool)" in bootstrap["changed_when"] + + +def test_initial_shc_restart_check_is_deferred_in_role_and_top_level_play(): + role_tasks = load_yaml("roles/splunk_search_head/tasks/main.yml") + restart_check = next( + task for task in role_tasks + if task.get("include_tasks") == "../../../roles/splunk_common/tasks/check_for_required_restarts.yml" + ) + site = read_file("site.yml") + + assert "not (shc_prestart_defer_initial_restart | default(false) | bool)" in restart_check["when"] + assert "not (shc_prestart_defer_initial_restart | default(false) | bool)" in site + + +def test_late_server_name_reconciliation_uses_the_real_shc_stanza(): + text = read_file("roles/splunk_common/tasks/set_server_name.yml") + + assert "section: shclustering" in text + assert "section: shcclustering" not in text def test_restart_handler_keeps_the_existing_notification_contract(): From 40aa2ca51eed717f71f3d2200e9c7b571ce211ce Mon Sep 17 00:00:00 2001 From: Vivek Reddy Date: Wed, 26 Aug 2026 13:36:01 -0700 Subject: [PATCH 3/4] fix: configure SHC deployer before initial start --- .../tasks/configure_deployer_prestart.yml | 69 +++++++++++++++++++ roles/splunk_common/tasks/main.yml | 13 +++- roles/splunk_deployer/tasks/main.yml | 1 + tests/small/test_noah_role_configuration.py | 27 ++++++++ 4 files changed, 108 insertions(+), 2 deletions(-) create mode 100644 roles/splunk_common/tasks/configure_deployer_prestart.yml diff --git a/roles/splunk_common/tasks/configure_deployer_prestart.yml b/roles/splunk_common/tasks/configure_deployer_prestart.yml new file mode 100644 index 00000000..7ad86554 --- /dev/null +++ b/roles/splunk_common/tasks/configure_deployer_prestart.yml @@ -0,0 +1,69 @@ +--- +- name: Validate pre-start SHC deployer configuration inputs + assert: + that: + - splunk.shc.pass4SymmKey | default("") | length > 0 + - splunk.shc.label | default("") | length > 0 + fail_msg: >- + Pre-start SHC deployer configuration requires the SHC label and shared + symmetric key. + no_log: true + +- name: Create the local Splunk configuration directory for the SHC deployer + file: + path: "{{ splunk.home }}/etc/system/local" + state: directory + mode: 0770 + owner: "{{ splunk.user }}" + group: "{{ splunk.group }}" + become: yes + become_user: "{{ splunk.user }}" + +- name: Configure the SHC deployer before the first splunkd start + ini_file: + path: "{{ splunk.home }}/etc/system/local/server.conf" + section: "shclustering" + option: "{{ item.option }}" + value: "{{ item.value }}" + allow_no_value: false + state: present + mode: 0660 + owner: "{{ splunk.user }}" + group: "{{ splunk.group }}" + loop: + - option: "pass4SymmKey" + value: "{{ splunk.shc.pass4SymmKey }}" + - option: "shcluster_label" + value: "{{ splunk.shc.label }}" + loop_control: + label: "{{ item.option }}" + become: yes + become_user: "{{ splunk.user }}" + no_log: true + +- name: Read effective pre-start SHC deployer configuration + command: + argv: + - "{{ splunk.exec }}" + - "btool" + - "server" + - "list" + - "shclustering" + - "--no-log" + register: deployer_prestart_btool + changed_when: false + become: yes + become_user: "{{ splunk.user }}" + no_log: true + +- name: Validate effective pre-start SHC deployer configuration + assert: + that: + - (deployer_prestart_btool.stdout | regex_search("(?m)^pass4SymmKey\\s*=\\s*\\S+\\s*$")) is not none + - ("shcluster_label = " ~ splunk.shc.label) in deployer_prestart_btool.stdout + fail_msg: "Effective pre-start SHC deployer configuration does not match the requested SHC." + no_log: true + +- name: Record declarative SHC deployer pre-start configuration + set_fact: + deployer_prestart_configured: true diff --git a/roles/splunk_common/tasks/main.yml b/roles/splunk_common/tasks/main.yml index 7e8bde65..362dba70 100644 --- a/roles/splunk_common/tasks/main.yml +++ b/roles/splunk_common/tasks/main.yml @@ -150,11 +150,11 @@ tasks_from: post_config when: splunk_noah_enabled | default(false) | bool -- name: Check SHC process state before declarative configuration +- name: Check SHC participant process state before declarative configuration include_tasks: get_splunk_status.yml when: - ansible_system is match("Linux") - - splunk.role == "splunk_search_head" or splunk.role == "splunk_search_head_captain" + - splunk.role in ["splunk_search_head", "splunk_search_head_captain", "splunk_deployer"] - splunk_search_head_cluster | bool - name: Configure SHC before splunkd starts @@ -165,6 +165,15 @@ - splunk_search_head_cluster | bool - splunk_status.rc != 0 +- name: Configure SHC deployer before splunkd starts + include_tasks: configure_deployer_prestart.yml + when: + - ansible_system is match("Linux") + - splunk.role == "splunk_deployer" + - splunk_search_head_cluster | bool + - first_run | bool + - splunk_status.rc != 0 + # Generate outputs.conf before splunk starts to prevent data being indexed locally - include_tasks: enable_forwarding.yml when: diff --git a/roles/splunk_deployer/tasks/main.yml b/roles/splunk_deployer/tasks/main.yml index 6255ebae..68654e02 100644 --- a/roles/splunk_deployer/tasks/main.yml +++ b/roles/splunk_deployer/tasks/main.yml @@ -17,6 +17,7 @@ no_log: "{{ hide_password }}" notify: - Restart the splunkd service + when: not (deployer_prestart_configured | default(false) | bool) # https://docs.splunk.com/Documentation/Splunk/latest/DistSearch/PropagateSHCconfigurationchanges#Set_the_deployer_push_mode - name: Set deployer push mode diff --git a/tests/small/test_noah_role_configuration.py b/tests/small/test_noah_role_configuration.py index 1d28aa2e..9a186f5c 100644 --- a/tests/small/test_noah_role_configuration.py +++ b/tests/small/test_noah_role_configuration.py @@ -82,6 +82,33 @@ def test_common_role_configures_every_stopped_shc_before_splunkd_start(): ) +def test_common_role_configures_a_fresh_deployer_before_splunkd_start(): + tasks = load_yaml("roles/splunk_common/tasks/main.yml") + status = named_task(tasks, "Check SHC participant process state before declarative configuration") + prestart = named_task(tasks, "Configure SHC deployer before splunkd starts") + deployer_tasks = load_yaml("roles/splunk_deployer/tasks/main.yml") + late_reconcile = named_task(deployer_tasks, "Set deployer SHC key and label") + + assert "splunk_deployer" in status["when"][1] + assert prestart["include_tasks"] == "configure_deployer_prestart.yml" + assert "first_run | bool" in prestart["when"] + assert "splunk_status.rc != 0" in prestart["when"] + assert tasks.index(prestart) < next( + i for i, task in enumerate(tasks) if task.get("include_tasks") == "start_splunk.yml" + ) + assert "deployer_prestart_configured" in late_reconcile["when"] + + +def test_deployer_prestart_writes_and_validates_the_shc_contract(): + text = read_file("roles/splunk_common/tasks/configure_deployer_prestart.yml") + + assert 'section: "shclustering"' in text + assert 'option: "pass4SymmKey"' in text + assert 'option: "shcluster_label"' in text + assert 'deployer_prestart_configured: true' in text + assert "Restart the splunkd service" not in text + + def test_pre_auth_keeps_noah_disabled_and_writes_a_safe_heartbeat(): text = read_file("roles/splunk_noah/tasks/pre_auth.yml") assert "'true' if item.key == 'disabled'" in text From 723b3b7d7189821fd3666927a76893306b457900 Mon Sep 17 00:00:00 2001 From: Patryk Wasielewski Date: Mon, 31 Aug 2026 15:43:07 +0200 Subject: [PATCH 4/4] refactor: support both dict and list formats for splunk.conf in Noah tasks The Noah pre_auth.yml was hardcoded to dict-format access (splunk.conf.server.content.noahService). This fails when splunk.conf is a list (the operator's native ConfFileEntry format). Fix: detect whether splunk.conf is a mapping or list and extract the noahService stanza accordingly. Apply the same dual-format pattern to the CSPL-4006 SSL replication port checks in indexer_clustering.yml and setup_multisite.yml. The existing dict2items fallback in main.yml is preserved for backward compatibility with users passing splunk.conf as a dict. Changes: - pre_auth.yml: two set_fact branches (dict vs list) for noah_service_stanza - indexer_clustering.yml: dual-format extraction for SSL replication check - setup_multisite.yml: same for multisite - test_noah_role_configuration.py: assert both format branches exist --- .../tasks/indexer_clustering.yml | 27 +++++++++++++--- .../splunk_indexer/tasks/setup_multisite.yml | 27 +++++++++++++--- roles/splunk_noah/tasks/pre_auth.yml | 31 ++++++++++++------- tests/small/test_noah_role_configuration.py | 5 ++- 4 files changed, 67 insertions(+), 23 deletions(-) diff --git a/roles/splunk_indexer/tasks/indexer_clustering.yml b/roles/splunk_indexer/tasks/indexer_clustering.yml index e7b2991e..4ee23e2f 100644 --- a/roles/splunk_indexer/tasks/indexer_clustering.yml +++ b/roles/splunk_indexer/tasks/indexer_clustering.yml @@ -5,9 +5,28 @@ # CSPL-4006: Check if SSL replication port is configured in server.conf # If SSL port exists, we should NOT use -replication_port flag as it will overwrite the SSL config +- name: Extract server.conf content from splunk.conf (dict format) + set_fact: + idxc_server_conf_content: "{{ splunk.conf.server.content }}" + when: + - "'conf' in splunk and splunk.conf" + - splunk.conf is mapping + - "'server' in splunk.conf" + - "'content' in splunk.conf.server" + +- name: Extract server.conf content from splunk.conf (list format) + set_fact: + idxc_server_conf_content: >- + {{ (splunk.conf | selectattr('key', 'equalto', 'server') | first | default({})).get('value', {}).get('content', {}) }} + when: + - "'conf' in splunk and splunk.conf" + - splunk.conf is not mapping + - name: Check if SSL replication port is configured in server.conf set_fact: - has_ssl_replication_port_in_conf: "{{ 'conf' in splunk and 'server' in splunk.conf and 'content' in splunk.conf.server and (splunk.conf.server.content.keys() | select('match', '^replication_port-ssl://') | list | length > 0) }}" + has_ssl_replication_port_in_conf: >- + {{ (idxc_server_conf_content | default({})).keys() + | select('match', '^replication_port-ssl://') | list | length > 0 }} # Determine if SSL replication should be used (either via flag or detected in config) - name: Determine SSL replication mode @@ -51,11 +70,9 @@ vars: conf_file: "server.conf" conf_directory: "{{ splunk.home }}/etc/system/local" - conf_stanzas: "{{ splunk.conf.server.content }}" + conf_stanzas: "{{ idxc_server_conf_content }}" when: - has_ssl_replication_port_in_conf | default(false) | bool - - "'conf' in splunk" - - "'server' in splunk.conf" - - "'content' in splunk.conf.server" + - idxc_server_conf_content | default({}) | length > 0 #INFRA-38882: Update \ No newline at end of file diff --git a/roles/splunk_indexer/tasks/setup_multisite.yml b/roles/splunk_indexer/tasks/setup_multisite.yml index 7646e739..3392c75a 100644 --- a/roles/splunk_indexer/tasks/setup_multisite.yml +++ b/roles/splunk_indexer/tasks/setup_multisite.yml @@ -9,9 +9,28 @@ # CSPL-4006: Check if SSL replication port is configured in server.conf # If SSL port exists, we should NOT use -replication_port flag as it will overwrite the SSL config +- name: Extract server.conf content from splunk.conf (dict format, multisite) + set_fact: + multisite_server_conf_content: "{{ splunk.conf.server.content }}" + when: + - "'conf' in splunk and splunk.conf" + - splunk.conf is mapping + - "'server' in splunk.conf" + - "'content' in splunk.conf.server" + +- name: Extract server.conf content from splunk.conf (list format, multisite) + set_fact: + multisite_server_conf_content: >- + {{ (splunk.conf | selectattr('key', 'equalto', 'server') | first | default({})).get('value', {}).get('content', {}) }} + when: + - "'conf' in splunk and splunk.conf" + - splunk.conf is not mapping + - name: Check if SSL replication port is configured in server.conf (multisite) set_fact: - has_ssl_replication_port_in_conf_multisite: "{{ 'conf' in splunk and 'server' in splunk.conf and 'content' in splunk.conf.server and (splunk.conf.server.content.keys() | select('match', '^replication_port-ssl://') | list | length > 0) }}" + has_ssl_replication_port_in_conf_multisite: >- + {{ (multisite_server_conf_content | default({})).keys() + | select('match', '^replication_port-ssl://') | list | length > 0 }} # Determine if SSL replication should be used (either via flag or detected in config) - name: Determine SSL replication mode (multisite) @@ -53,11 +72,9 @@ vars: conf_file: "server.conf" conf_directory: "{{ splunk.home }}/etc/system/local" - conf_stanzas: "{{ splunk.conf.server.content }}" + conf_stanzas: "{{ multisite_server_conf_content }}" when: - has_ssl_replication_port_in_conf_multisite | default(false) | bool - - "'conf' in splunk" - - "'server' in splunk.conf" - - "'content' in splunk.conf.server" + - multisite_server_conf_content | default({}) | length > 0 #INFRA-38882: Update \ No newline at end of file diff --git a/roles/splunk_noah/tasks/pre_auth.yml b/roles/splunk_noah/tasks/pre_auth.yml index b3207bc5..b91c54b8 100644 --- a/roles/splunk_noah/tasks/pre_auth.yml +++ b/roles/splunk_noah/tasks/pre_auth.yml @@ -3,21 +3,28 @@ # for that process, but write every required field so its config parser is safe. - include_tasks: validate.yml +- name: Extract the noahService stanza from splunk.conf (dict format) + set_fact: + noah_service_stanza: >- + {{ splunk.conf.server.content.noahService }} + when: splunk.conf is mapping + +- name: Extract the noahService stanza from splunk.conf (list format) + set_fact: + noah_service_stanza: >- + {{ (splunk.conf | selectattr('key', 'equalto', 'server') | first | default({})).get('value', {}).get('content', {}).get('noahService', {}) }} + when: splunk.conf is not mapping + - name: Validate the operator-provided Noah service configuration assert: that: - - >- - (((splunk.conf | default({})).get('server', {})).get('content', {})).get('noahService') - is mapping - - >- - (((splunk.conf | default({})).get('server', {})).get('content', {})).get('noahService', {}).get('uri', '') - | length > 0 - - >- - (((splunk.conf | default({})).get('server', {})).get('content', {})).get('noahService', {}).get('tenant', '') - | length > 0 + - noah_service_stanza is mapping + - noah_service_stanza | length > 0 + - noah_service_stanza.get('uri', '') | length > 0 + - noah_service_stanza.get('tenant', '') | length > 0 fail_msg: >- - SPLUNK_NOAH_ENABLED=true requires splunk.conf.server.content.noahService - in the docker-splunk defaults. + SPLUNK_NOAH_ENABLED=true requires splunk.conf to contain a server entry + with content.noahService including at least 'uri' and 'tenant'. - name: Write Noah service configuration for temporary authentication startup ini_file: @@ -27,7 +34,7 @@ value: "{{ 'true' if item.key == 'disabled' else item.value }}" owner: "{{ splunk.user }}" group: "{{ splunk.group }}" - with_dict: "{{ splunk.conf.server.content.noahService }}" + with_dict: "{{ noah_service_stanza }}" no_log: "{{ hide_password }}" - name: Keep Noah disabled during temporary authentication startup diff --git a/tests/small/test_noah_role_configuration.py b/tests/small/test_noah_role_configuration.py index 9a186f5c..27c06c34 100644 --- a/tests/small/test_noah_role_configuration.py +++ b/tests/small/test_noah_role_configuration.py @@ -114,7 +114,10 @@ def test_pre_auth_keeps_noah_disabled_and_writes_a_safe_heartbeat(): assert "'true' if item.key == 'disabled'" in text assert "Keep Noah disabled during temporary authentication startup" in text assert "splunk_noah_client_profile.heartbeat_period" in text - assert "splunk.conf.server.content.noahService" in text + assert "noah_service_stanza" in text + assert "splunk.conf is mapping" in text + assert "splunk.conf is not mapping" in text + assert "selectattr('key', 'equalto', 'server')" in text def test_each_supported_role_has_only_its_intended_noah_behavior():