From a03de5d753f6ec23e494ee98b2e6d4aacc1f322d Mon Sep 17 00:00:00 2001
From: Markus Hilger <markus.hilger@megware.com>
Date: Sat, 11 Jul 2026 06:04:59 +0200
Subject: [PATCH 1/9] Add remote log forwarding for deployed nodes

A new `logging.servers` attribute, when set, makes OS deployment configure the node to forward its logs to the given servers. It is surfaced as loggingservers/loggingmethod in confluent.deploycfg and applied by the new common `setuplogging` script, which is invoked in the profiles of all OSes shipping rsyslog by default and does nothing when the attribute is unset or the selected forwarder is absent.

The forwarding method is selected by `logging.method`: `rsyslog` (the default) writes an rsyslog drop-in forwarding syslog to the given servers over UDP port 514; `journal-remote` configures systemd-journal-upload to upload the journal to systemd-journal-remote on port 19532 (a single destination only - the first server is used). A `logging.tls` attribute is defined to encrypt the forwarding with TLS authenticated via the confluent certificate authority (not implemented yet)

Example receiver configurations for the confluent server side (rsyslog per-node logs plus logrotate, journal-remote drop-in) are shipped under `/opt/confluent/share/examples/logging`.
---
 .../common/profile/scripts/setuplogging       | 80 +++++++++++++++++++
 .../debian/profiles/default/scripts/post.sh   |  1 +
 .../profiles/default/scripts/onboot.sh        |  1 +
 .../el7/profiles/default/scripts/post.sh      |  1 +
 .../profiles/default/scripts/onboot.sh        |  1 +
 .../el8/profiles/default/scripts/post.sh      |  1 +
 .../profiles/default/scripts/onboot.sh        |  1 +
 .../profiles/default/scripts/onboot.sh        |  1 +
 .../suse15/profiles/hpc/scripts/post.sh       |  1 +
 .../suse15/profiles/server/scripts/post.sh    |  1 +
 .../suse16/profiles/server/scripts/post.sh    |  1 +
 .../profiles/default/scripts/post.sh          |  1 +
 .../profiles/default/scripts/onboot.sh        |  1 +
 .../profiles/default/scripts/post.sh          |  1 +
 .../profiles/default/scripts/post.sh          |  1 +
 confluent_server/MANIFEST.in                  |  2 +
 .../confluent/config/attributes.py            | 26 ++++++
 confluent_server/confluent/selfservice.py     | 11 ++-
 .../confluent-journal-remote.conf             | 22 +++++
 .../logging/rsyslog/confluent-nodes.conf      | 27 +++++++
 .../logging/rsyslog/confluent-nodes.logrotate | 21 +++++
 confluent_server/setup.py.tmpl                |  3 +
 22 files changed, 205 insertions(+), 1 deletion(-)
 create mode 100644 confluent_osdeploy/common/profile/scripts/setuplogging
 create mode 100644 confluent_server/logging/journal-remote/confluent-journal-remote.conf
 create mode 100644 confluent_server/logging/rsyslog/confluent-nodes.conf
 create mode 100644 confluent_server/logging/rsyslog/confluent-nodes.logrotate

diff --git a/confluent_osdeploy/common/profile/scripts/setuplogging b/confluent_osdeploy/common/profile/scripts/setuplogging
new file mode 100644
index 000000000..1d3e8d9af
--- /dev/null
+++ b/confluent_osdeploy/common/profile/scripts/setuplogging
@@ -0,0 +1,80 @@
+#!/bin/sh
+# Configure the deployed node to forward logs to the servers from the
+# logging.servers attribute (surfaced as loggingservers in
+# confluent.deploycfg), using the method from the logging.method attribute
+# (surfaced as loggingmethod): rsyslog (the default) or journal-remote.
+# Does nothing when logging.servers is unset.
+
+if ! grep '^loggingservers:' /etc/confluent/confluent.deploycfg > /dev/null 2>&1; then
+    exit 0
+fi
+loggingservers=$(sed -n '/^loggingservers:/,/^[^-]/p' /etc/confluent/confluent.deploycfg|sed 1d|sed '$d' | sed -e 's/^- //')
+loggingmethod=$(sed -n 's/^loggingmethod: //p' /etc/confluent/confluent.deploycfg)
+if [ -z "$loggingmethod" ]; then
+    loggingmethod=rsyslog
+fi
+if [ "$loggingmethod" = journal-remote ]; then
+    # drop forwarding state left by a previous rsyslog-method configuration
+    if [ -f /etc/rsyslog.d/60-confluent.conf ]; then
+        rm -f /etc/rsyslog.d/60-confluent.conf
+        if [ -d /run/systemd/system ]; then
+            systemctl try-restart rsyslog.service 2>/dev/null
+        fi
+    fi
+    uploadbin=""
+    for candidate in /usr/lib/systemd/systemd-journal-upload /lib/systemd/systemd-journal-upload; do
+        if [ -x $candidate ]; then
+            uploadbin=$candidate
+            break
+        fi
+    done
+    if [ -z "$uploadbin" ]; then
+        echo "systemd-journal-upload does not appear to be installed, skipping log forwarding setup"
+        exit 0
+    fi
+    loggingsrv=$(echo "$loggingservers" | sed -n 1p)
+    if [ "$(echo "$loggingservers" | wc -l)" -gt 1 ]; then
+        echo "journal-remote supports only a single destination, forwarding to $loggingsrv and ignoring the other logging.servers entries"
+    fi
+    case $loggingsrv in
+        *:*) loggingsrv="[$loggingsrv]" ;;  # bracket IPv6 literals for the URL
+    esac
+    mkdir -p /etc/systemd/journal-upload.conf.d
+    journalconf=/etc/systemd/journal-upload.conf.d/confluent.conf
+    echo '# Log forwarding configured by confluent during deployment' > $journalconf
+    echo '[Upload]' >> $journalconf
+    echo 'URL=http://'$loggingsrv':19532' >> $journalconf
+    if [ -x /usr/sbin/restorecon ]; then
+        /usr/sbin/restorecon $journalconf 2>/dev/null
+    fi
+    systemctl enable systemd-journal-upload.service 2>/dev/null
+    if [ -d /run/systemd/system ]; then
+        systemctl restart systemd-journal-upload.service 2>/dev/null
+    fi
+else
+    # drop forwarding state left by a previous journal-remote-method
+    # configuration
+    if [ -f /etc/systemd/journal-upload.conf.d/confluent.conf ]; then
+        rm -f /etc/systemd/journal-upload.conf.d/confluent.conf
+        systemctl disable systemd-journal-upload.service 2>/dev/null
+        if [ -d /run/systemd/system ]; then
+            systemctl stop systemd-journal-upload.service 2>/dev/null
+        fi
+    fi
+    if [ ! -d /etc/rsyslog.d ]; then
+        echo "rsyslog does not appear to be installed, skipping log forwarding setup"
+        exit 0
+    fi
+    syslogconf=/etc/rsyslog.d/60-confluent.conf
+    echo '# Syslog forwarding configured by confluent during deployment' > $syslogconf
+    for loggingsrv in $loggingservers; do
+        echo '*.* action(type="omfwd" target="'$loggingsrv'" port="514" protocol="udp")' >> $syslogconf
+    done
+    if [ -x /usr/sbin/restorecon ]; then
+        /usr/sbin/restorecon $syslogconf 2>/dev/null
+    fi
+    if [ -d /run/systemd/system ]; then
+        systemctl try-restart rsyslog.service 2>/dev/null
+    fi
+fi
+exit 0
diff --git a/confluent_osdeploy/debian/profiles/default/scripts/post.sh b/confluent_osdeploy/debian/profiles/default/scripts/post.sh
index f30d980f9..130d13c10 100755
--- a/confluent_osdeploy/debian/profiles/default/scripts/post.sh
+++ b/confluent_osdeploy/debian/profiles/default/scripts/post.sh
@@ -59,6 +59,7 @@ if [ -e /sys/firmware/efi ]; then
         efibootmgr -D
     fi
 fi
+run_remote setuplogging
 run_remote_python syncfileclient
 run_remote_parts post.d
 run_remote_config post
diff --git a/confluent_osdeploy/el7-diskless/profiles/default/scripts/onboot.sh b/confluent_osdeploy/el7-diskless/profiles/default/scripts/onboot.sh
index 818fb3d07..862e163c7 100644
--- a/confluent_osdeploy/el7-diskless/profiles/default/scripts/onboot.sh
+++ b/confluent_osdeploy/el7-diskless/profiles/default/scripts/onboot.sh
@@ -39,6 +39,7 @@ chmod 600 /var/log/confluent/confluent-onboot.log
 tail -f /var/log/confluent/confluent-onboot.log > /dev/console &
 logshowpid=$!
 
+run_remote setuplogging
 run_remote_python syncfileclient
 run_remote_python confignet
 
diff --git a/confluent_osdeploy/el7/profiles/default/scripts/post.sh b/confluent_osdeploy/el7/profiles/default/scripts/post.sh
index 362a79454..9dc7d2584 100644
--- a/confluent_osdeploy/el7/profiles/default/scripts/post.sh
+++ b/confluent_osdeploy/el7/profiles/default/scripts/post.sh
@@ -34,6 +34,7 @@ run_remote_python add_local_repositories
 # run_remote_python will use the appropriate python interpreter path to run the specified script
 # A post.custom is provided to more conveniently hold customizations, see the post.custom file.
 
+run_remote setuplogging
 # This will induce server side processing of the syncfile contents if
 # present
 run_remote_python syncfileclient
diff --git a/confluent_osdeploy/el8-diskless/profiles/default/scripts/onboot.sh b/confluent_osdeploy/el8-diskless/profiles/default/scripts/onboot.sh
index d2d50a7d3..2d915ef43 100644
--- a/confluent_osdeploy/el8-diskless/profiles/default/scripts/onboot.sh
+++ b/confluent_osdeploy/el8-diskless/profiles/default/scripts/onboot.sh
@@ -51,6 +51,7 @@ logshowpid=$!
 rpm --import /etc/pki/rpm-gpg/*
 
 run_remote_python add_local_repositories
+run_remote setuplogging
 run_remote_python syncfileclient
 run_remote_python confignet -c $confluent_mgr
 
diff --git a/confluent_osdeploy/el8/profiles/default/scripts/post.sh b/confluent_osdeploy/el8/profiles/default/scripts/post.sh
index db1c511d8..666494225 100644
--- a/confluent_osdeploy/el8/profiles/default/scripts/post.sh
+++ b/confluent_osdeploy/el8/profiles/default/scripts/post.sh
@@ -39,6 +39,7 @@ run_remote_python add_local_repositories
 run_remote_python autoconsole
 
 
+run_remote setuplogging
 # This will induce server side processing of the syncfile contents if
 # present
 run_remote_python syncfileclient
diff --git a/confluent_osdeploy/el9-diskless/profiles/default/scripts/onboot.sh b/confluent_osdeploy/el9-diskless/profiles/default/scripts/onboot.sh
index 44e972356..3ebfc767a 100644
--- a/confluent_osdeploy/el9-diskless/profiles/default/scripts/onboot.sh
+++ b/confluent_osdeploy/el9-diskless/profiles/default/scripts/onboot.sh
@@ -45,6 +45,7 @@ logshowpid=$!
 rpm --import /etc/pki/rpm-gpg/*
 
 run_remote_python add_local_repositories
+run_remote setuplogging
 run_remote_python syncfileclient
 run_remote_python confignet -c $confluent_mgr
 
diff --git a/confluent_osdeploy/suse15-diskless/profiles/default/scripts/onboot.sh b/confluent_osdeploy/suse15-diskless/profiles/default/scripts/onboot.sh
index 985d1c026..29befc3b9 100644
--- a/confluent_osdeploy/suse15-diskless/profiles/default/scripts/onboot.sh
+++ b/confluent_osdeploy/suse15-diskless/profiles/default/scripts/onboot.sh
@@ -20,6 +20,7 @@ chmod 600 /var/log/confluent/confluent-onboot.log
 tail -f /var/log/confluent/confluent-onboot.log > /dev/console &
 logshowpid=$!
 
+run_remote setuplogging
 run_remote_python syncfileclient
 run_remote_python confignet
 run_remote onboot.custom
diff --git a/confluent_osdeploy/suse15/profiles/hpc/scripts/post.sh b/confluent_osdeploy/suse15/profiles/hpc/scripts/post.sh
index be4e2d80d..8f5917742 100644
--- a/confluent_osdeploy/suse15/profiles/hpc/scripts/post.sh
+++ b/confluent_osdeploy/suse15/profiles/hpc/scripts/post.sh
@@ -25,6 +25,7 @@ chmod og-rwx /etc/confluent/*
 export confluent_mgr confluent_profile nodename
 . /etc/confluent/functions
 
+run_remote setuplogging
 # This will induce server side processing of the syncfile contents if
 # present
 run_remote_python syncfileclient
diff --git a/confluent_osdeploy/suse15/profiles/server/scripts/post.sh b/confluent_osdeploy/suse15/profiles/server/scripts/post.sh
index be4e2d80d..8f5917742 100644
--- a/confluent_osdeploy/suse15/profiles/server/scripts/post.sh
+++ b/confluent_osdeploy/suse15/profiles/server/scripts/post.sh
@@ -25,6 +25,7 @@ chmod og-rwx /etc/confluent/*
 export confluent_mgr confluent_profile nodename
 . /etc/confluent/functions
 
+run_remote setuplogging
 # This will induce server side processing of the syncfile contents if
 # present
 run_remote_python syncfileclient
diff --git a/confluent_osdeploy/suse16/profiles/server/scripts/post.sh b/confluent_osdeploy/suse16/profiles/server/scripts/post.sh
index fd54308da..76c8134d1 100644
--- a/confluent_osdeploy/suse16/profiles/server/scripts/post.sh
+++ b/confluent_osdeploy/suse16/profiles/server/scripts/post.sh
@@ -29,6 +29,7 @@ chmod og-rwx /etc/confluent/*
 export confluent_mgr confluent_profile nodename
 . /etc/confluent/functions
 
+run_remote setuplogging
 # This will induce server side processing of the syncfile contents if
 # present
 run_remote_python syncfileclient
diff --git a/confluent_osdeploy/ubuntu18.04/profiles/default/scripts/post.sh b/confluent_osdeploy/ubuntu18.04/profiles/default/scripts/post.sh
index f30d980f9..130d13c10 100755
--- a/confluent_osdeploy/ubuntu18.04/profiles/default/scripts/post.sh
+++ b/confluent_osdeploy/ubuntu18.04/profiles/default/scripts/post.sh
@@ -59,6 +59,7 @@ if [ -e /sys/firmware/efi ]; then
         efibootmgr -D
     fi
 fi
+run_remote setuplogging
 run_remote_python syncfileclient
 run_remote_parts post.d
 run_remote_config post
diff --git a/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/onboot.sh b/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/onboot.sh
index cc470d6fa..f887c4822 100644
--- a/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/onboot.sh
+++ b/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/onboot.sh
@@ -27,6 +27,7 @@ chmod 600 /var/log/confluent/confluent-onboot.log
 tail -f /var/log/confluent/confluent-onboot.log > /dev/console &
 logshowpid=$!
 
+run_remote setuplogging
 run_remote_python syncfileclient
 run_remote_python confignet
 
diff --git a/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/post.sh b/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/post.sh
index d97308893..071e58eef 100755
--- a/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/post.sh
+++ b/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/post.sh
@@ -82,6 +82,7 @@ cat /target/etc/confluent/tls/*.pem > /target/etc/confluent/ca.pem
 cat /target/etc/confluent/tls/*.pem > /target/usr/local/share/ca-certificates/confluent.crt
 cat /target/etc/confluent/tls/*.pem > /etc/confluent/ca.pem
 chroot /target update-ca-certificates
+chroot /target bash -c "source /etc/confluent/functions; run_remote setuplogging"
 chroot /target bash -c "source /etc/confluent/functions; run_remote_python syncfileclient"
 chroot /target bash -c "source /etc/confluent/functions; run_remote_python confignet"
 chroot /target bash -c "source /etc/confluent/functions; run_remote_parts post.d"
diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh
index 78e6b411b..47e243ada 100755
--- a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh
+++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh
@@ -88,6 +88,7 @@ chroot /target update-ca-certificates
 # Ubuntu mangles grub function for serial users, undo that mangling
 chroot /target bash -c "source /etc/confluent/functions; run_remote_python autoconsole"
 
+chroot /target bash -c "source /etc/confluent/functions; run_remote setuplogging"
 chroot /target bash -c "source /etc/confluent/functions; run_remote_python syncfileclient"
 chroot /target bash -c "source /etc/confluent/functions; run_remote_parts post.d"
 source /target/etc/confluent/functions
diff --git a/confluent_server/MANIFEST.in b/confluent_server/MANIFEST.in
index b71e09982..dbef8c178 100644
--- a/confluent_server/MANIFEST.in
+++ b/confluent_server/MANIFEST.in
@@ -2,3 +2,5 @@ include pam/*
 include sysvinit/*
 include systemd/*
 include sysctl/*
+include logging/rsyslog/*
+include logging/journal-remote/*
diff --git a/confluent_server/confluent/config/attributes.py b/confluent_server/confluent/config/attributes.py
index 06f430ed7..a8d48fcb9 100644
--- a/confluent_server/confluent/config/attributes.py
+++ b/confluent_server/confluent/config/attributes.py
@@ -705,4 +705,30 @@
     'dns.servers': {
         'description': 'DNS Server or servers to provide to node',
     },
+    'logging.servers': {
+        'description': 'Remote logging server or servers (comma separated) '
+                       'for the OS deployment to configure the node to '
+                       'forward logs to, using the method indicated by '
+                       'logging.method. May be set to the address of the '
+                       'deployment server or any other logging server. See '
+                       '/opt/confluent/share/examples/logging for cfg '
+                       'examples. If unset, no forwarding is configured.',
+    },
+    'logging.method': {
+        'description': 'Method used to forward logs to logging.servers. '
+                       '"rsyslog" (the default if unset) forwards syslog '
+                       'using rsyslog over UDP port 514. "journal-remote" '
+                       'uploads the systemd journal using '
+                       'systemd-journal-upload to systemd-journal-remote on '
+                       'port 19532; note that systemd-journal-upload supports '
+                       'only a single destination, so only the first entry of '
+                       'logging.servers is used.',
+        'validvalues': ('rsyslog', 'journal-remote', ''),
+    },
+    'logging.tls': {
+        'description': 'Whether to encrypt the forwarded logs with TLS, '
+                       'authenticated via the confluent TLS certificate '
+                       'authority.',
+        'type': bool,
+    },
 }
diff --git a/confluent_server/confluent/selfservice.py b/confluent_server/confluent/selfservice.py
index c9499a7f1..efc8db720 100644
--- a/confluent_server/confluent/selfservice.py
+++ b/confluent_server/confluent/selfservice.py
@@ -295,7 +295,7 @@ def verify_cert(certificate):
             ncfg['ipv4_method'] = 'static'
         deployinfo = cfg.get_node_attributes(
             nodename, ('deployment.*', 'console.method', 'crypted.*',
-                       'dns.*', 'ntp.*'))
+                       'dns.*', 'ntp.*', 'logging.*'))
         deployinfo = deployinfo.get(nodename, {})
         profile = deployinfo.get(
             'deployment.pendingprofile', {}).get('value', '')
@@ -405,6 +405,15 @@ def verify_cert(certificate):
             ncfg['ntpservers'] = []
             for ntpsrv in ntpsrvs:
                 ncfg['ntpservers'].append(ntpsrv)
+        logsrvs = deployinfo.get('logging.servers', {}).get('value', '')
+        if logsrvs:
+            logsrvs = logsrvs.split(',')
+        if logsrvs:
+            ncfg['loggingservers'] = []
+            for logsrv in logsrvs:
+                ncfg['loggingservers'].append(logsrv)
+            ncfg['loggingmethod'] = deployinfo.get(
+                'logging.method', {}).get('value', '') or 'rsyslog'
         dnsdomain = deployinfo.get('dns.domain', {}).get('value', None)
         ncfg['dnsdomain'] = dnsdomain
         return await make_response(mimetype, 200, 'OK', body=dumper(ncfg))
diff --git a/confluent_server/logging/journal-remote/confluent-journal-remote.conf b/confluent_server/logging/journal-remote/confluent-journal-remote.conf
new file mode 100644
index 000000000..69cdef76b
--- /dev/null
+++ b/confluent_server/logging/journal-remote/confluent-journal-remote.conf
@@ -0,0 +1,22 @@
+# Example systemd-journal-remote configuration for a confluent server to
+# receive journal entries uploaded by deployed nodes (see the logging.servers
+# and logging.method node attributes). The stock systemd-journal-remote unit
+# only listens for HTTPS; this drop-in switches it to plain HTTP on port
+# 19532, matching what systemd-journal-upload is configured for on the nodes.
+#
+# To activate:
+#   dnf/zypper/apt install systemd-journal-remote
+#   mkdir -p /etc/systemd/system/systemd-journal-remote.service.d
+#   cp confluent-journal-remote.conf /etc/systemd/system/systemd-journal-remote.service.d/confluent.conf
+#   systemctl daemon-reload
+#   systemctl enable --now systemd-journal-remote.socket
+# and permit the port through the firewall, e.g. on a firewalld based system:
+#   firewall-cmd --permanent --add-port=19532/tcp && firewall-cmd --reload
+#
+# Received entries are written to
+# /var/log/journal/remote/remote-<node>.journal and can be read with:
+#   journalctl -D /var/log/journal/remote
+
+[Service]
+ExecStart=
+ExecStart=/usr/lib/systemd/systemd-journal-remote --listen-http=-3 --output=/var/log/journal/remote/
diff --git a/confluent_server/logging/rsyslog/confluent-nodes.conf b/confluent_server/logging/rsyslog/confluent-nodes.conf
new file mode 100644
index 000000000..5aa8928d9
--- /dev/null
+++ b/confluent_server/logging/rsyslog/confluent-nodes.conf
@@ -0,0 +1,27 @@
+# Example rsyslog configuration for a confluent server to receive syslog
+# forwarded by deployed nodes (see the logging.servers node attribute) and
+# write per-node logs under /var/log/confluent/nodes/
+#
+# To activate:
+#   cp confluent-nodes.conf /etc/rsyslog.d/confluent-nodes.conf
+#   systemctl restart rsyslog
+# and permit syslog through the firewall, e.g. on a firewalld based system:
+#   firewall-cmd --permanent --add-service=syslog && firewall-cmd --reload
+#
+# Note: on systems where rsyslogd does not run as root (e.g. Ubuntu runs it
+# as the "syslog" user), ensure that user may create and write
+# /var/log/confluent/nodes
+#
+# The per-node logs can grow large; see confluent-nodes.logrotate for an
+# example logrotate configuration that rotates and compresses them.
+
+module(load="imudp")
+input(type="imudp" port="514" ruleset="confluentnodes")
+
+template(name="confluentnodelog" type="string"
+         string="/var/log/confluent/nodes/%HOSTNAME%.log")
+
+ruleset(name="confluentnodes") {
+    action(type="omfile" dynaFile="confluentnodelog"
+           dirCreateMode="0700" fileCreateMode="0600")
+}
diff --git a/confluent_server/logging/rsyslog/confluent-nodes.logrotate b/confluent_server/logging/rsyslog/confluent-nodes.logrotate
new file mode 100644
index 000000000..8b4937724
--- /dev/null
+++ b/confluent_server/logging/rsyslog/confluent-nodes.logrotate
@@ -0,0 +1,21 @@
+# Example logrotate configuration for the per-node logs written by the
+# confluent-nodes.conf rsyslog example under /var/log/confluent/nodes/
+#
+# To activate:
+#   cp confluent-nodes.logrotate /etc/logrotate.d/confluent-nodes
+#
+# The following settings rotate logs once a week for up to 52 weeks (1 year)
+# and compress all logs older than 2 weeks
+
+/var/log/confluent/nodes/*.log {
+    weekly
+    rotate 52
+    compress
+    delaycompress
+    missingok
+    notifempty
+    sharedscripts
+    postrotate
+        /usr/bin/systemctl kill -s HUP rsyslog.service >/dev/null 2>&1 || true
+    endscript
+}
diff --git a/confluent_server/setup.py.tmpl b/confluent_server/setup.py.tmpl
index fc481798d..d79c844a1 100644
--- a/confluent_server/setup.py.tmpl
+++ b/confluent_server/setup.py.tmpl
@@ -40,6 +40,9 @@ setup(
     data_files=[('/etc/init.d', ['sysvinit/confluent']),
                 ('/usr/lib/sysctl.d', ['sysctl/confluent.conf']),
                 ('/opt/confluent/share/licenses/confluent_server', ['LICENSE', 'COPYRIGHT']),
+                ('/opt/confluent/share/examples/logging/rsyslog', ['logging/rsyslog/confluent-nodes.conf',
+                                                                   'logging/rsyslog/confluent-nodes.logrotate']),
+                ('/opt/confluent/share/examples/logging/journal-remote', ['logging/journal-remote/confluent-journal-remote.conf']),
                 ('/usr/lib/systemd/system', ['systemd/confluent.service']),
 		('/opt/confluent/lib/python/confluent/plugins/console/', [])],
 

From 0d2724471c78c868888f5b92cf5caccc5ee61fcd Mon Sep 17 00:00:00 2001
From: Markus Hilger <markus.hilger@megware.com>
Date: Tue, 14 Jul 2026 18:54:24 +0200
Subject: [PATCH 2/9] Implement TLS for remote log forwarding

When the logging.tls attribute is set, the log forwarding configured from logging.servers is now encrypted and mutually authenticated via the confluent TLS certificate authority. During deployment, setuplogging requests a node certificate from the deployment API (/confluent-api/self/tlscert, validity governed by pubkeys.tls_lifetime, 47 days unless raised) and trusts /etc/confluent/ca.pem. With the rsyslog method, logs are forwarded over TCP port 6514 using the rsyslog-openssl or rsyslog-gnutls stream driver with x509/certvalid authentication; with journal-remote, systemd-journal-upload uploads over HTTPS port 19532 with explicit key, certificate and trust anchor settings. If TLS is requested but prerequisites are missing, the node is left unconfigured.

The certificate material is stored in dedicated directories readable by the consuming service (/etc/rsyslog.d/confluent-tls for rsyslog, /etc/ssl/confluent-logging for the journal tools).

The receiving side is handled by a new confluent-logging-receiver-setup helper shipped with the examples under /opt/confluent/share/examples/logging. It issues a server certificate signed offline by the local confluent CA (SANs covering the server names and addresses), installs the mutual TLS reception configuration for either method, setting TrustedCertificateFile explicitly for systemd-journal-remote, and can alternatively install the plaintext example configurations.

Note: Some distros (e.g., EL10, Ubuntu 24.04) build systemd-journal-remote without GnuTLS. This silently disables client certificate verification (accepting any client), though encryption remains active.
The helper warns of this issue. If client verification is required, use rsyslog (as noted in logging.tls).
---
 .../common/profile/scripts/setuplogging       | 117 ++++++++++++-
 confluent_server/MANIFEST.in                  |   1 +
 .../confluent/config/attributes.py            |  23 ++-
 confluent_server/confluent/selfservice.py     |   2 +
 .../logging/confluent-logging-receiver-setup  | 163 ++++++++++++++++++
 .../confluent-journal-remote.conf             |   3 +-
 .../logging/rsyslog/confluent-nodes-tls.conf  |  42 +++++
 confluent_server/setup.py.tmpl                |   2 +
 8 files changed, 343 insertions(+), 10 deletions(-)
 create mode 100755 confluent_server/logging/confluent-logging-receiver-setup
 create mode 100644 confluent_server/logging/rsyslog/confluent-nodes-tls.conf

diff --git a/confluent_osdeploy/common/profile/scripts/setuplogging b/confluent_osdeploy/common/profile/scripts/setuplogging
index 1d3e8d9af..4ec95a0c5 100644
--- a/confluent_osdeploy/common/profile/scripts/setuplogging
+++ b/confluent_osdeploy/common/profile/scripts/setuplogging
@@ -1,18 +1,78 @@
-#!/bin/sh
+#!/bin/bash
 # Configure the deployed node to forward logs to the servers from the
 # logging.servers attribute (surfaced as loggingservers in
 # confluent.deploycfg), using the method from the logging.method attribute
 # (surfaced as loggingmethod): rsyslog (the default) or journal-remote.
+# When logging.tls (surfaced as loggingtls) is true, the forwarding is
+# encrypted and mutually authenticated using certificates from the confluent
+# certificate authority (see the confluent-logging-receiver-setup helper
+# under /opt/confluent/share/examples/logging for the receiving side).
 # Does nothing when logging.servers is unset.
 
+[ -f /lib/confluent/functions ] && . /lib/confluent/functions
+[ -f /etc/confluent/functions ] && . /etc/confluent/functions
+
 if ! grep '^loggingservers:' /etc/confluent/confluent.deploycfg > /dev/null 2>&1; then
     exit 0
 fi
 loggingservers=$(sed -n '/^loggingservers:/,/^[^-]/p' /etc/confluent/confluent.deploycfg|sed 1d|sed '$d' | sed -e 's/^- //')
 loggingmethod=$(sed -n 's/^loggingmethod: //p' /etc/confluent/confluent.deploycfg)
+loggingtls=$(sed -n 's/^loggingtls: //p' /etc/confluent/confluent.deploycfg)
 if [ -z "$loggingmethod" ]; then
     loggingmethod=rsyslog
 fi
+
+# request_tls_cert <keyout> <certout>: generate a key and have the confluent
+# CA sign a certificate for this node via the deployment API
+request_tls_cert() {
+    if ! type confluentpython > /dev/null 2>&1; then
+        confluentpython() { python3 "$@"; }
+    fi
+    confapiclient=""
+    [ -f /opt/confluent/bin/apiclient ] && confapiclient=/opt/confluent/bin/apiclient
+    [ -f /etc/confluent/apiclient ] && confapiclient=/etc/confluent/apiclient
+    if [ -z "$confapiclient" ]; then
+        echo "logging.tls: unable to locate the confluent apiclient, skipping log forwarding setup"
+        return 1
+    fi
+    if ! command -v openssl > /dev/null 2>&1; then
+        echo "logging.tls: openssl is required but not available, skipping log forwarding setup"
+        return 1
+    fi
+    if [ ! -s /etc/confluent/ca.pem ]; then
+        echo "logging.tls: /etc/confluent/ca.pem is missing, skipping log forwarding setup"
+        return 1
+    fi
+    # $3 is the destination for the CA trust anchor. A dedicated directory is
+    # used per consumer: /etc/ssl/private is not traversable by service users
+    # on Debian-style systems, and Ubuntu's rsyslogd AppArmor profile only
+    # permits reads under /etc/rsyslog.d
+    # stage beside the live files (same filesystem), then replace atomically;
+    # a failure never disturbs material a previous run may have left in use
+    mkdir -p $(dirname $1)
+    cat /etc/confluent/ca.pem > $3.new
+    CRTDIR=$(mktemp -d)
+    openssl req -new -newkey ec -pkeyopt ec_paramgen_curve:secp384r1 -nodes \
+        -keyout $1.new -out $CRTDIR/csr.pem -subj /CN=$(hostname) > /dev/null 2>&1
+    echo -n > $2.new
+    confluentpython $confapiclient /confluent-api/self/tlscert $CRTDIR/csr.pem -o $2.new
+    rm -rf $CRTDIR
+    if [ ! -s "$2.new" ]; then
+        echo "logging.tls: failed to obtain a certificate from the confluent CA, skipping log forwarding setup"
+        rm -f $1.new $2.new $3.new
+        return 1
+    fi
+    chmod 600 $1.new
+    chmod 644 $2.new $3.new
+    if [ -x /usr/sbin/restorecon ]; then
+        /usr/sbin/restorecon $1.new $2.new $3.new 2>/dev/null
+    fi
+    mv $1.new $1
+    mv $2.new $2
+    mv $3.new $3
+    return 0
+}
+
 if [ "$loggingmethod" = journal-remote ]; then
     # drop forwarding state left by a previous rsyslog-method configuration
     if [ -f /etc/rsyslog.d/60-confluent.conf ]; then
@@ -43,7 +103,21 @@ if [ "$loggingmethod" = journal-remote ]; then
     journalconf=/etc/systemd/journal-upload.conf.d/confluent.conf
     echo '# Log forwarding configured by confluent during deployment' > $journalconf
     echo '[Upload]' >> $journalconf
-    echo 'URL=http://'$loggingsrv':19532' >> $journalconf
+    if [ "$loggingtls" = true ]; then
+        tlsdir=/etc/ssl/confluent-logging
+        if ! request_tls_cert $tlsdir/journal-upload.key $tlsdir/journal-upload.crt $tlsdir/ca.pem; then
+            rm -f $journalconf
+            exit 0
+        fi
+        chgrp systemd-journal $tlsdir/journal-upload.key 2>/dev/null
+        chmod g+r $tlsdir/journal-upload.key
+        echo 'URL=https://'$loggingsrv':19532' >> $journalconf
+        echo 'ServerKeyFile='$tlsdir'/journal-upload.key' >> $journalconf
+        echo 'ServerCertificateFile='$tlsdir'/journal-upload.crt' >> $journalconf
+        echo 'TrustedCertificateFile='$tlsdir'/ca.pem' >> $journalconf
+    else
+        echo 'URL=http://'$loggingsrv':19532' >> $journalconf
+    fi
     if [ -x /usr/sbin/restorecon ]; then
         /usr/sbin/restorecon $journalconf 2>/dev/null
     fi
@@ -66,10 +140,41 @@ else
         exit 0
     fi
     syslogconf=/etc/rsyslog.d/60-confluent.conf
-    echo '# Syslog forwarding configured by confluent during deployment' > $syslogconf
-    for loggingsrv in $loggingservers; do
-        echo '*.* action(type="omfwd" target="'$loggingsrv'" port="514" protocol="udp")' >> $syslogconf
-    done
+    if [ "$loggingtls" = true ]; then
+        # forwarding with TLS needs a network stream driver module
+        # (rsyslog-openssl or rsyslog-gnutls)
+        tlsdriver=""
+        for moddir in /usr/lib64/rsyslog /usr/lib/rsyslog /usr/lib/*-linux-gnu/rsyslog; do
+            [ -f $moddir/lmnsd_ossl.so ] && tlsdriver=ossl && break
+            [ -f $moddir/lmnsd_gtls.so ] && tlsdriver=gtls && break
+        done
+        if [ -z "$tlsdriver" ]; then
+            echo "logging.tls requires the rsyslog-openssl or rsyslog-gnutls module on the node, skipping log forwarding setup"
+            exit 0
+        fi
+        tlsdir=/etc/rsyslog.d/confluent-tls
+        if ! request_tls_cert $tlsdir/rsyslog.key $tlsdir/rsyslog.crt $tlsdir/ca.pem; then
+            exit 0
+        fi
+        if getent passwd syslog > /dev/null 2>&1; then
+            # e.g. Ubuntu runs rsyslogd as the syslog user
+            chgrp syslog $tlsdir/rsyslog.key 2>/dev/null
+            chmod g+r $tlsdir/rsyslog.key
+        fi
+        echo '# Syslog forwarding configured by confluent during deployment' > $syslogconf
+        echo 'global(DefaultNetstreamDriver="'$tlsdriver'"' >> $syslogconf
+        echo '       DefaultNetstreamDriverCAFile="'$tlsdir'/ca.pem"' >> $syslogconf
+        echo '       DefaultNetstreamDriverCertFile="'$tlsdir'/rsyslog.crt"' >> $syslogconf
+        echo '       DefaultNetstreamDriverKeyFile="'$tlsdir'/rsyslog.key")' >> $syslogconf
+        for loggingsrv in $loggingservers; do
+            echo '*.* action(type="omfwd" target="'$loggingsrv'" port="6514" protocol="tcp" StreamDriverMode="1" StreamDriverAuthMode="x509/certvalid")' >> $syslogconf
+        done
+    else
+        echo '# Syslog forwarding configured by confluent during deployment' > $syslogconf
+        for loggingsrv in $loggingservers; do
+            echo '*.* action(type="omfwd" target="'$loggingsrv'" port="514" protocol="udp")' >> $syslogconf
+        done
+    fi
     if [ -x /usr/sbin/restorecon ]; then
         /usr/sbin/restorecon $syslogconf 2>/dev/null
     fi
diff --git a/confluent_server/MANIFEST.in b/confluent_server/MANIFEST.in
index dbef8c178..ac4d307db 100644
--- a/confluent_server/MANIFEST.in
+++ b/confluent_server/MANIFEST.in
@@ -2,5 +2,6 @@ include pam/*
 include sysvinit/*
 include systemd/*
 include sysctl/*
+include logging/*
 include logging/rsyslog/*
 include logging/journal-remote/*
diff --git a/confluent_server/confluent/config/attributes.py b/confluent_server/confluent/config/attributes.py
index a8d48fcb9..3e1aec9ff 100644
--- a/confluent_server/confluent/config/attributes.py
+++ b/confluent_server/confluent/config/attributes.py
@@ -726,9 +726,26 @@
         'validvalues': ('rsyslog', 'journal-remote', ''),
     },
     'logging.tls': {
-        'description': 'Whether to encrypt the forwarded logs with TLS, '
-                       'authenticated via the confluent TLS certificate '
-                       'authority.',
+        'description': 'Whether to encrypt the forwarded logs with TLS, with '
+                       'node and logging server mutually authenticated via '
+                       'the confluent TLS certificate authority. With '
+                       'rsyslog, logs are forwarded over TCP port 6514 and '
+                       'the node needs the rsyslog-openssl or rsyslog-gnutls '
+                       'module; with journal-remote, over HTTPS port 19532. '
+                       'Caution: some distributions build '
+                       'systemd-journal-remote without client certificate '
+                       'verification (e.g. EL10 and Ubuntu). '
+                       'The transport is still encrypted and '
+                       'the node verifies the server, but such receivers '
+                       'accept uploads from any client; use the rsyslog '
+                       'method when client verification is required. '
+                       'The node certificate is issued during deployment and '
+                       'its validity is governed by pubkeys.tls_lifetime '
+                       '(default 47 days) - consider raising it (e.g. 3650) '
+                       'so forwarding does not stop when the certificate '
+                       'expires. To set up the receiving side, see the '
+                       'confluent-logging-receiver-setup helper under '
+                       '/opt/confluent/share/examples/logging.',
         'type': bool,
     },
 }
diff --git a/confluent_server/confluent/selfservice.py b/confluent_server/confluent/selfservice.py
index efc8db720..1f7f8bf23 100644
--- a/confluent_server/confluent/selfservice.py
+++ b/confluent_server/confluent/selfservice.py
@@ -414,6 +414,8 @@ def verify_cert(certificate):
                 ncfg['loggingservers'].append(logsrv)
             ncfg['loggingmethod'] = deployinfo.get(
                 'logging.method', {}).get('value', '') or 'rsyslog'
+            ncfg['loggingtls'] = bool(deployinfo.get(
+                'logging.tls', {}).get('value', False))
         dnsdomain = deployinfo.get('dns.domain', {}).get('value', None)
         ncfg['dnsdomain'] = dnsdomain
         return await make_response(mimetype, 200, 'OK', body=dumper(ncfg))
diff --git a/confluent_server/logging/confluent-logging-receiver-setup b/confluent_server/logging/confluent-logging-receiver-setup
new file mode 100755
index 000000000..8bca3defa
--- /dev/null
+++ b/confluent_server/logging/confluent-logging-receiver-setup
@@ -0,0 +1,163 @@
+#!/bin/bash
+# Set up a confluent server to receive logs forwarded by deployed nodes
+# (see the logging.servers, logging.method and logging.tls node attributes).
+#
+# Usage: sh confluent-logging-receiver-setup <rsyslog|journal-remote> [--tls|--plain]
+#
+# With --tls (the default), the transport is encrypted and mutually
+# authenticated via the confluent certificate authority: a server certificate
+# is issued from the local confluent CA and only clients presenting a
+# certificate from that CA are accepted. With --plain, the plaintext receiver
+# examples are installed instead (rsyslog UDP port 514, journal-remote HTTP
+# port 19532).
+#
+# rsyslog reception writes per-node logs under /var/log/confluent/nodes/
+# (see confluent-nodes.logrotate for log rotation); journal-remote reception
+# writes journals under /var/log/journal/remote/, readable with
+#   journalctl -D /var/log/journal/remote
+#
+# Safe to re-run; regenerates the certificate and overwrites the
+# configuration it owns.
+
+set -e
+cd "$(dirname "$0")"
+
+method=""
+tls=1
+for arg in "$@"; do
+    case $arg in
+        rsyslog|journal-remote) method=$arg;;
+        --tls) tls=1;;
+        --plain) tls=0;;
+        *) echo "Unknown argument: $arg"; exit 1;;
+    esac
+done
+if [ -z "$method" ]; then
+    echo "Usage: sh confluent-logging-receiver-setup <rsyslog|journal-remote> [--tls|--plain]"
+    exit 1
+fi
+
+# issue_tls_cert <keyout> <certout> <cadest>: generate a key and certificate
+# signed by the local confluent CA, with SANs covering this server's names and
+# addresses. A dedicated directory is used per consumer: /etc/ssl/private is
+# not traversable by service users on Debian-style systems, and Ubuntu's
+# rsyslogd AppArmor profile only permits reads under /etc/rsyslog.d
+issue_tls_cert() {
+    mkdir -p $(dirname $1)
+    # Store the confluent CA certs into the trusted bundle used to
+    # authenticate clients
+    cat /var/lib/confluent/public/site/tls/*.pem > $3
+    myips=$(ip -j addr show 2>/dev/null | python3 -c 'import json,sys; print(",".join(a["local"] for i in json.load(sys.stdin) for a in i.get("addr_info", []) if a.get("scope") == "global"))')
+    mynames=$(hostname)
+    mydomain=$(dnsdomainname 2>/dev/null)
+    [ -n "$mydomain" ] && mynames=$mynames,$(hostname).$mydomain
+    [ -n "$myips" ] && mynames=$mynames,$myips
+    TDIR=$(mktemp -d)
+    # certutil runs the actual signing as the owner of /etc/confluent, which
+    # must be able to reach the CSR; the directory never holds the key
+    chmod 711 $TDIR
+    # Request a new certificate while generating a new key to go with it,
+    # the subject is replaced during signing
+    openssl req -new -newkey ec -pkeyopt ec_paramgen_curve:secp384r1 -nodes \
+        -keyout $1 -out $TDIR/csr.pem -subj /CN=$(hostname) > /dev/null 2>&1
+    # Have the confluent CA do an offline signing of the CSR (default
+    # validity of 3650 days)
+    (cd $TDIR && python3 /opt/confluent/lib/python/confluent/certutil.py -s $mynames -r csr.pem)
+    cp $TDIR/cert.pem $2
+    rm -rf $TDIR
+    chmod 600 $1
+    chmod 644 $2 $3
+    if [ -x /usr/sbin/restorecon ]; then
+        /usr/sbin/restorecon $1 $2 $3 2>/dev/null || true
+    fi
+}
+
+if [ $method = journal-remote ]; then
+    jrbin=""
+    for candidate in /usr/lib/systemd/systemd-journal-remote /lib/systemd/systemd-journal-remote; do
+        [ -f $candidate ] && jrbin=$candidate && break
+    done
+    if [ -z "$jrbin" ]; then
+        echo "systemd-journal-remote does not appear to be installed (install the systemd-journal-remote package)"
+        exit 1
+    fi
+    if [ $tls = 1 ]; then
+        # builds without gnutls reject --trust with "not available"; the exit
+        # status alone cannot be used, as gnutls builds also fail (with a
+        # conflict complaint) once a TrustedCertificateFile is configured
+        if $jrbin --trust=all --version 2>&1 | grep -qi 'not available'; then
+            echo "WARNING: this build of systemd-journal-remote was compiled without client"
+            echo "WARNING: certificate verification (its TrustedCertificateFile setting is"
+            echo "WARNING: silently ignored). The transport will be encrypted and nodes will"
+            echo "WARNING: verify this server, but uploads from ANY client will be accepted."
+            echo "WARNING: Use logging.method=rsyslog if client verification is required."
+        fi
+        tlsdir=/etc/ssl/confluent-logging
+        issue_tls_cert $tlsdir/journal-remote.key $tlsdir/journal-remote.crt $tlsdir/ca.pem
+        chown systemd-journal-remote $tlsdir/journal-remote.key
+        # Setting TrustedCertificateFile explicitly is what makes
+        # systemd-journal-remote verify client certificates; do not rely on
+        # the compiled-in defaults
+        mkdir -p /etc/systemd/journal-remote.conf.d
+        cat > /etc/systemd/journal-remote.conf.d/confluent.conf << 'EOF'
+# Log reception configured by confluent-logging-receiver-setup
+# Note: TrustedCertificateFile (client certificate verification) is silently
+# ignored by systemd-journal-remote builds lacking gnutls support
+[Remote]
+ServerKeyFile=/etc/ssl/confluent-logging/journal-remote.key
+ServerCertificateFile=/etc/ssl/confluent-logging/journal-remote.crt
+TrustedCertificateFile=/etc/ssl/confluent-logging/ca.pem
+EOF
+        # remove a plain-HTTP override from a previous --plain run
+        rm -f /etc/systemd/system/systemd-journal-remote.service.d/confluent.conf
+    else
+        mkdir -p /etc/systemd/system/systemd-journal-remote.service.d
+        cp journal-remote/confluent-journal-remote.conf /etc/systemd/system/systemd-journal-remote.service.d/confluent.conf
+        rm -f /etc/systemd/journal-remote.conf.d/confluent.conf
+    fi
+    systemctl daemon-reload
+    systemctl enable systemd-journal-remote.socket > /dev/null 2>&1
+    systemctl try-restart systemd-journal-remote.service 2>/dev/null || true
+    systemctl restart systemd-journal-remote.socket
+    echo "journal-remote reception configured; permit TCP port 19532 through the firewall, e.g.:"
+    echo "  firewall-cmd --permanent --add-port=19532/tcp && firewall-cmd --reload"
+else
+    if [ ! -d /etc/rsyslog.d ]; then
+        echo "rsyslog does not appear to be installed"
+        exit 1
+    fi
+    if [ $tls = 1 ]; then
+        tlsdriver=""
+        for moddir in /usr/lib64/rsyslog /usr/lib/rsyslog /usr/lib/*-linux-gnu/rsyslog; do
+            [ -f $moddir/lmnsd_ossl.so ] && tlsdriver=ossl && break
+            [ -f $moddir/lmnsd_gtls.so ] && tlsdriver=gtls && break
+        done
+        if [ -z "$tlsdriver" ]; then
+            echo "TLS reception requires the rsyslog-openssl or rsyslog-gnutls module"
+            exit 1
+        fi
+        tlsdir=/etc/rsyslog.d/confluent-tls
+        issue_tls_cert $tlsdir/rsyslog.key $tlsdir/rsyslog.crt $tlsdir/ca.pem
+        if getent passwd syslog > /dev/null 2>&1; then
+            # e.g. Ubuntu runs rsyslogd as the syslog user
+            chgrp syslog $tlsdir/rsyslog.key
+            chmod g+r $tlsdir/rsyslog.key
+        fi
+        sed 's/"ossl"/"'$tlsdriver'"/' rsyslog/confluent-nodes-tls.conf > /etc/rsyslog.d/confluent-nodes.conf
+        echo "TLS rsyslog reception configured; permit TCP port 6514 through the firewall, e.g.:"
+        echo "  firewall-cmd --permanent --add-port=6514/tcp && firewall-cmd --reload"
+    else
+        cp rsyslog/confluent-nodes.conf /etc/rsyslog.d/confluent-nodes.conf
+        echo "Plain rsyslog reception configured; permit UDP port 514 through the firewall, e.g.:"
+        echo "  firewall-cmd --permanent --add-service=syslog && firewall-cmd --reload"
+    fi
+    if [ -x /usr/sbin/restorecon ]; then
+        /usr/sbin/restorecon /etc/rsyslog.d/confluent-nodes.conf 2>/dev/null || true
+    fi
+    mkdir -p /var/log/confluent/nodes
+    if getent passwd syslog > /dev/null 2>&1; then
+        # e.g. Ubuntu runs rsyslogd as the syslog user
+        chown syslog /var/log/confluent/nodes
+    fi
+    systemctl restart rsyslog
+fi
diff --git a/confluent_server/logging/journal-remote/confluent-journal-remote.conf b/confluent_server/logging/journal-remote/confluent-journal-remote.conf
index 69cdef76b..15cd14356 100644
--- a/confluent_server/logging/journal-remote/confluent-journal-remote.conf
+++ b/confluent_server/logging/journal-remote/confluent-journal-remote.conf
@@ -4,7 +4,8 @@
 # only listens for HTTPS; this drop-in switches it to plain HTTP on port
 # 19532, matching what systemd-journal-upload is configured for on the nodes.
 #
-# To activate:
+# To activate (or run the confluent-logging-receiver-setup helper one
+# directory up with "journal-remote --plain"):
 #   dnf/zypper/apt install systemd-journal-remote
 #   mkdir -p /etc/systemd/system/systemd-journal-remote.service.d
 #   cp confluent-journal-remote.conf /etc/systemd/system/systemd-journal-remote.service.d/confluent.conf
diff --git a/confluent_server/logging/rsyslog/confluent-nodes-tls.conf b/confluent_server/logging/rsyslog/confluent-nodes-tls.conf
new file mode 100644
index 000000000..fb08efb69
--- /dev/null
+++ b/confluent_server/logging/rsyslog/confluent-nodes-tls.conf
@@ -0,0 +1,42 @@
+# Example rsyslog configuration for a confluent server to receive TLS
+# encrypted syslog forwarded by deployed nodes (see the logging.servers and
+# logging.tls node attributes) and write per-node logs under
+# /var/log/confluent/nodes/. Clients are mutually authenticated: only nodes
+# presenting a certificate issued by the confluent certificate authority are
+# accepted.
+#
+# The confluent-logging-receiver-setup helper in this examples directory
+# generates the referenced certificate material and installs this file; to
+# activate manually, provide the key/cert/CA files below, then:
+#   cp confluent-nodes-tls.conf /etc/rsyslog.d/confluent-nodes.conf
+#   systemctl restart rsyslog
+# and permit TCP port 6514 through the firewall, e.g. on a firewalld based
+# system:
+#   firewall-cmd --permanent --add-port=6514/tcp && firewall-cmd --reload
+#
+# The TLS stream driver requires the rsyslog-openssl (driver "ossl") or
+# rsyslog-gnutls (driver "gtls") module to be installed.
+#
+# Note: on systems where rsyslogd does not run as root (e.g. Ubuntu runs it
+# as the "syslog" user), ensure that user may read the certificate material
+# and may create and write /var/log/confluent/nodes
+#
+# The per-node logs can grow large; see confluent-nodes.logrotate for an
+# example logrotate configuration that rotates and compresses them.
+
+global(DefaultNetstreamDriver="ossl"
+       DefaultNetstreamDriverCAFile="/etc/rsyslog.d/confluent-tls/ca.pem"
+       DefaultNetstreamDriverCertFile="/etc/rsyslog.d/confluent-tls/rsyslog.crt"
+       DefaultNetstreamDriverKeyFile="/etc/rsyslog.d/confluent-tls/rsyslog.key")
+
+module(load="imtcp" StreamDriver.Name="ossl" StreamDriver.Mode="1"
+       StreamDriver.AuthMode="x509/certvalid")
+input(type="imtcp" port="6514" ruleset="confluentnodes")
+
+template(name="confluentnodelog" type="string"
+         string="/var/log/confluent/nodes/%HOSTNAME%.log")
+
+ruleset(name="confluentnodes") {
+    action(type="omfile" dynaFile="confluentnodelog"
+           dirCreateMode="0700" fileCreateMode="0600")
+}
diff --git a/confluent_server/setup.py.tmpl b/confluent_server/setup.py.tmpl
index d79c844a1..b11bb4a0d 100644
--- a/confluent_server/setup.py.tmpl
+++ b/confluent_server/setup.py.tmpl
@@ -40,7 +40,9 @@ setup(
     data_files=[('/etc/init.d', ['sysvinit/confluent']),
                 ('/usr/lib/sysctl.d', ['sysctl/confluent.conf']),
                 ('/opt/confluent/share/licenses/confluent_server', ['LICENSE', 'COPYRIGHT']),
+                ('/opt/confluent/share/examples/logging', ['logging/confluent-logging-receiver-setup']),
                 ('/opt/confluent/share/examples/logging/rsyslog', ['logging/rsyslog/confluent-nodes.conf',
+                                                                   'logging/rsyslog/confluent-nodes-tls.conf',
                                                                    'logging/rsyslog/confluent-nodes.logrotate']),
                 ('/opt/confluent/share/examples/logging/journal-remote', ['logging/journal-remote/confluent-journal-remote.conf']),
                 ('/usr/lib/systemd/system', ['systemd/confluent.service']),

From ac79c13514bc8adfac47c265a43d0eb71a530397 Mon Sep 17 00:00:00 2001
From: Markus Hilger <markus.hilger@megware.com>
Date: Tue, 14 Jul 2026 20:01:16 +0200
Subject: [PATCH 3/9] File received node logs by sender identity

The per-node log files on the receiving server were named after the client-supplied syslog HOSTNAME, which any authenticated sender can forge, allowing one node to inject entries into another node's log file. Name the files by FROMHOST instead (the sender identity as resolved by the receiving server from the peer address) so injected entries always land under the actual sender.
---
 confluent_server/logging/rsyslog/confluent-nodes-tls.conf | 8 +++++++-
 confluent_server/logging/rsyslog/confluent-nodes.conf     | 7 ++++++-
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/confluent_server/logging/rsyslog/confluent-nodes-tls.conf b/confluent_server/logging/rsyslog/confluent-nodes-tls.conf
index fb08efb69..db582f9f8 100644
--- a/confluent_server/logging/rsyslog/confluent-nodes-tls.conf
+++ b/confluent_server/logging/rsyslog/confluent-nodes-tls.conf
@@ -23,6 +23,12 @@
 #
 # The per-node logs can grow large; see confluent-nodes.logrotate for an
 # example logrotate configuration that rotates and compresses them.
+#
+# Logs are filed by the sender identity as determined by this server
+# (resolution of the peer address, falling back to the address itself), not
+# by the client-supplied hostname, so nodes cannot inject entries into each
+# other's logs. Note that any client presenting a certificate from the
+# confluent CA is accepted.
 
 global(DefaultNetstreamDriver="ossl"
        DefaultNetstreamDriverCAFile="/etc/rsyslog.d/confluent-tls/ca.pem"
@@ -34,7 +40,7 @@ module(load="imtcp" StreamDriver.Name="ossl" StreamDriver.Mode="1"
 input(type="imtcp" port="6514" ruleset="confluentnodes")
 
 template(name="confluentnodelog" type="string"
-         string="/var/log/confluent/nodes/%HOSTNAME%.log")
+         string="/var/log/confluent/nodes/%FROMHOST%.log")
 
 ruleset(name="confluentnodes") {
     action(type="omfile" dynaFile="confluentnodelog"
diff --git a/confluent_server/logging/rsyslog/confluent-nodes.conf b/confluent_server/logging/rsyslog/confluent-nodes.conf
index 5aa8928d9..f30062823 100644
--- a/confluent_server/logging/rsyslog/confluent-nodes.conf
+++ b/confluent_server/logging/rsyslog/confluent-nodes.conf
@@ -14,12 +14,17 @@
 #
 # The per-node logs can grow large; see confluent-nodes.logrotate for an
 # example logrotate configuration that rotates and compresses them.
+#
+# Logs are filed by the sender identity as determined by this server
+# (resolution of the peer address, falling back to the address itself), not
+# by the client-supplied hostname, so nodes cannot trivially inject entries
+# into each other's logs.
 
 module(load="imudp")
 input(type="imudp" port="514" ruleset="confluentnodes")
 
 template(name="confluentnodelog" type="string"
-         string="/var/log/confluent/nodes/%HOSTNAME%.log")
+         string="/var/log/confluent/nodes/%FROMHOST%.log")
 
 ruleset(name="confluentnodes") {
     action(type="omfile" dynaFile="confluentnodelog"

From f4b86829701f64bdb1a15f2a429373554334325d Mon Sep 17 00:00:00 2001
From: Markus Hilger <markus.hilger@megware.com>
Date: Tue, 14 Jul 2026 20:07:17 +0200
Subject: [PATCH 4/9] Document the revocation limitation of log forwarding TLS

The log transports offer no certificate revocation, so note in the logging.tls attribute description that pubkeys.tls_lifetime should be chosen with node decommissioning in mind when raising it.
---
 confluent_server/confluent/config/attributes.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/confluent_server/confluent/config/attributes.py b/confluent_server/confluent/config/attributes.py
index 3e1aec9ff..610c4e0cc 100644
--- a/confluent_server/confluent/config/attributes.py
+++ b/confluent_server/confluent/config/attributes.py
@@ -743,7 +743,10 @@
                        'its validity is governed by pubkeys.tls_lifetime '
                        '(default 47 days) - consider raising it (e.g. 3650) '
                        'so forwarding does not stop when the certificate '
-                       'expires. To set up the receiving side, see the '
+                       'expires, but note that the log transports offer no '
+                       'certificate revocation, so choose the lifetime with '
+                       'node decommissioning in mind. To set up the '
+                       'receiving side, see the '
                        'confluent-logging-receiver-setup helper under '
                        '/opt/confluent/share/examples/logging.',
         'type': bool,

From ad39b3c051c8189715a70558a4fd67de1e72b04a Mon Sep 17 00:00:00 2001
From: Markus Hilger <markus.hilger@megware.com>
Date: Tue, 14 Jul 2026 23:21:55 +0200
Subject: [PATCH 5/9] Pin the log server identity for name-valued
 logging.servers

The confluent CA issues certificates to every node and BMC, all technically usable for TLS server authentication, so validating only the certificate chain allows any of them to pose as the logging server towards forwarding nodes if traffic can be redirected. When a logging.servers entry is a DNS name, forward with x509/name authentication pinning that name as the only permitted peer. The receiver certificates issued by confluent-logging-receiver-setup already carry the server names as DNS-type subject alternative names. rsyslog matches pinned peers only against DNS-type names, so address-valued entries keep chain validation (x509/certvalid); the logging.servers documentation now recommends names for this reason. The journal-remote method needs no distinction, as systemd-journal-upload verifies the URL host against the certificate either way.
---
 .../common/profile/scripts/setuplogging          | 16 +++++++++++++++-
 confluent_server/confluent/config/attributes.py  |  9 +++++++--
 2 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/confluent_osdeploy/common/profile/scripts/setuplogging b/confluent_osdeploy/common/profile/scripts/setuplogging
index 4ec95a0c5..b3f4fe02b 100644
--- a/confluent_osdeploy/common/profile/scripts/setuplogging
+++ b/confluent_osdeploy/common/profile/scripts/setuplogging
@@ -166,8 +166,22 @@ else
         echo '       DefaultNetstreamDriverCAFile="'$tlsdir'/ca.pem"' >> $syslogconf
         echo '       DefaultNetstreamDriverCertFile="'$tlsdir'/rsyslog.crt"' >> $syslogconf
         echo '       DefaultNetstreamDriverKeyFile="'$tlsdir'/rsyslog.key")' >> $syslogconf
+        # For name-valued targets, pin the server identity (x509/name): the
+        # confluent CA also issues certificates to every node and BMC, and
+        # with chain validation alone any of them could pose as the server.
+        # rsyslog matches pinned peers only against DNS-type SANs, so IP
+        # targets can only validate the chain (x509/certvalid).
         for loggingsrv in $loggingservers; do
-            echo '*.* action(type="omfwd" target="'$loggingsrv'" port="6514" protocol="tcp" StreamDriverMode="1" StreamDriverAuthMode="x509/certvalid")' >> $syslogconf
+            case $loggingsrv in
+                *:*) isname=0 ;;        # IPv6 literal
+                *[!0-9.]*) isname=1 ;;  # anything besides digits and dots
+                *) isname=0 ;;          # IPv4 literal
+            esac
+            if [ $isname = 1 ]; then
+                echo '*.* action(type="omfwd" target="'$loggingsrv'" port="6514" protocol="tcp" StreamDriverMode="1" StreamDriverAuthMode="x509/name" StreamDriverPermittedPeers="'$loggingsrv'")' >> $syslogconf
+            else
+                echo '*.* action(type="omfwd" target="'$loggingsrv'" port="6514" protocol="tcp" StreamDriverMode="1" StreamDriverAuthMode="x509/certvalid")' >> $syslogconf
+            fi
         done
     else
         echo '# Syslog forwarding configured by confluent during deployment' > $syslogconf
diff --git a/confluent_server/confluent/config/attributes.py b/confluent_server/confluent/config/attributes.py
index 610c4e0cc..9bdba9ed0 100644
--- a/confluent_server/confluent/config/attributes.py
+++ b/confluent_server/confluent/config/attributes.py
@@ -709,8 +709,13 @@
         'description': 'Remote logging server or servers (comma separated) '
                        'for the OS deployment to configure the node to '
                        'forward logs to, using the method indicated by '
-                       'logging.method. May be set to the address of the '
-                       'deployment server or any other logging server. See '
+                       'logging.method. May be set to the deployment server '
+                       'or any other logging server. With logging.tls and '
+                       'the rsyslog method, prefer DNS names resolvable by '
+                       'the nodes over IP addresses: a name lets the node '
+                       'verify the specific server identity, while an IP '
+                       'only validates that the certificate came from the '
+                       'confluent CA. See '
                        '/opt/confluent/share/examples/logging for cfg '
                        'examples. If unset, no forwarding is configured.',
     },

From e844ea2d20a80cd6398822a02bd7a4a24bf75f0f Mon Sep 17 00:00:00 2001
From: Markus Hilger <markus.hilger@megware.com>
Date: Wed, 15 Jul 2026 21:23:37 +0200
Subject: [PATCH 6/9] Buffer forwarded logs across log server outages

The TLS forwarding actions run in rsyslog's main queue by default, so a
log server that stops reading (as opposed to one that is down, which
merely suspends the action) blocks the omfwd send and with it every
other action on the node, including local log files. Give each
forwarding action a bounded in-memory queue so an unreachable or hung
receiver costs at most the queued messages: local logging continues,
messages buffer across shorter outages and are dropped only once the
queue fills. The non-TLS UDP method needs no queue, as it cannot block.
---
 confluent_osdeploy/common/profile/scripts/setuplogging | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/confluent_osdeploy/common/profile/scripts/setuplogging b/confluent_osdeploy/common/profile/scripts/setuplogging
index b3f4fe02b..821d367ea 100644
--- a/confluent_osdeploy/common/profile/scripts/setuplogging
+++ b/confluent_osdeploy/common/profile/scripts/setuplogging
@@ -171,6 +171,10 @@ else
         # with chain validation alone any of them could pose as the server.
         # rsyslog matches pinned peers only against DNS-type SANs, so IP
         # targets can only validate the chain (x509/certvalid).
+        # The bounded async queue keeps an unreachable or hung log server
+        # from wedging the main queue (and with it local logging); once it
+        # fills, further forwarded messages are dropped instead.
+        fwdqueue='action.resumeRetryCount="-1" queue.type="linkedList" queue.size="10000"'
         for loggingsrv in $loggingservers; do
             case $loggingsrv in
                 *:*) isname=0 ;;        # IPv6 literal
@@ -178,9 +182,9 @@ else
                 *) isname=0 ;;          # IPv4 literal
             esac
             if [ $isname = 1 ]; then
-                echo '*.* action(type="omfwd" target="'$loggingsrv'" port="6514" protocol="tcp" StreamDriverMode="1" StreamDriverAuthMode="x509/name" StreamDriverPermittedPeers="'$loggingsrv'")' >> $syslogconf
+                echo '*.* action(type="omfwd" target="'$loggingsrv'" port="6514" protocol="tcp" StreamDriverMode="1" StreamDriverAuthMode="x509/name" StreamDriverPermittedPeers="'$loggingsrv'" '"$fwdqueue"')' >> $syslogconf
             else
-                echo '*.* action(type="omfwd" target="'$loggingsrv'" port="6514" protocol="tcp" StreamDriverMode="1" StreamDriverAuthMode="x509/certvalid")' >> $syslogconf
+                echo '*.* action(type="omfwd" target="'$loggingsrv'" port="6514" protocol="tcp" StreamDriverMode="1" StreamDriverAuthMode="x509/certvalid" '"$fwdqueue"')' >> $syslogconf
             fi
         done
     else

From b162eaac42e0c393cace46135f9d03e3f6e1eb56 Mon Sep 17 00:00:00 2001
From: Markus Hilger <markus.hilger@megware.com>
Date: Wed, 15 Jul 2026 21:48:58 +0200
Subject: [PATCH 7/9] Tune the example rsyslog receivers for 4K nodes

The minimal receiver examples run into two rsyslog defaults well below
the cluster sizes confluent deploys: imtcp refuses more than 200
concurrent connections, so most of a larger cluster of TLS forwarders
(one persistent connection each) cannot connect at all, and omfile
caches only 10 dynafile handles, reopening a file for nearly every
message once logs from many nodes interleave. Size both for clusters on
the order of 4k nodes, reap dead sessions with TCP keepalive, absorb
bursts (whole racks booting, reconnect storms after a receiver restart)
with more UDP receiver threads, a larger receive buffer and a dedicated
queue that also keeps node traffic and the server's own logging from
delaying each other, and batch the file writing asynchronously. All
parameters are accepted by the oldest rsyslog among the supported
distributions (8.2312 in Ubuntu 24.04); imtcp workerthreads is not, so
the receiver setup helper probes for it and enables it where available
rather than placing it in the example.
---
 .../logging/confluent-logging-receiver-setup  |  9 +++++++-
 .../logging/rsyslog/confluent-nodes-tls.conf  | 23 +++++++++++++++++--
 .../logging/rsyslog/confluent-nodes.conf      | 21 ++++++++++++++---
 3 files changed, 47 insertions(+), 6 deletions(-)

diff --git a/confluent_server/logging/confluent-logging-receiver-setup b/confluent_server/logging/confluent-logging-receiver-setup
index 8bca3defa..0fefd7fe5 100755
--- a/confluent_server/logging/confluent-logging-receiver-setup
+++ b/confluent_server/logging/confluent-logging-receiver-setup
@@ -143,7 +143,14 @@ else
             chgrp syslog $tlsdir/rsyslog.key
             chmod g+r $tlsdir/rsyslog.key
         fi
-        sed 's/"ossl"/"'$tlsdriver'"/' rsyslog/confluent-nodes-tls.conf > /etc/rsyslog.d/confluent-nodes.conf
+        # spread the TLS sessions over several input workers where the
+        # imtcp workerthreads parameter exists (>=8.2504.0)
+        # has it); probe the local rsyslogd rather than track versions
+        workerthreads=""
+        if ! echo 'module(load="imtcp" workerthreads="4")' | rsyslogd -N1 -f /dev/stdin 2>&1 | grep -q 'not known'; then
+            workerthreads=' workerthreads="4"'
+        fi
+        sed -e 's/"ossl"/"'$tlsdriver'"/' -e 's/keepAlive="on"/&'"$workerthreads"'/' rsyslog/confluent-nodes-tls.conf > /etc/rsyslog.d/confluent-nodes.conf
         echo "TLS rsyslog reception configured; permit TCP port 6514 through the firewall, e.g.:"
         echo "  firewall-cmd --permanent --add-port=6514/tcp && firewall-cmd --reload"
     else
diff --git a/confluent_server/logging/rsyslog/confluent-nodes-tls.conf b/confluent_server/logging/rsyslog/confluent-nodes-tls.conf
index db582f9f8..e1095514d 100644
--- a/confluent_server/logging/rsyslog/confluent-nodes-tls.conf
+++ b/confluent_server/logging/rsyslog/confluent-nodes-tls.conf
@@ -35,14 +35,33 @@ global(DefaultNetstreamDriver="ossl"
        DefaultNetstreamDriverCertFile="/etc/rsyslog.d/confluent-tls/rsyslog.crt"
        DefaultNetstreamDriverKeyFile="/etc/rsyslog.d/confluent-tls/rsyslog.key")
 
+# Every node holds a persistent TLS connection: keep maxSessions at or
+# above the node count, or rsyslog refuses the connections beyond it (the
+# default is 200). keepAlive reaps the sessions of crashed or reset nodes so
+# they do not accumulate against the limit. On rsyslog builds recent enough
+# to know the imtcp workerthreads parameter (>=8.2504.0),
+# adding e.g. workerthreads="4" additionally spreads the TLS sessions over
+# several input workers; confluent-logging-receiver-setup adds it when the
+# local rsyslogd accepts it.
 module(load="imtcp" StreamDriver.Name="ossl" StreamDriver.Mode="1"
-       StreamDriver.AuthMode="x509/certvalid")
+       StreamDriver.AuthMode="x509/certvalid"
+       maxSessions="4096" keepAlive="on")
 input(type="imtcp" port="6514" ruleset="confluentnodes")
 
 template(name="confluentnodelog" type="string"
          string="/var/log/confluent/nodes/%FROMHOST%.log")
 
-ruleset(name="confluentnodes") {
+# The dedicated queue keeps bursts from the nodes and this server's own
+# logging from delaying each other, and buffers reconnect storms after a
+# restart. The dynafile handle cache must stay at or above the node count
+# (the default of 10 would thrash with logs interleaving from many per-node
+# files, closing and reopening one for nearly every message); handles and
+# their io buffers are only consumed for files actually open. Async batched
+# writing means a node's file may trail the sender by up to ~2s when
+# tailed live.
+ruleset(name="confluentnodes" queue.type="fixedArray" queue.size="50000") {
     action(type="omfile" dynaFile="confluentnodelog"
+           dynaFileCacheSize="4096" asyncWriting="on" ioBufferSize="64k"
+           flushOnTXEnd="off" flushInterval="2"
            dirCreateMode="0700" fileCreateMode="0600")
 }
diff --git a/confluent_server/logging/rsyslog/confluent-nodes.conf b/confluent_server/logging/rsyslog/confluent-nodes.conf
index f30062823..caea42584 100644
--- a/confluent_server/logging/rsyslog/confluent-nodes.conf
+++ b/confluent_server/logging/rsyslog/confluent-nodes.conf
@@ -20,13 +20,28 @@
 # by the client-supplied hostname, so nodes cannot trivially inject entries
 # into each other's logs.
 
-module(load="imudp")
-input(type="imudp" port="514" ruleset="confluentnodes")
+# Sized for clusters on the order of 4k nodes: rsyslog's default single
+# receiver thread and socket buffer keep up with steady traffic but drop
+# messages on synchronized bursts (e.g. a whole rack booting, or a reconnect
+# storm), so spread reception over several threads and enlarge the receive
+# buffer. rsyslog raises the buffer with SO_RCVBUFFORCE while still
+# privileged, so no rmem sysctl is needed.
+module(load="imudp" threads="4" batchSize="128")
+input(type="imudp" port="514" ruleset="confluentnodes" rcvbufSize="16m")
 
 template(name="confluentnodelog" type="string"
          string="/var/log/confluent/nodes/%FROMHOST%.log")
 
-ruleset(name="confluentnodes") {
+# The dedicated queue keeps bursts from the nodes and this server's own
+# logging from delaying each other. The dynafile handle cache must stay at
+# or above the node count (the default of 10 would thrash with logs
+# interleaving from many per-node files, closing and reopening one for
+# nearly every message); handles and their io buffers are only consumed for
+# files actually open. Async batched writing means a node's file may trail
+# the sender by up to ~2s when tailed live.
+ruleset(name="confluentnodes" queue.type="fixedArray" queue.size="50000") {
     action(type="omfile" dynaFile="confluentnodelog"
+           dynaFileCacheSize="4096" asyncWriting="on" ioBufferSize="64k"
+           flushOnTXEnd="off" flushInterval="2"
            dirCreateMode="0700" fileCreateMode="0600")
 }

From 7803e08665cc533e6252f7da372fe796ed5387fc Mon Sep 17 00:00:00 2001
From: Markus Hilger <markus.hilger@megware.com>
Date: Wed, 15 Jul 2026 22:24:07 +0200
Subject: [PATCH 8/9] Automatically renew log forwarding certificates

The forwarding certificate was issued once during deployment, so with
the default 47 day pubkeys.tls_lifetime the log forwarding silently
stopped when it expired, and the documented workaround of a multi-year
lifetime directly worsened the lack of revocation in the log
transports. Have setuplogging install a renewal script and a daily
randomized systemd timer that requests a fresh certificate through the
deployment API once less than half of the validity remains.
A failed renewal never disturbs the material in use: the replacement
is staged beside the live files and only swapped in, and the consuming
service only restarted, on success.
---
 .../common/profile/scripts/setuplogging       | 106 ++++++++++++++++++
 .../confluent/config/attributes.py            |  14 +--
 2 files changed, 112 insertions(+), 8 deletions(-)

diff --git a/confluent_osdeploy/common/profile/scripts/setuplogging b/confluent_osdeploy/common/profile/scripts/setuplogging
index 821d367ea..077ddb03f 100644
--- a/confluent_osdeploy/common/profile/scripts/setuplogging
+++ b/confluent_osdeploy/common/profile/scripts/setuplogging
@@ -21,6 +21,108 @@ loggingtls=$(sed -n 's/^loggingtls: //p' /etc/confluent/confluent.deploycfg)
 if [ -z "$loggingmethod" ]; then
     loggingmethod=rsyslog
 fi
+if [ "$loggingtls" != true ]; then
+    # drop renewal artifacts from a previous TLS-enabled run
+    rm -f /etc/confluent/logging-tls-renew
+    if [ -d /run/systemd/system ]; then
+        systemctl disable --now confluent-logging-renew.timer 2>/dev/null
+    fi
+    rm -f /etc/systemd/system/confluent-logging-renew.service /etc/systemd/system/confluent-logging-renew.timer
+fi
+
+# setup_cert_renewal <keyfile> <certfile> <keygroup> <service>: install a
+# daily randomized systemd timer that renews the forwarding certificate once
+# less than half of its validity remains, so short pubkeys.tls_lifetime
+# values (including the 47 day default) keep working. Renewal authenticates
+# with the api key persisted during deployment, which remains valid
+# regardless of deployment.apiarmed; the server derives the certificate
+# identity from its node configuration, not from the request.
+setup_cert_renewal() {
+    renewscript=/etc/confluent/logging-tls-renew
+    {
+        echo '#!/bin/sh'
+        echo '# Renew the confluent log forwarding TLS certificate when less than half'
+        echo '# of its validity remains; installed by the confluent setuplogging'
+        echo '# deployment script (see the logging.tls node attribute)'
+        echo "keyfile=$1"
+        echo "certfile=$2"
+        echo "keygroup=$3"
+        echo "service=$4"
+        cat << 'RENEWEOF'
+umask 077
+[ -s "$certfile" ] || exit 0
+notbefore=$(openssl x509 -startdate -noout -in "$certfile" | cut -d= -f2-)
+notafter=$(openssl x509 -enddate -noout -in "$certfile" | cut -d= -f2-)
+startts=$(date -d "$notbefore" +%s 2>/dev/null)
+endts=$(date -d "$notafter" +%s 2>/dev/null)
+now=$(date +%s)
+if [ -n "$startts" ] && [ -n "$endts" ] && [ $((endts - now)) -gt $(( (endts - startts) / 2 )) ]; then
+    exit 0
+fi
+confapiclient=""
+[ -f /opt/confluent/bin/apiclient ] && confapiclient=/opt/confluent/bin/apiclient
+[ -f /etc/confluent/apiclient ] && confapiclient=/etc/confluent/apiclient
+if [ -z "$confapiclient" ]; then
+    logger -t confluent-logging-renew "unable to locate the confluent apiclient, cannot renew $certfile"
+    exit 0
+fi
+TDIR=$(mktemp -d)
+openssl req -new -newkey ec -pkeyopt ec_paramgen_curve:secp384r1 -nodes \
+    -keyout $TDIR/key.pem -out $TDIR/csr.pem -subj /CN=$(hostname) > /dev/null 2>&1
+python3 $confapiclient /confluent-api/self/tlscert $TDIR/csr.pem -o $TDIR/cert.pem
+if [ ! -s $TDIR/cert.pem ]; then
+    rm -rf $TDIR
+    logger -t confluent-logging-renew "failed to obtain a renewed certificate from the confluent CA for $certfile, will retry"
+    exit 0
+fi
+# stage beside the live files (same filesystem), then replace atomically;
+# a failure never disturbs the material currently in use
+cp $TDIR/key.pem $keyfile.new
+cp $TDIR/cert.pem $certfile.new
+rm -rf $TDIR
+chmod 644 $certfile.new
+if [ -n "$keygroup" ]; then
+    chgrp $keygroup $keyfile.new 2>/dev/null
+    chmod g+r $keyfile.new
+fi
+if [ -x /usr/sbin/restorecon ]; then
+    /usr/sbin/restorecon $keyfile.new $certfile.new 2>/dev/null
+fi
+mv $keyfile.new $keyfile
+mv $certfile.new $certfile
+logger -t confluent-logging-renew "renewed $certfile (expires $(openssl x509 -enddate -noout -in $certfile | cut -d= -f2-))"
+systemctl try-restart $service 2>/dev/null
+exit 0
+RENEWEOF
+    } > $renewscript
+    chmod 755 $renewscript
+    if [ -x /usr/sbin/restorecon ]; then
+        /usr/sbin/restorecon $renewscript 2>/dev/null
+    fi
+    [ -d /run/systemd/system ] || return 0
+    cat > /etc/systemd/system/confluent-logging-renew.service << EOF
+[Unit]
+Description=Renew the confluent log forwarding TLS certificate
+
+[Service]
+Type=oneshot
+ExecStart=$renewscript
+EOF
+    cat > /etc/systemd/system/confluent-logging-renew.timer << EOF
+[Unit]
+Description=Daily confluent log forwarding TLS certificate renewal check
+
+[Timer]
+OnCalendar=daily
+RandomizedDelaySec=6h
+Persistent=true
+
+[Install]
+WantedBy=timers.target
+EOF
+    systemctl daemon-reload 2>/dev/null
+    systemctl enable --now confluent-logging-renew.timer > /dev/null 2>&1
+}
 
 # request_tls_cert <keyout> <certout>: generate a key and have the confluent
 # CA sign a certificate for this node via the deployment API
@@ -111,6 +213,7 @@ if [ "$loggingmethod" = journal-remote ]; then
         fi
         chgrp systemd-journal $tlsdir/journal-upload.key 2>/dev/null
         chmod g+r $tlsdir/journal-upload.key
+        setup_cert_renewal $tlsdir/journal-upload.key $tlsdir/journal-upload.crt systemd-journal systemd-journal-upload.service
         echo 'URL=https://'$loggingsrv':19532' >> $journalconf
         echo 'ServerKeyFile='$tlsdir'/journal-upload.key' >> $journalconf
         echo 'ServerCertificateFile='$tlsdir'/journal-upload.crt' >> $journalconf
@@ -156,11 +259,14 @@ else
         if ! request_tls_cert $tlsdir/rsyslog.key $tlsdir/rsyslog.crt $tlsdir/ca.pem; then
             exit 0
         fi
+        rsyslogkeygroup=""
         if getent passwd syslog > /dev/null 2>&1; then
             # e.g. Ubuntu runs rsyslogd as the syslog user
+            rsyslogkeygroup=syslog
             chgrp syslog $tlsdir/rsyslog.key 2>/dev/null
             chmod g+r $tlsdir/rsyslog.key
         fi
+        setup_cert_renewal $tlsdir/rsyslog.key $tlsdir/rsyslog.crt "$rsyslogkeygroup" rsyslog.service
         echo '# Syslog forwarding configured by confluent during deployment' > $syslogconf
         echo 'global(DefaultNetstreamDriver="'$tlsdriver'"' >> $syslogconf
         echo '       DefaultNetstreamDriverCAFile="'$tlsdir'/ca.pem"' >> $syslogconf
diff --git a/confluent_server/confluent/config/attributes.py b/confluent_server/confluent/config/attributes.py
index 9bdba9ed0..c9a88a823 100644
--- a/confluent_server/confluent/config/attributes.py
+++ b/confluent_server/confluent/config/attributes.py
@@ -744,14 +744,12 @@
                        'the node verifies the server, but such receivers '
                        'accept uploads from any client; use the rsyslog '
                        'method when client verification is required. '
-                       'The node certificate is issued during deployment and '
-                       'its validity is governed by pubkeys.tls_lifetime '
-                       '(default 47 days) - consider raising it (e.g. 3650) '
-                       'so forwarding does not stop when the certificate '
-                       'expires, but note that the log transports offer no '
-                       'certificate revocation, so choose the lifetime with '
-                       'node decommissioning in mind. To set up the '
-                       'receiving side, see the '
+                       'The node certificate is issued during deployment '
+                       'with validity governed by pubkeys.tls_lifetime '
+                       '(default 47 days) and renewed automatically: a '
+                       'daily randomized systemd timer on the node requests '
+                       'a fresh certificate once less than half of the '
+                       'validity remains. To set up the receiving side, see the '
                        'confluent-logging-receiver-setup helper under '
                        '/opt/confluent/share/examples/logging.',
         'type': bool,

From 3ca0cb07904b42bb635b5935fa9eff9d1730e3c8 Mon Sep 17 00:00:00 2001
From: Markus Hilger <markus.hilger@megware.com>
Date: Thu, 16 Jul 2026 03:32:18 +0200
Subject: [PATCH 9/9] Receive plaintext node logs over TCP instead of UDP

The plaintext log-forwarding path used UDP (imudp receiver, omfwd
protocol="udp"), which silently drops messages once a burst outruns the
receiver's socket buffer -- exactly the synchronized bursts a large cluster
produces (a whole rack booting, a reconnect storm after the receiver
restarts). Switch it to plain TCP: imptcp on the receiver and protocol="tcp"
on the node side. TCP back-pressures the senders instead of dropping, so
bursts are absorbed rather than lost, and imptcp (unlike imtcp) imposes no
session limit, so every node can hold a persistent connection up to the
receiver's open-file limit. The node side gains the same bounded async retry
queue the TLS path already uses, since a TCP action can block where UDP could
not.

Measured on ~4k simulated senders, imptcp delivered every message losslessly
where imudp shed 50-75% under the same overload, for the cost of one
persistent connection per node.
---
 .../common/profile/scripts/setuplogging          |  4 +++-
 confluent_server/confluent/config/attributes.py  |  2 +-
 .../logging/confluent-logging-receiver-setup     |  6 +++---
 .../logging/rsyslog/confluent-nodes.conf         | 16 ++++++----------
 4 files changed, 13 insertions(+), 15 deletions(-)

diff --git a/confluent_osdeploy/common/profile/scripts/setuplogging b/confluent_osdeploy/common/profile/scripts/setuplogging
index 077ddb03f..8372778b4 100644
--- a/confluent_osdeploy/common/profile/scripts/setuplogging
+++ b/confluent_osdeploy/common/profile/scripts/setuplogging
@@ -294,9 +294,11 @@ else
             fi
         done
     else
+        # Plain TCP forwarding (no encryption)
+        fwdqueue='action.resumeRetryCount="-1" queue.type="linkedList" queue.size="10000"'
         echo '# Syslog forwarding configured by confluent during deployment' > $syslogconf
         for loggingsrv in $loggingservers; do
-            echo '*.* action(type="omfwd" target="'$loggingsrv'" port="514" protocol="udp")' >> $syslogconf
+            echo '*.* action(type="omfwd" target="'$loggingsrv'" port="514" protocol="tcp" '"$fwdqueue"')' >> $syslogconf
         done
     fi
     if [ -x /usr/sbin/restorecon ]; then
diff --git a/confluent_server/confluent/config/attributes.py b/confluent_server/confluent/config/attributes.py
index c9a88a823..85580cd01 100644
--- a/confluent_server/confluent/config/attributes.py
+++ b/confluent_server/confluent/config/attributes.py
@@ -722,7 +722,7 @@
     'logging.method': {
         'description': 'Method used to forward logs to logging.servers. '
                        '"rsyslog" (the default if unset) forwards syslog '
-                       'using rsyslog over UDP port 514. "journal-remote" '
+                       'using rsyslog over TCP port 514. "journal-remote" '
                        'uploads the systemd journal using '
                        'systemd-journal-upload to systemd-journal-remote on '
                        'port 19532; note that systemd-journal-upload supports '
diff --git a/confluent_server/logging/confluent-logging-receiver-setup b/confluent_server/logging/confluent-logging-receiver-setup
index 0fefd7fe5..3a708e002 100755
--- a/confluent_server/logging/confluent-logging-receiver-setup
+++ b/confluent_server/logging/confluent-logging-receiver-setup
@@ -8,7 +8,7 @@
 # authenticated via the confluent certificate authority: a server certificate
 # is issued from the local confluent CA and only clients presenting a
 # certificate from that CA are accepted. With --plain, the plaintext receiver
-# examples are installed instead (rsyslog UDP port 514, journal-remote HTTP
+# examples are installed instead (rsyslog TCP port 514, journal-remote HTTP
 # port 19532).
 #
 # rsyslog reception writes per-node logs under /var/log/confluent/nodes/
@@ -155,8 +155,8 @@ else
         echo "  firewall-cmd --permanent --add-port=6514/tcp && firewall-cmd --reload"
     else
         cp rsyslog/confluent-nodes.conf /etc/rsyslog.d/confluent-nodes.conf
-        echo "Plain rsyslog reception configured; permit UDP port 514 through the firewall, e.g.:"
-        echo "  firewall-cmd --permanent --add-service=syslog && firewall-cmd --reload"
+        echo "Plain rsyslog reception configured; permit TCP port 514 through the firewall, e.g.:"
+        echo "  firewall-cmd --permanent --add-port=514/tcp && firewall-cmd --reload"
     fi
     if [ -x /usr/sbin/restorecon ]; then
         /usr/sbin/restorecon /etc/rsyslog.d/confluent-nodes.conf 2>/dev/null || true
diff --git a/confluent_server/logging/rsyslog/confluent-nodes.conf b/confluent_server/logging/rsyslog/confluent-nodes.conf
index caea42584..c7eeed1ff 100644
--- a/confluent_server/logging/rsyslog/confluent-nodes.conf
+++ b/confluent_server/logging/rsyslog/confluent-nodes.conf
@@ -5,8 +5,9 @@
 # To activate:
 #   cp confluent-nodes.conf /etc/rsyslog.d/confluent-nodes.conf
 #   systemctl restart rsyslog
-# and permit syslog through the firewall, e.g. on a firewalld based system:
-#   firewall-cmd --permanent --add-service=syslog && firewall-cmd --reload
+# and permit plain-TCP syslog through the firewall, e.g. on a firewalld based
+# system:
+#   firewall-cmd --permanent --add-port=514/tcp && firewall-cmd --reload
 #
 # Note: on systems where rsyslogd does not run as root (e.g. Ubuntu runs it
 # as the "syslog" user), ensure that user may create and write
@@ -20,14 +21,9 @@
 # by the client-supplied hostname, so nodes cannot trivially inject entries
 # into each other's logs.
 
-# Sized for clusters on the order of 4k nodes: rsyslog's default single
-# receiver thread and socket buffer keep up with steady traffic but drop
-# messages on synchronized bursts (e.g. a whole rack booting, or a reconnect
-# storm), so spread reception over several threads and enlarge the receive
-# buffer. rsyslog raises the buffer with SO_RCVBUFFORCE while still
-# privileged, so no rmem sysctl is needed.
-module(load="imudp" threads="4" batchSize="128")
-input(type="imudp" port="514" ruleset="confluentnodes" rcvbufSize="16m")
+# Nodes forward over plain TCP: each node holds one persistent connection.
+module(load="imptcp" threads="4")
+input(type="imptcp" port="514" ruleset="confluentnodes" keepAlive="on")
 
 template(name="confluentnodelog" type="string"
          string="/var/log/confluent/nodes/%FROMHOST%.log")