diff --git a/confluent_osdeploy/common/profile/scripts/setuplogging b/confluent_osdeploy/common/profile/scripts/setuplogging new file mode 100644 index 000000000..8372778b4 --- /dev/null +++ b/confluent_osdeploy/common/profile/scripts/setuplogging @@ -0,0 +1,311 @@ +#!/bin/bash +# Configure the deployed node to forward logs to the servers from the +# logging.servers attribute (surfaced as loggingservers in +# confluent.deploycfg), using the method from the logging.method attribute +# (surfaced as loggingmethod): rsyslog (the default) or journal-remote. +# When logging.tls (surfaced as loggingtls) is true, the forwarding is +# encrypted and mutually authenticated using certificates from the confluent +# certificate authority (see the confluent-logging-receiver-setup helper +# under /opt/confluent/share/examples/logging for the receiving side). +# Does nothing when logging.servers is unset. + +[ -f /lib/confluent/functions ] && . /lib/confluent/functions +[ -f /etc/confluent/functions ] && . /etc/confluent/functions + +if ! grep '^loggingservers:' /etc/confluent/confluent.deploycfg > /dev/null 2>&1; then + exit 0 +fi +loggingservers=$(sed -n '/^loggingservers:/,/^[^-]/p' /etc/confluent/confluent.deploycfg|sed 1d|sed '$d' | sed -e 's/^- //') +loggingmethod=$(sed -n 's/^loggingmethod: //p' /etc/confluent/confluent.deploycfg) +loggingtls=$(sed -n 's/^loggingtls: //p' /etc/confluent/confluent.deploycfg) +if [ -z "$loggingmethod" ]; then + loggingmethod=rsyslog +fi +if [ "$loggingtls" != true ]; then + # drop renewal artifacts from a previous TLS-enabled run + rm -f /etc/confluent/logging-tls-renew + if [ -d /run/systemd/system ]; then + systemctl disable --now confluent-logging-renew.timer 2>/dev/null + fi + rm -f /etc/systemd/system/confluent-logging-renew.service /etc/systemd/system/confluent-logging-renew.timer +fi + +# setup_cert_renewal : install a +# daily randomized systemd timer that renews the forwarding certificate once +# less than half of its validity remains, so short pubkeys.tls_lifetime +# values (including the 47 day default) keep working. Renewal authenticates +# with the api key persisted during deployment, which remains valid +# regardless of deployment.apiarmed; the server derives the certificate +# identity from its node configuration, not from the request. +setup_cert_renewal() { + renewscript=/etc/confluent/logging-tls-renew + { + echo '#!/bin/sh' + echo '# Renew the confluent log forwarding TLS certificate when less than half' + echo '# of its validity remains; installed by the confluent setuplogging' + echo '# deployment script (see the logging.tls node attribute)' + echo "keyfile=$1" + echo "certfile=$2" + echo "keygroup=$3" + echo "service=$4" + cat << 'RENEWEOF' +umask 077 +[ -s "$certfile" ] || exit 0 +notbefore=$(openssl x509 -startdate -noout -in "$certfile" | cut -d= -f2-) +notafter=$(openssl x509 -enddate -noout -in "$certfile" | cut -d= -f2-) +startts=$(date -d "$notbefore" +%s 2>/dev/null) +endts=$(date -d "$notafter" +%s 2>/dev/null) +now=$(date +%s) +if [ -n "$startts" ] && [ -n "$endts" ] && [ $((endts - now)) -gt $(( (endts - startts) / 2 )) ]; then + exit 0 +fi +confapiclient="" +[ -f /opt/confluent/bin/apiclient ] && confapiclient=/opt/confluent/bin/apiclient +[ -f /etc/confluent/apiclient ] && confapiclient=/etc/confluent/apiclient +if [ -z "$confapiclient" ]; then + logger -t confluent-logging-renew "unable to locate the confluent apiclient, cannot renew $certfile" + exit 0 +fi +TDIR=$(mktemp -d) +openssl req -new -newkey ec -pkeyopt ec_paramgen_curve:secp384r1 -nodes \ + -keyout $TDIR/key.pem -out $TDIR/csr.pem -subj /CN=$(hostname) > /dev/null 2>&1 +python3 $confapiclient /confluent-api/self/tlscert $TDIR/csr.pem -o $TDIR/cert.pem +if [ ! -s $TDIR/cert.pem ]; then + rm -rf $TDIR + logger -t confluent-logging-renew "failed to obtain a renewed certificate from the confluent CA for $certfile, will retry" + exit 0 +fi +# stage beside the live files (same filesystem), then replace atomically; +# a failure never disturbs the material currently in use +cp $TDIR/key.pem $keyfile.new +cp $TDIR/cert.pem $certfile.new +rm -rf $TDIR +chmod 644 $certfile.new +if [ -n "$keygroup" ]; then + chgrp $keygroup $keyfile.new 2>/dev/null + chmod g+r $keyfile.new +fi +if [ -x /usr/sbin/restorecon ]; then + /usr/sbin/restorecon $keyfile.new $certfile.new 2>/dev/null +fi +mv $keyfile.new $keyfile +mv $certfile.new $certfile +logger -t confluent-logging-renew "renewed $certfile (expires $(openssl x509 -enddate -noout -in $certfile | cut -d= -f2-))" +systemctl try-restart $service 2>/dev/null +exit 0 +RENEWEOF + } > $renewscript + chmod 755 $renewscript + if [ -x /usr/sbin/restorecon ]; then + /usr/sbin/restorecon $renewscript 2>/dev/null + fi + [ -d /run/systemd/system ] || return 0 + cat > /etc/systemd/system/confluent-logging-renew.service << EOF +[Unit] +Description=Renew the confluent log forwarding TLS certificate + +[Service] +Type=oneshot +ExecStart=$renewscript +EOF + cat > /etc/systemd/system/confluent-logging-renew.timer << EOF +[Unit] +Description=Daily confluent log forwarding TLS certificate renewal check + +[Timer] +OnCalendar=daily +RandomizedDelaySec=6h +Persistent=true + +[Install] +WantedBy=timers.target +EOF + systemctl daemon-reload 2>/dev/null + systemctl enable --now confluent-logging-renew.timer > /dev/null 2>&1 +} + +# request_tls_cert : generate a key and have the confluent +# CA sign a certificate for this node via the deployment API +request_tls_cert() { + if ! type confluentpython > /dev/null 2>&1; then + confluentpython() { python3 "$@"; } + fi + confapiclient="" + [ -f /opt/confluent/bin/apiclient ] && confapiclient=/opt/confluent/bin/apiclient + [ -f /etc/confluent/apiclient ] && confapiclient=/etc/confluent/apiclient + if [ -z "$confapiclient" ]; then + echo "logging.tls: unable to locate the confluent apiclient, skipping log forwarding setup" + return 1 + fi + if ! command -v openssl > /dev/null 2>&1; then + echo "logging.tls: openssl is required but not available, skipping log forwarding setup" + return 1 + fi + if [ ! -s /etc/confluent/ca.pem ]; then + echo "logging.tls: /etc/confluent/ca.pem is missing, skipping log forwarding setup" + return 1 + fi + # $3 is the destination for the CA trust anchor. A dedicated directory is + # used per consumer: /etc/ssl/private is not traversable by service users + # on Debian-style systems, and Ubuntu's rsyslogd AppArmor profile only + # permits reads under /etc/rsyslog.d + # stage beside the live files (same filesystem), then replace atomically; + # a failure never disturbs material a previous run may have left in use + mkdir -p $(dirname $1) + cat /etc/confluent/ca.pem > $3.new + CRTDIR=$(mktemp -d) + openssl req -new -newkey ec -pkeyopt ec_paramgen_curve:secp384r1 -nodes \ + -keyout $1.new -out $CRTDIR/csr.pem -subj /CN=$(hostname) > /dev/null 2>&1 + echo -n > $2.new + confluentpython $confapiclient /confluent-api/self/tlscert $CRTDIR/csr.pem -o $2.new + rm -rf $CRTDIR + if [ ! -s "$2.new" ]; then + echo "logging.tls: failed to obtain a certificate from the confluent CA, skipping log forwarding setup" + rm -f $1.new $2.new $3.new + return 1 + fi + chmod 600 $1.new + chmod 644 $2.new $3.new + if [ -x /usr/sbin/restorecon ]; then + /usr/sbin/restorecon $1.new $2.new $3.new 2>/dev/null + fi + mv $1.new $1 + mv $2.new $2 + mv $3.new $3 + return 0 +} + +if [ "$loggingmethod" = journal-remote ]; then + # drop forwarding state left by a previous rsyslog-method configuration + if [ -f /etc/rsyslog.d/60-confluent.conf ]; then + rm -f /etc/rsyslog.d/60-confluent.conf + if [ -d /run/systemd/system ]; then + systemctl try-restart rsyslog.service 2>/dev/null + fi + fi + uploadbin="" + for candidate in /usr/lib/systemd/systemd-journal-upload /lib/systemd/systemd-journal-upload; do + if [ -x $candidate ]; then + uploadbin=$candidate + break + fi + done + if [ -z "$uploadbin" ]; then + echo "systemd-journal-upload does not appear to be installed, skipping log forwarding setup" + exit 0 + fi + loggingsrv=$(echo "$loggingservers" | sed -n 1p) + if [ "$(echo "$loggingservers" | wc -l)" -gt 1 ]; then + echo "journal-remote supports only a single destination, forwarding to $loggingsrv and ignoring the other logging.servers entries" + fi + case $loggingsrv in + *:*) loggingsrv="[$loggingsrv]" ;; # bracket IPv6 literals for the URL + esac + mkdir -p /etc/systemd/journal-upload.conf.d + journalconf=/etc/systemd/journal-upload.conf.d/confluent.conf + echo '# Log forwarding configured by confluent during deployment' > $journalconf + echo '[Upload]' >> $journalconf + if [ "$loggingtls" = true ]; then + tlsdir=/etc/ssl/confluent-logging + if ! request_tls_cert $tlsdir/journal-upload.key $tlsdir/journal-upload.crt $tlsdir/ca.pem; then + rm -f $journalconf + exit 0 + fi + chgrp systemd-journal $tlsdir/journal-upload.key 2>/dev/null + chmod g+r $tlsdir/journal-upload.key + setup_cert_renewal $tlsdir/journal-upload.key $tlsdir/journal-upload.crt systemd-journal systemd-journal-upload.service + echo 'URL=https://'$loggingsrv':19532' >> $journalconf + echo 'ServerKeyFile='$tlsdir'/journal-upload.key' >> $journalconf + echo 'ServerCertificateFile='$tlsdir'/journal-upload.crt' >> $journalconf + echo 'TrustedCertificateFile='$tlsdir'/ca.pem' >> $journalconf + else + echo 'URL=http://'$loggingsrv':19532' >> $journalconf + fi + if [ -x /usr/sbin/restorecon ]; then + /usr/sbin/restorecon $journalconf 2>/dev/null + fi + systemctl enable systemd-journal-upload.service 2>/dev/null + if [ -d /run/systemd/system ]; then + systemctl restart systemd-journal-upload.service 2>/dev/null + fi +else + # drop forwarding state left by a previous journal-remote-method + # configuration + if [ -f /etc/systemd/journal-upload.conf.d/confluent.conf ]; then + rm -f /etc/systemd/journal-upload.conf.d/confluent.conf + systemctl disable systemd-journal-upload.service 2>/dev/null + if [ -d /run/systemd/system ]; then + systemctl stop systemd-journal-upload.service 2>/dev/null + fi + fi + if [ ! -d /etc/rsyslog.d ]; then + echo "rsyslog does not appear to be installed, skipping log forwarding setup" + exit 0 + fi + syslogconf=/etc/rsyslog.d/60-confluent.conf + if [ "$loggingtls" = true ]; then + # forwarding with TLS needs a network stream driver module + # (rsyslog-openssl or rsyslog-gnutls) + tlsdriver="" + for moddir in /usr/lib64/rsyslog /usr/lib/rsyslog /usr/lib/*-linux-gnu/rsyslog; do + [ -f $moddir/lmnsd_ossl.so ] && tlsdriver=ossl && break + [ -f $moddir/lmnsd_gtls.so ] && tlsdriver=gtls && break + done + if [ -z "$tlsdriver" ]; then + echo "logging.tls requires the rsyslog-openssl or rsyslog-gnutls module on the node, skipping log forwarding setup" + exit 0 + fi + tlsdir=/etc/rsyslog.d/confluent-tls + if ! request_tls_cert $tlsdir/rsyslog.key $tlsdir/rsyslog.crt $tlsdir/ca.pem; then + exit 0 + fi + rsyslogkeygroup="" + if getent passwd syslog > /dev/null 2>&1; then + # e.g. Ubuntu runs rsyslogd as the syslog user + rsyslogkeygroup=syslog + chgrp syslog $tlsdir/rsyslog.key 2>/dev/null + chmod g+r $tlsdir/rsyslog.key + fi + setup_cert_renewal $tlsdir/rsyslog.key $tlsdir/rsyslog.crt "$rsyslogkeygroup" rsyslog.service + echo '# Syslog forwarding configured by confluent during deployment' > $syslogconf + echo 'global(DefaultNetstreamDriver="'$tlsdriver'"' >> $syslogconf + echo ' DefaultNetstreamDriverCAFile="'$tlsdir'/ca.pem"' >> $syslogconf + echo ' DefaultNetstreamDriverCertFile="'$tlsdir'/rsyslog.crt"' >> $syslogconf + echo ' DefaultNetstreamDriverKeyFile="'$tlsdir'/rsyslog.key")' >> $syslogconf + # For name-valued targets, pin the server identity (x509/name): the + # confluent CA also issues certificates to every node and BMC, and + # with chain validation alone any of them could pose as the server. + # rsyslog matches pinned peers only against DNS-type SANs, so IP + # targets can only validate the chain (x509/certvalid). + # The bounded async queue keeps an unreachable or hung log server + # from wedging the main queue (and with it local logging); once it + # fills, further forwarded messages are dropped instead. + fwdqueue='action.resumeRetryCount="-1" queue.type="linkedList" queue.size="10000"' + for loggingsrv in $loggingservers; do + case $loggingsrv in + *:*) isname=0 ;; # IPv6 literal + *[!0-9.]*) isname=1 ;; # anything besides digits and dots + *) isname=0 ;; # IPv4 literal + esac + if [ $isname = 1 ]; then + echo '*.* action(type="omfwd" target="'$loggingsrv'" port="6514" protocol="tcp" StreamDriverMode="1" StreamDriverAuthMode="x509/name" StreamDriverPermittedPeers="'$loggingsrv'" '"$fwdqueue"')' >> $syslogconf + else + echo '*.* action(type="omfwd" target="'$loggingsrv'" port="6514" protocol="tcp" StreamDriverMode="1" StreamDriverAuthMode="x509/certvalid" '"$fwdqueue"')' >> $syslogconf + fi + done + else + # Plain TCP forwarding (no encryption) + fwdqueue='action.resumeRetryCount="-1" queue.type="linkedList" queue.size="10000"' + echo '# Syslog forwarding configured by confluent during deployment' > $syslogconf + for loggingsrv in $loggingservers; do + echo '*.* action(type="omfwd" target="'$loggingsrv'" port="514" protocol="tcp" '"$fwdqueue"')' >> $syslogconf + done + fi + if [ -x /usr/sbin/restorecon ]; then + /usr/sbin/restorecon $syslogconf 2>/dev/null + fi + if [ -d /run/systemd/system ]; then + systemctl try-restart rsyslog.service 2>/dev/null + fi +fi +exit 0 diff --git a/confluent_osdeploy/debian/profiles/default/scripts/post.sh b/confluent_osdeploy/debian/profiles/default/scripts/post.sh index f30d980f9..130d13c10 100755 --- a/confluent_osdeploy/debian/profiles/default/scripts/post.sh +++ b/confluent_osdeploy/debian/profiles/default/scripts/post.sh @@ -59,6 +59,7 @@ if [ -e /sys/firmware/efi ]; then efibootmgr -D fi fi +run_remote setuplogging run_remote_python syncfileclient run_remote_parts post.d run_remote_config post diff --git a/confluent_osdeploy/el7-diskless/profiles/default/scripts/onboot.sh b/confluent_osdeploy/el7-diskless/profiles/default/scripts/onboot.sh index 818fb3d07..862e163c7 100644 --- a/confluent_osdeploy/el7-diskless/profiles/default/scripts/onboot.sh +++ b/confluent_osdeploy/el7-diskless/profiles/default/scripts/onboot.sh @@ -39,6 +39,7 @@ chmod 600 /var/log/confluent/confluent-onboot.log tail -f /var/log/confluent/confluent-onboot.log > /dev/console & logshowpid=$! +run_remote setuplogging run_remote_python syncfileclient run_remote_python confignet diff --git a/confluent_osdeploy/el7/profiles/default/scripts/post.sh b/confluent_osdeploy/el7/profiles/default/scripts/post.sh index 362a79454..9dc7d2584 100644 --- a/confluent_osdeploy/el7/profiles/default/scripts/post.sh +++ b/confluent_osdeploy/el7/profiles/default/scripts/post.sh @@ -34,6 +34,7 @@ run_remote_python add_local_repositories # run_remote_python will use the appropriate python interpreter path to run the specified script # A post.custom is provided to more conveniently hold customizations, see the post.custom file. +run_remote setuplogging # This will induce server side processing of the syncfile contents if # present run_remote_python syncfileclient diff --git a/confluent_osdeploy/el8-diskless/profiles/default/scripts/onboot.sh b/confluent_osdeploy/el8-diskless/profiles/default/scripts/onboot.sh index d2d50a7d3..2d915ef43 100644 --- a/confluent_osdeploy/el8-diskless/profiles/default/scripts/onboot.sh +++ b/confluent_osdeploy/el8-diskless/profiles/default/scripts/onboot.sh @@ -51,6 +51,7 @@ logshowpid=$! rpm --import /etc/pki/rpm-gpg/* run_remote_python add_local_repositories +run_remote setuplogging run_remote_python syncfileclient run_remote_python confignet -c $confluent_mgr diff --git a/confluent_osdeploy/el8/profiles/default/scripts/post.sh b/confluent_osdeploy/el8/profiles/default/scripts/post.sh index db1c511d8..666494225 100644 --- a/confluent_osdeploy/el8/profiles/default/scripts/post.sh +++ b/confluent_osdeploy/el8/profiles/default/scripts/post.sh @@ -39,6 +39,7 @@ run_remote_python add_local_repositories run_remote_python autoconsole +run_remote setuplogging # This will induce server side processing of the syncfile contents if # present run_remote_python syncfileclient diff --git a/confluent_osdeploy/el9-diskless/profiles/default/scripts/onboot.sh b/confluent_osdeploy/el9-diskless/profiles/default/scripts/onboot.sh index 44e972356..3ebfc767a 100644 --- a/confluent_osdeploy/el9-diskless/profiles/default/scripts/onboot.sh +++ b/confluent_osdeploy/el9-diskless/profiles/default/scripts/onboot.sh @@ -45,6 +45,7 @@ logshowpid=$! rpm --import /etc/pki/rpm-gpg/* run_remote_python add_local_repositories +run_remote setuplogging run_remote_python syncfileclient run_remote_python confignet -c $confluent_mgr diff --git a/confluent_osdeploy/suse15-diskless/profiles/default/scripts/onboot.sh b/confluent_osdeploy/suse15-diskless/profiles/default/scripts/onboot.sh index 985d1c026..29befc3b9 100644 --- a/confluent_osdeploy/suse15-diskless/profiles/default/scripts/onboot.sh +++ b/confluent_osdeploy/suse15-diskless/profiles/default/scripts/onboot.sh @@ -20,6 +20,7 @@ chmod 600 /var/log/confluent/confluent-onboot.log tail -f /var/log/confluent/confluent-onboot.log > /dev/console & logshowpid=$! +run_remote setuplogging run_remote_python syncfileclient run_remote_python confignet run_remote onboot.custom diff --git a/confluent_osdeploy/suse15/profiles/hpc/scripts/post.sh b/confluent_osdeploy/suse15/profiles/hpc/scripts/post.sh index be4e2d80d..8f5917742 100644 --- a/confluent_osdeploy/suse15/profiles/hpc/scripts/post.sh +++ b/confluent_osdeploy/suse15/profiles/hpc/scripts/post.sh @@ -25,6 +25,7 @@ chmod og-rwx /etc/confluent/* export confluent_mgr confluent_profile nodename . /etc/confluent/functions +run_remote setuplogging # This will induce server side processing of the syncfile contents if # present run_remote_python syncfileclient diff --git a/confluent_osdeploy/suse15/profiles/server/scripts/post.sh b/confluent_osdeploy/suse15/profiles/server/scripts/post.sh index be4e2d80d..8f5917742 100644 --- a/confluent_osdeploy/suse15/profiles/server/scripts/post.sh +++ b/confluent_osdeploy/suse15/profiles/server/scripts/post.sh @@ -25,6 +25,7 @@ chmod og-rwx /etc/confluent/* export confluent_mgr confluent_profile nodename . /etc/confluent/functions +run_remote setuplogging # This will induce server side processing of the syncfile contents if # present run_remote_python syncfileclient diff --git a/confluent_osdeploy/suse16/profiles/server/scripts/post.sh b/confluent_osdeploy/suse16/profiles/server/scripts/post.sh index fd54308da..76c8134d1 100644 --- a/confluent_osdeploy/suse16/profiles/server/scripts/post.sh +++ b/confluent_osdeploy/suse16/profiles/server/scripts/post.sh @@ -29,6 +29,7 @@ chmod og-rwx /etc/confluent/* export confluent_mgr confluent_profile nodename . /etc/confluent/functions +run_remote setuplogging # This will induce server side processing of the syncfile contents if # present run_remote_python syncfileclient diff --git a/confluent_osdeploy/ubuntu18.04/profiles/default/scripts/post.sh b/confluent_osdeploy/ubuntu18.04/profiles/default/scripts/post.sh index f30d980f9..130d13c10 100755 --- a/confluent_osdeploy/ubuntu18.04/profiles/default/scripts/post.sh +++ b/confluent_osdeploy/ubuntu18.04/profiles/default/scripts/post.sh @@ -59,6 +59,7 @@ if [ -e /sys/firmware/efi ]; then efibootmgr -D fi fi +run_remote setuplogging run_remote_python syncfileclient run_remote_parts post.d run_remote_config post diff --git a/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/onboot.sh b/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/onboot.sh index cc470d6fa..f887c4822 100644 --- a/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/onboot.sh +++ b/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/onboot.sh @@ -27,6 +27,7 @@ chmod 600 /var/log/confluent/confluent-onboot.log tail -f /var/log/confluent/confluent-onboot.log > /dev/console & logshowpid=$! +run_remote setuplogging run_remote_python syncfileclient run_remote_python confignet diff --git a/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/post.sh b/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/post.sh index d97308893..071e58eef 100755 --- a/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/post.sh +++ b/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/post.sh @@ -82,6 +82,7 @@ cat /target/etc/confluent/tls/*.pem > /target/etc/confluent/ca.pem cat /target/etc/confluent/tls/*.pem > /target/usr/local/share/ca-certificates/confluent.crt cat /target/etc/confluent/tls/*.pem > /etc/confluent/ca.pem chroot /target update-ca-certificates +chroot /target bash -c "source /etc/confluent/functions; run_remote setuplogging" chroot /target bash -c "source /etc/confluent/functions; run_remote_python syncfileclient" chroot /target bash -c "source /etc/confluent/functions; run_remote_python confignet" chroot /target bash -c "source /etc/confluent/functions; run_remote_parts post.d" diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh index 78e6b411b..47e243ada 100755 --- a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh @@ -88,6 +88,7 @@ chroot /target update-ca-certificates # Ubuntu mangles grub function for serial users, undo that mangling chroot /target bash -c "source /etc/confluent/functions; run_remote_python autoconsole" +chroot /target bash -c "source /etc/confluent/functions; run_remote setuplogging" chroot /target bash -c "source /etc/confluent/functions; run_remote_python syncfileclient" chroot /target bash -c "source /etc/confluent/functions; run_remote_parts post.d" source /target/etc/confluent/functions diff --git a/confluent_server/MANIFEST.in b/confluent_server/MANIFEST.in index b71e09982..ac4d307db 100644 --- a/confluent_server/MANIFEST.in +++ b/confluent_server/MANIFEST.in @@ -2,3 +2,6 @@ include pam/* include sysvinit/* include systemd/* include sysctl/* +include logging/* +include logging/rsyslog/* +include logging/journal-remote/* diff --git a/confluent_server/confluent/config/attributes.py b/confluent_server/confluent/config/attributes.py index 06f430ed7..85580cd01 100644 --- a/confluent_server/confluent/config/attributes.py +++ b/confluent_server/confluent/config/attributes.py @@ -705,4 +705,53 @@ 'dns.servers': { 'description': 'DNS Server or servers to provide to node', }, + 'logging.servers': { + 'description': 'Remote logging server or servers (comma separated) ' + 'for the OS deployment to configure the node to ' + 'forward logs to, using the method indicated by ' + 'logging.method. May be set to the deployment server ' + 'or any other logging server. With logging.tls and ' + 'the rsyslog method, prefer DNS names resolvable by ' + 'the nodes over IP addresses: a name lets the node ' + 'verify the specific server identity, while an IP ' + 'only validates that the certificate came from the ' + 'confluent CA. See ' + '/opt/confluent/share/examples/logging for cfg ' + 'examples. If unset, no forwarding is configured.', + }, + 'logging.method': { + 'description': 'Method used to forward logs to logging.servers. ' + '"rsyslog" (the default if unset) forwards syslog ' + 'using rsyslog over TCP port 514. "journal-remote" ' + 'uploads the systemd journal using ' + 'systemd-journal-upload to systemd-journal-remote on ' + 'port 19532; note that systemd-journal-upload supports ' + 'only a single destination, so only the first entry of ' + 'logging.servers is used.', + 'validvalues': ('rsyslog', 'journal-remote', ''), + }, + 'logging.tls': { + 'description': 'Whether to encrypt the forwarded logs with TLS, with ' + 'node and logging server mutually authenticated via ' + 'the confluent TLS certificate authority. With ' + 'rsyslog, logs are forwarded over TCP port 6514 and ' + 'the node needs the rsyslog-openssl or rsyslog-gnutls ' + 'module; with journal-remote, over HTTPS port 19532. ' + 'Caution: some distributions build ' + 'systemd-journal-remote without client certificate ' + 'verification (e.g. EL10 and Ubuntu). ' + 'The transport is still encrypted and ' + 'the node verifies the server, but such receivers ' + 'accept uploads from any client; use the rsyslog ' + 'method when client verification is required. ' + 'The node certificate is issued during deployment ' + 'with validity governed by pubkeys.tls_lifetime ' + '(default 47 days) and renewed automatically: a ' + 'daily randomized systemd timer on the node requests ' + 'a fresh certificate once less than half of the ' + 'validity remains. To set up the receiving side, see the ' + 'confluent-logging-receiver-setup helper under ' + '/opt/confluent/share/examples/logging.', + 'type': bool, + }, } diff --git a/confluent_server/confluent/selfservice.py b/confluent_server/confluent/selfservice.py index c9499a7f1..1f7f8bf23 100644 --- a/confluent_server/confluent/selfservice.py +++ b/confluent_server/confluent/selfservice.py @@ -295,7 +295,7 @@ def verify_cert(certificate): ncfg['ipv4_method'] = 'static' deployinfo = cfg.get_node_attributes( nodename, ('deployment.*', 'console.method', 'crypted.*', - 'dns.*', 'ntp.*')) + 'dns.*', 'ntp.*', 'logging.*')) deployinfo = deployinfo.get(nodename, {}) profile = deployinfo.get( 'deployment.pendingprofile', {}).get('value', '') @@ -405,6 +405,17 @@ def verify_cert(certificate): ncfg['ntpservers'] = [] for ntpsrv in ntpsrvs: ncfg['ntpservers'].append(ntpsrv) + logsrvs = deployinfo.get('logging.servers', {}).get('value', '') + if logsrvs: + logsrvs = logsrvs.split(',') + if logsrvs: + ncfg['loggingservers'] = [] + for logsrv in logsrvs: + ncfg['loggingservers'].append(logsrv) + ncfg['loggingmethod'] = deployinfo.get( + 'logging.method', {}).get('value', '') or 'rsyslog' + ncfg['loggingtls'] = bool(deployinfo.get( + 'logging.tls', {}).get('value', False)) dnsdomain = deployinfo.get('dns.domain', {}).get('value', None) ncfg['dnsdomain'] = dnsdomain return await make_response(mimetype, 200, 'OK', body=dumper(ncfg)) diff --git a/confluent_server/logging/confluent-logging-receiver-setup b/confluent_server/logging/confluent-logging-receiver-setup new file mode 100755 index 000000000..3a708e002 --- /dev/null +++ b/confluent_server/logging/confluent-logging-receiver-setup @@ -0,0 +1,170 @@ +#!/bin/bash +# Set up a confluent server to receive logs forwarded by deployed nodes +# (see the logging.servers, logging.method and logging.tls node attributes). +# +# Usage: sh confluent-logging-receiver-setup [--tls|--plain] +# +# With --tls (the default), the transport is encrypted and mutually +# authenticated via the confluent certificate authority: a server certificate +# is issued from the local confluent CA and only clients presenting a +# certificate from that CA are accepted. With --plain, the plaintext receiver +# examples are installed instead (rsyslog TCP port 514, journal-remote HTTP +# port 19532). +# +# rsyslog reception writes per-node logs under /var/log/confluent/nodes/ +# (see confluent-nodes.logrotate for log rotation); journal-remote reception +# writes journals under /var/log/journal/remote/, readable with +# journalctl -D /var/log/journal/remote +# +# Safe to re-run; regenerates the certificate and overwrites the +# configuration it owns. + +set -e +cd "$(dirname "$0")" + +method="" +tls=1 +for arg in "$@"; do + case $arg in + rsyslog|journal-remote) method=$arg;; + --tls) tls=1;; + --plain) tls=0;; + *) echo "Unknown argument: $arg"; exit 1;; + esac +done +if [ -z "$method" ]; then + echo "Usage: sh confluent-logging-receiver-setup [--tls|--plain]" + exit 1 +fi + +# issue_tls_cert : generate a key and certificate +# signed by the local confluent CA, with SANs covering this server's names and +# addresses. A dedicated directory is used per consumer: /etc/ssl/private is +# not traversable by service users on Debian-style systems, and Ubuntu's +# rsyslogd AppArmor profile only permits reads under /etc/rsyslog.d +issue_tls_cert() { + mkdir -p $(dirname $1) + # Store the confluent CA certs into the trusted bundle used to + # authenticate clients + cat /var/lib/confluent/public/site/tls/*.pem > $3 + myips=$(ip -j addr show 2>/dev/null | python3 -c 'import json,sys; print(",".join(a["local"] for i in json.load(sys.stdin) for a in i.get("addr_info", []) if a.get("scope") == "global"))') + mynames=$(hostname) + mydomain=$(dnsdomainname 2>/dev/null) + [ -n "$mydomain" ] && mynames=$mynames,$(hostname).$mydomain + [ -n "$myips" ] && mynames=$mynames,$myips + TDIR=$(mktemp -d) + # certutil runs the actual signing as the owner of /etc/confluent, which + # must be able to reach the CSR; the directory never holds the key + chmod 711 $TDIR + # Request a new certificate while generating a new key to go with it, + # the subject is replaced during signing + openssl req -new -newkey ec -pkeyopt ec_paramgen_curve:secp384r1 -nodes \ + -keyout $1 -out $TDIR/csr.pem -subj /CN=$(hostname) > /dev/null 2>&1 + # Have the confluent CA do an offline signing of the CSR (default + # validity of 3650 days) + (cd $TDIR && python3 /opt/confluent/lib/python/confluent/certutil.py -s $mynames -r csr.pem) + cp $TDIR/cert.pem $2 + rm -rf $TDIR + chmod 600 $1 + chmod 644 $2 $3 + if [ -x /usr/sbin/restorecon ]; then + /usr/sbin/restorecon $1 $2 $3 2>/dev/null || true + fi +} + +if [ $method = journal-remote ]; then + jrbin="" + for candidate in /usr/lib/systemd/systemd-journal-remote /lib/systemd/systemd-journal-remote; do + [ -f $candidate ] && jrbin=$candidate && break + done + if [ -z "$jrbin" ]; then + echo "systemd-journal-remote does not appear to be installed (install the systemd-journal-remote package)" + exit 1 + fi + if [ $tls = 1 ]; then + # builds without gnutls reject --trust with "not available"; the exit + # status alone cannot be used, as gnutls builds also fail (with a + # conflict complaint) once a TrustedCertificateFile is configured + if $jrbin --trust=all --version 2>&1 | grep -qi 'not available'; then + echo "WARNING: this build of systemd-journal-remote was compiled without client" + echo "WARNING: certificate verification (its TrustedCertificateFile setting is" + echo "WARNING: silently ignored). The transport will be encrypted and nodes will" + echo "WARNING: verify this server, but uploads from ANY client will be accepted." + echo "WARNING: Use logging.method=rsyslog if client verification is required." + fi + tlsdir=/etc/ssl/confluent-logging + issue_tls_cert $tlsdir/journal-remote.key $tlsdir/journal-remote.crt $tlsdir/ca.pem + chown systemd-journal-remote $tlsdir/journal-remote.key + # Setting TrustedCertificateFile explicitly is what makes + # systemd-journal-remote verify client certificates; do not rely on + # the compiled-in defaults + mkdir -p /etc/systemd/journal-remote.conf.d + cat > /etc/systemd/journal-remote.conf.d/confluent.conf << 'EOF' +# Log reception configured by confluent-logging-receiver-setup +# Note: TrustedCertificateFile (client certificate verification) is silently +# ignored by systemd-journal-remote builds lacking gnutls support +[Remote] +ServerKeyFile=/etc/ssl/confluent-logging/journal-remote.key +ServerCertificateFile=/etc/ssl/confluent-logging/journal-remote.crt +TrustedCertificateFile=/etc/ssl/confluent-logging/ca.pem +EOF + # remove a plain-HTTP override from a previous --plain run + rm -f /etc/systemd/system/systemd-journal-remote.service.d/confluent.conf + else + mkdir -p /etc/systemd/system/systemd-journal-remote.service.d + cp journal-remote/confluent-journal-remote.conf /etc/systemd/system/systemd-journal-remote.service.d/confluent.conf + rm -f /etc/systemd/journal-remote.conf.d/confluent.conf + fi + systemctl daemon-reload + systemctl enable systemd-journal-remote.socket > /dev/null 2>&1 + systemctl try-restart systemd-journal-remote.service 2>/dev/null || true + systemctl restart systemd-journal-remote.socket + echo "journal-remote reception configured; permit TCP port 19532 through the firewall, e.g.:" + echo " firewall-cmd --permanent --add-port=19532/tcp && firewall-cmd --reload" +else + if [ ! -d /etc/rsyslog.d ]; then + echo "rsyslog does not appear to be installed" + exit 1 + fi + if [ $tls = 1 ]; then + tlsdriver="" + for moddir in /usr/lib64/rsyslog /usr/lib/rsyslog /usr/lib/*-linux-gnu/rsyslog; do + [ -f $moddir/lmnsd_ossl.so ] && tlsdriver=ossl && break + [ -f $moddir/lmnsd_gtls.so ] && tlsdriver=gtls && break + done + if [ -z "$tlsdriver" ]; then + echo "TLS reception requires the rsyslog-openssl or rsyslog-gnutls module" + exit 1 + fi + tlsdir=/etc/rsyslog.d/confluent-tls + issue_tls_cert $tlsdir/rsyslog.key $tlsdir/rsyslog.crt $tlsdir/ca.pem + if getent passwd syslog > /dev/null 2>&1; then + # e.g. Ubuntu runs rsyslogd as the syslog user + chgrp syslog $tlsdir/rsyslog.key + chmod g+r $tlsdir/rsyslog.key + fi + # spread the TLS sessions over several input workers where the + # imtcp workerthreads parameter exists (>=8.2504.0) + # has it); probe the local rsyslogd rather than track versions + workerthreads="" + if ! echo 'module(load="imtcp" workerthreads="4")' | rsyslogd -N1 -f /dev/stdin 2>&1 | grep -q 'not known'; then + workerthreads=' workerthreads="4"' + fi + sed -e 's/"ossl"/"'$tlsdriver'"/' -e 's/keepAlive="on"/&'"$workerthreads"'/' rsyslog/confluent-nodes-tls.conf > /etc/rsyslog.d/confluent-nodes.conf + echo "TLS rsyslog reception configured; permit TCP port 6514 through the firewall, e.g.:" + echo " firewall-cmd --permanent --add-port=6514/tcp && firewall-cmd --reload" + else + cp rsyslog/confluent-nodes.conf /etc/rsyslog.d/confluent-nodes.conf + echo "Plain rsyslog reception configured; permit TCP port 514 through the firewall, e.g.:" + echo " firewall-cmd --permanent --add-port=514/tcp && firewall-cmd --reload" + fi + if [ -x /usr/sbin/restorecon ]; then + /usr/sbin/restorecon /etc/rsyslog.d/confluent-nodes.conf 2>/dev/null || true + fi + mkdir -p /var/log/confluent/nodes + if getent passwd syslog > /dev/null 2>&1; then + # e.g. Ubuntu runs rsyslogd as the syslog user + chown syslog /var/log/confluent/nodes + fi + systemctl restart rsyslog +fi diff --git a/confluent_server/logging/journal-remote/confluent-journal-remote.conf b/confluent_server/logging/journal-remote/confluent-journal-remote.conf new file mode 100644 index 000000000..15cd14356 --- /dev/null +++ b/confluent_server/logging/journal-remote/confluent-journal-remote.conf @@ -0,0 +1,23 @@ +# Example systemd-journal-remote configuration for a confluent server to +# receive journal entries uploaded by deployed nodes (see the logging.servers +# and logging.method node attributes). The stock systemd-journal-remote unit +# only listens for HTTPS; this drop-in switches it to plain HTTP on port +# 19532, matching what systemd-journal-upload is configured for on the nodes. +# +# To activate (or run the confluent-logging-receiver-setup helper one +# directory up with "journal-remote --plain"): +# dnf/zypper/apt install systemd-journal-remote +# mkdir -p /etc/systemd/system/systemd-journal-remote.service.d +# cp confluent-journal-remote.conf /etc/systemd/system/systemd-journal-remote.service.d/confluent.conf +# systemctl daemon-reload +# systemctl enable --now systemd-journal-remote.socket +# and permit the port through the firewall, e.g. on a firewalld based system: +# firewall-cmd --permanent --add-port=19532/tcp && firewall-cmd --reload +# +# Received entries are written to +# /var/log/journal/remote/remote-.journal and can be read with: +# journalctl -D /var/log/journal/remote + +[Service] +ExecStart= +ExecStart=/usr/lib/systemd/systemd-journal-remote --listen-http=-3 --output=/var/log/journal/remote/ diff --git a/confluent_server/logging/rsyslog/confluent-nodes-tls.conf b/confluent_server/logging/rsyslog/confluent-nodes-tls.conf new file mode 100644 index 000000000..e1095514d --- /dev/null +++ b/confluent_server/logging/rsyslog/confluent-nodes-tls.conf @@ -0,0 +1,67 @@ +# Example rsyslog configuration for a confluent server to receive TLS +# encrypted syslog forwarded by deployed nodes (see the logging.servers and +# logging.tls node attributes) and write per-node logs under +# /var/log/confluent/nodes/. Clients are mutually authenticated: only nodes +# presenting a certificate issued by the confluent certificate authority are +# accepted. +# +# The confluent-logging-receiver-setup helper in this examples directory +# generates the referenced certificate material and installs this file; to +# activate manually, provide the key/cert/CA files below, then: +# cp confluent-nodes-tls.conf /etc/rsyslog.d/confluent-nodes.conf +# systemctl restart rsyslog +# and permit TCP port 6514 through the firewall, e.g. on a firewalld based +# system: +# firewall-cmd --permanent --add-port=6514/tcp && firewall-cmd --reload +# +# The TLS stream driver requires the rsyslog-openssl (driver "ossl") or +# rsyslog-gnutls (driver "gtls") module to be installed. +# +# Note: on systems where rsyslogd does not run as root (e.g. Ubuntu runs it +# as the "syslog" user), ensure that user may read the certificate material +# and may create and write /var/log/confluent/nodes +# +# The per-node logs can grow large; see confluent-nodes.logrotate for an +# example logrotate configuration that rotates and compresses them. +# +# Logs are filed by the sender identity as determined by this server +# (resolution of the peer address, falling back to the address itself), not +# by the client-supplied hostname, so nodes cannot inject entries into each +# other's logs. Note that any client presenting a certificate from the +# confluent CA is accepted. + +global(DefaultNetstreamDriver="ossl" + DefaultNetstreamDriverCAFile="/etc/rsyslog.d/confluent-tls/ca.pem" + DefaultNetstreamDriverCertFile="/etc/rsyslog.d/confluent-tls/rsyslog.crt" + DefaultNetstreamDriverKeyFile="/etc/rsyslog.d/confluent-tls/rsyslog.key") + +# Every node holds a persistent TLS connection: keep maxSessions at or +# above the node count, or rsyslog refuses the connections beyond it (the +# default is 200). keepAlive reaps the sessions of crashed or reset nodes so +# they do not accumulate against the limit. On rsyslog builds recent enough +# to know the imtcp workerthreads parameter (>=8.2504.0), +# adding e.g. workerthreads="4" additionally spreads the TLS sessions over +# several input workers; confluent-logging-receiver-setup adds it when the +# local rsyslogd accepts it. +module(load="imtcp" StreamDriver.Name="ossl" StreamDriver.Mode="1" + StreamDriver.AuthMode="x509/certvalid" + maxSessions="4096" keepAlive="on") +input(type="imtcp" port="6514" ruleset="confluentnodes") + +template(name="confluentnodelog" type="string" + string="/var/log/confluent/nodes/%FROMHOST%.log") + +# The dedicated queue keeps bursts from the nodes and this server's own +# logging from delaying each other, and buffers reconnect storms after a +# restart. The dynafile handle cache must stay at or above the node count +# (the default of 10 would thrash with logs interleaving from many per-node +# files, closing and reopening one for nearly every message); handles and +# their io buffers are only consumed for files actually open. Async batched +# writing means a node's file may trail the sender by up to ~2s when +# tailed live. +ruleset(name="confluentnodes" queue.type="fixedArray" queue.size="50000") { + action(type="omfile" dynaFile="confluentnodelog" + dynaFileCacheSize="4096" asyncWriting="on" ioBufferSize="64k" + flushOnTXEnd="off" flushInterval="2" + dirCreateMode="0700" fileCreateMode="0600") +} diff --git a/confluent_server/logging/rsyslog/confluent-nodes.conf b/confluent_server/logging/rsyslog/confluent-nodes.conf new file mode 100644 index 000000000..c7eeed1ff --- /dev/null +++ b/confluent_server/logging/rsyslog/confluent-nodes.conf @@ -0,0 +1,43 @@ +# Example rsyslog configuration for a confluent server to receive syslog +# forwarded by deployed nodes (see the logging.servers node attribute) and +# write per-node logs under /var/log/confluent/nodes/ +# +# To activate: +# cp confluent-nodes.conf /etc/rsyslog.d/confluent-nodes.conf +# systemctl restart rsyslog +# and permit plain-TCP syslog through the firewall, e.g. on a firewalld based +# system: +# firewall-cmd --permanent --add-port=514/tcp && firewall-cmd --reload +# +# Note: on systems where rsyslogd does not run as root (e.g. Ubuntu runs it +# as the "syslog" user), ensure that user may create and write +# /var/log/confluent/nodes +# +# The per-node logs can grow large; see confluent-nodes.logrotate for an +# example logrotate configuration that rotates and compresses them. +# +# Logs are filed by the sender identity as determined by this server +# (resolution of the peer address, falling back to the address itself), not +# by the client-supplied hostname, so nodes cannot trivially inject entries +# into each other's logs. + +# Nodes forward over plain TCP: each node holds one persistent connection. +module(load="imptcp" threads="4") +input(type="imptcp" port="514" ruleset="confluentnodes" keepAlive="on") + +template(name="confluentnodelog" type="string" + string="/var/log/confluent/nodes/%FROMHOST%.log") + +# The dedicated queue keeps bursts from the nodes and this server's own +# logging from delaying each other. The dynafile handle cache must stay at +# or above the node count (the default of 10 would thrash with logs +# interleaving from many per-node files, closing and reopening one for +# nearly every message); handles and their io buffers are only consumed for +# files actually open. Async batched writing means a node's file may trail +# the sender by up to ~2s when tailed live. +ruleset(name="confluentnodes" queue.type="fixedArray" queue.size="50000") { + action(type="omfile" dynaFile="confluentnodelog" + dynaFileCacheSize="4096" asyncWriting="on" ioBufferSize="64k" + flushOnTXEnd="off" flushInterval="2" + dirCreateMode="0700" fileCreateMode="0600") +} diff --git a/confluent_server/logging/rsyslog/confluent-nodes.logrotate b/confluent_server/logging/rsyslog/confluent-nodes.logrotate new file mode 100644 index 000000000..8b4937724 --- /dev/null +++ b/confluent_server/logging/rsyslog/confluent-nodes.logrotate @@ -0,0 +1,21 @@ +# Example logrotate configuration for the per-node logs written by the +# confluent-nodes.conf rsyslog example under /var/log/confluent/nodes/ +# +# To activate: +# cp confluent-nodes.logrotate /etc/logrotate.d/confluent-nodes +# +# The following settings rotate logs once a week for up to 52 weeks (1 year) +# and compress all logs older than 2 weeks + +/var/log/confluent/nodes/*.log { + weekly + rotate 52 + compress + delaycompress + missingok + notifempty + sharedscripts + postrotate + /usr/bin/systemctl kill -s HUP rsyslog.service >/dev/null 2>&1 || true + endscript +} diff --git a/confluent_server/setup.py.tmpl b/confluent_server/setup.py.tmpl index fc481798d..b11bb4a0d 100644 --- a/confluent_server/setup.py.tmpl +++ b/confluent_server/setup.py.tmpl @@ -40,6 +40,11 @@ setup( data_files=[('/etc/init.d', ['sysvinit/confluent']), ('/usr/lib/sysctl.d', ['sysctl/confluent.conf']), ('/opt/confluent/share/licenses/confluent_server', ['LICENSE', 'COPYRIGHT']), + ('/opt/confluent/share/examples/logging', ['logging/confluent-logging-receiver-setup']), + ('/opt/confluent/share/examples/logging/rsyslog', ['logging/rsyslog/confluent-nodes.conf', + 'logging/rsyslog/confluent-nodes-tls.conf', + 'logging/rsyslog/confluent-nodes.logrotate']), + ('/opt/confluent/share/examples/logging/journal-remote', ['logging/journal-remote/confluent-journal-remote.conf']), ('/usr/lib/systemd/system', ['systemd/confluent.service']), ('/opt/confluent/lib/python/confluent/plugins/console/', [])],